23 std::uint64_t fnv1a64(
const std::string& s)
25 std::uint64_t hash = 14695981039346656037ull;
26 for (
const char c : s)
28 hash ^=
static_cast<std::uint64_t
>(
static_cast<unsigned char>(c));
29 hash *= 1099511628211ull;
36 std::string sanitize_name(
const std::string& designName)
39 out.reserve(designName.size() < 48 ? designName.size() : 48);
40 for (
const char c : designName)
42 const bool ok = (c >=
'a' && c <=
'z') || (c >=
'A' && c <=
'Z') ||
43 (c >=
'0' && c <=
'9') || c ==
'_' || c ==
'-' || c ==
'.';
44 out.push_back(
ok ? c :
'_');
45 if (out.size() >= 48)
break;
50 unsigned long current_pid()
53 return static_cast<unsigned long>(_getpid());
55 return static_cast<unsigned long>(::getpid());
59 void append_varint(std::string& out, std::uint64_t value)
63 out.push_back(
static_cast<char>((value & 0x7f) | 0x80));
66 out.push_back(
static_cast<char>(value));
69 enum class VarintResult
76 VarintResult read_varint(std::istream& in, std::uint64_t& valueOut)
78 std::uint64_t value = 0;
80 for (
int i = 0; i < 10; ++i)
82 const int byte = in.get();
83 if (
byte == std::char_traits<char>::eof())
85 return (i == 0) ? VarintResult::CleanEof : VarintResult::Error;
87 value |=
static_cast<std::uint64_t
>(
byte & 0x7f) << shift;
88 if ((
byte & 0x80) == 0)
91 return VarintResult::Ok;
95 return VarintResult::Error;
101 constexpr std::uint64_t kMaxRecordBytes = 512ull * 1024ull * 1024ull;
109 std::atomic<std::uint64_t> g_fillCounter{ 0 };
116 if (m_failed || !m_stream.is_open())
121 std::uint64_t length = 0;
122 const auto varint = read_varint(m_stream, length);
123 if (varint == VarintResult::CleanEof)
127 if (varint == VarintResult::Error || length == 0 || length > kMaxRecordBytes)
133 std::string bytes(
static_cast<std::size_t
>(length),
'\0');
134 m_stream.read(bytes.data(), bytes.size());
135 if (
static_cast<std::uint64_t
>(m_stream.gcount()) != length)
140 if (!record.ParseFromString(bytes))
153 if (m_openForWrite && !m_committed)
160 : m_stream(std::move(other.m_stream)),
161 m_path(std::move(other.m_path)),
162 m_pCache(other.m_pCache),
163 m_designName(std::move(other.m_designName)),
164 m_generation(other.m_generation),
165 m_bytesWritten(other.m_bytesWritten),
166 m_recordsWritten(other.m_recordsWritten),
167 m_openForWrite(other.m_openForWrite),
168 m_committed(other.m_committed),
169 m_publishSucceeded(other.m_publishSucceeded)
171 other.m_pCache =
nullptr;
172 other.m_openForWrite =
false;
173 other.m_committed =
false;
174 other.m_publishSucceeded =
false;
181 if (m_openForWrite && !m_committed)
185 m_stream = std::move(other.m_stream);
186 m_path = std::move(other.m_path);
187 m_pCache = other.m_pCache;
188 m_designName = std::move(other.m_designName);
189 m_generation = other.m_generation;
190 m_bytesWritten = other.m_bytesWritten;
191 m_recordsWritten = other.m_recordsWritten;
192 m_openForWrite = other.m_openForWrite;
193 m_committed = other.m_committed;
194 m_publishSucceeded = other.m_publishSucceeded;
195 other.m_pCache =
nullptr;
196 other.m_openForWrite =
false;
197 other.m_committed =
false;
198 other.m_publishSucceeded =
false;
205 if (!m_openForWrite || !m_stream.is_open())
211 if (!record.SerializeToString(&bytes))
217 framed.reserve(bytes.size() + 10);
218 append_varint(framed, bytes.size());
219 framed.append(bytes);
221 m_stream.write(framed.data(), framed.size());
222 if (!m_stream.good())
226 m_bytesWritten += framed.size();
233 if (!m_openForWrite || m_committed)
238 return m_publishSucceeded;
243 if (!m_openForWrite || m_committed)
250 void StreamSpoolCache::Writer::Close(
bool commit)
252 m_openForWrite =
false;
253 bool streamOk =
true;
254 if (m_stream.is_open())
261 streamOk = m_stream.good();
268 m_publishSucceeded = streamOk && m_pCache !=
nullptr;
269 if (m_publishSucceeded)
271 m_publishSucceeded = m_pCache->PublishEntry(m_designName,
272 Entry{ m_path, m_generation, m_bytesWritten, m_recordsWritten, 0 });
277 std::filesystem::remove(m_path, ec);
278 logerror(
"StreamSpoolCache: spool flush/close failed for \"" + m_designName +
279 "\"; incomplete spool deleted, not published (next request refills cold)");
285 std::filesystem::remove(m_path, ec);
293 namespace fs = std::filesystem;
296 const auto base = fs::temp_directory_path(ec);
299 logerror(
"StreamSpoolCache: cannot resolve temp directory; spooling disabled");
305 const auto pid = current_pid();
306 for (
int attempt = 0; attempt < 8; ++attempt)
308 const auto tick =
static_cast<std::uint64_t
>(
309 std::chrono::steady_clock::now().time_since_epoch().count()) + attempt;
310 const auto candidate = base / (
"odbdesign-spool-" + std::to_string(pid) +
"-" + std::to_string(tick));
311 std::error_code createEc;
312 if (fs::create_directory(candidate, createEc) && !createEc)
314 InitializeDirectory(candidate,
true, maxTotalBytes);
318 logerror(
"StreamSpoolCache: failed to create spool directory; spooling disabled");
323 InitializeDirectory(directory,
false, maxTotalBytes);
331 std::filesystem::remove_all(m_directory, ec);
341 std::lock_guard<std::mutex> lock(m_mutex);
342 for (
const auto& kv : m_entriesByName)
345 std::filesystem::remove(kv.second.path, ec);
349 void StreamSpoolCache::InitializeDirectory(
const std::filesystem::path& directory,
bool ownsDirectory, std::uint64_t maxTotalBytes)
351 namespace fs = std::filesystem;
353 fs::create_directories(directory, ec);
354 if (ec || !fs::is_directory(directory))
356 logerror(
"StreamSpoolCache: spool directory unusable (" + directory.string() +
"); spooling disabled");
359 m_directory = directory;
360 m_ownsDirectory = ownsDirectory;
361 m_maxTotalBytes = maxTotalBytes;
365 std::filesystem::path StreamSpoolCache::MakeSpoolPath(
const std::string& designName, std::uint64_t generation, std::uint64_t fillId)
const
367 auto name = sanitize_name(designName);
377 return m_directory / (name +
"-" + std::to_string(fnv1a64(designName)) +
"-" +
378 std::to_string(generation) +
"-" +
379 std::to_string(current_pid()) +
"-" + std::to_string(fillId) +
".spool");
389 std::filesystem::path path;
390 std::uint64_t expectedRecords = 0;
392 std::lock_guard<std::mutex> lock(m_mutex);
393 auto findIt = m_entriesByName.find(designName);
394 if (findIt == m_entriesByName.end())
398 if (findIt->second.generation != generation)
403 std::filesystem::remove(findIt->second.path, ec);
404 m_totalBytes -= findIt->second.bytes;
405 m_entriesByName.erase(findIt);
408 path = findIt->second.path;
409 expectedRecords = findIt->second.records;
410 findIt->second.lastUsed = ++m_useCounter;
414 if (!std::filesystem::is_regular_file(path, ec))
426 readerOut.m_stream.open(path, std::ios::binary);
427 if (!readerOut.m_stream.is_open())
431 readerOut.m_recordsRead = 0;
436 readerOut.m_expectedRecords = expectedRecords;
437 readerOut.m_failed =
false;
439 std::lock_guard<std::mutex> lock(m_mutex);
462 std::uint64_t fillId = 0;
470 fillId = ++g_fillCounter;
472 const auto path = MakeSpoolPath(designName, generation, fillId);
474 writerOut.m_stream.open(path, std::ios::binary | std::ios::trunc);
475 if (!writerOut.m_stream.is_open())
479 writerOut.m_pCache =
this;
480 writerOut.m_path = path;
481 writerOut.m_designName = designName;
482 writerOut.m_generation = generation;
483 writerOut.m_bytesWritten = 0;
484 writerOut.m_recordsWritten = 0;
485 writerOut.m_committed =
false;
486 writerOut.m_openForWrite =
true;
490 bool StreamSpoolCache::PublishEntry(
const std::string& designName,
const Entry& entry)
492 std::lock_guard<std::mutex> lock(m_mutex);
493 Entry published = entry;
494 published.lastUsed = ++m_useCounter;
502 if (m_maxTotalBytes != 0 && published.bytes > m_maxTotalBytes)
505 std::filesystem::remove(published.path, ec);
506 logwarn(
"StreamSpoolCache: spool for \"" + designName +
"\" (" +
507 std::to_string(published.bytes) +
" bytes) exceeds spool_max_bytes (" +
508 std::to_string(m_maxTotalBytes) +
"); not published (design serves cold)");
512 auto findIt = m_entriesByName.find(designName);
513 if (findIt != m_entriesByName.end())
516 std::filesystem::remove(findIt->second.path, ec);
517 m_totalBytes -= findIt->second.bytes;
518 findIt->second = published;
522 m_entriesByName.emplace(designName, published);
524 m_totalBytes += published.bytes;
526 EvictOverBudgetLocked(designName);
532 std::lock_guard<std::mutex> lock(m_mutex);
533 auto findIt = m_entriesByName.find(designName);
534 if (findIt == m_entriesByName.end())
539 std::filesystem::remove(findIt->second.path, ec);
540 m_totalBytes -= findIt->second.bytes;
541 m_entriesByName.erase(findIt);
546 std::lock_guard<std::mutex> lock(m_mutex);
547 auto findIt = m_entriesByName.find(designName);
548 if (findIt == m_entriesByName.end() || findIt->second.path != expectedPath)
553 std::filesystem::remove(findIt->second.path, ec);
554 m_totalBytes -= findIt->second.bytes;
555 m_entriesByName.erase(findIt);
558 void StreamSpoolCache::EvictOverBudgetLocked(
const std::string& protectedName)
560 if (m_maxTotalBytes == 0)
564 while (m_totalBytes > m_maxTotalBytes)
567 auto victimIt = m_entriesByName.end();
568 for (
auto it = m_entriesByName.begin(); it != m_entriesByName.end(); ++it)
570 if (it->first == protectedName)
continue;
571 if (victimIt == m_entriesByName.end() || it->second.lastUsed < victimIt->second.lastUsed)
576 if (victimIt == m_entriesByName.end())
581 std::filesystem::remove(victimIt->second.path, ec);
582 m_totalBytes -= victimIt->second.bytes;
583 m_entriesByName.erase(victimIt);
589 std::lock_guard<std::mutex> lock(m_mutex);
595 std::lock_guard<std::mutex> lock(m_mutex);
601 std::lock_guard<std::mutex> lock(m_mutex);
602 return m_entriesByName.size();
bool Next(google::protobuf::Message &record)
bool Append(const google::protobuf::Message &record)
Writer & operator=(Writer &&other) noexcept
std::uint64_t TotalSpoolBytes() const
bool BeginWrite(const std::string &designName, std::uint64_t generation, Writer &writerOut)
void DropStaleEntry(const std::string &designName, const std::filesystem::path &expectedPath)
bool TryOpenReader(const std::string &designName, std::uint64_t generation, Reader &readerOut)
StreamSpoolCache(std::uint64_t maxTotalBytes=kDefaultMaxTotalBytes)
void Drop(const std::string &designName)
std::uint64_t WarmOpens() const
std::size_t EntryCount() const