OdbDesign
OdbDesign ODB++ Parser - C++ library, gRPC/REST server, CLI, tests
 
Loading...
Searching...
No Matches
FileArchive.cpp
Go to the documentation of this file.
1#include "FileArchive.h"
2#include <filesystem>
3#include "ArchiveExtractor.h"
4#include "CrossPlatform.h"
5#include "MiscInfoFile.h"
6#include "Logger.h"
7#include "../invalid_odb_error.h"
8#include "StopWatch.h"
9#include "fastmove.h"
10#include <system_error>
11#include <cstdio>
12#include <string>
13#include <memory>
14#include "../../Text/Utf8Sanitizer.h"
15
16using namespace Utils;
17using namespace std::filesystem;
18
20{
22 : m_filePath()
23 {
24 }
25
26 FileArchive::FileArchive(const std::string& path)
27 : m_filePath(path)
28 {
29 }
30
32 {
33 m_stepsByName.clear();
34 m_symbolsDirectoriesByName.clear();
35 }
36
37 std::string FileArchive::GetRootDir() const
38 {
39 return m_rootDir;
40 }
41
42 std::string FileArchive::GetProductName() const
43 {
44 return m_productName;
45 }
46
47 std::string FileArchive::GetFilePath() const
48 {
49 return m_filePath;
50 }
51
52 std::string FileArchive::GetFilename() const
53 {
54 //return m_filename;
55 return path(m_filePath).filename().string();
56 }
57
59 {
60 return m_stepsByName;
61 }
62
64 {
65 return m_symbolsDirectoriesByName;
66 }
67
69 {
70 //try
71 {
72 StopWatch timer(true);
73
74 if (!exists(m_filePath)) return false;
75
76 if (is_regular_file(m_filePath))
77 {
78 path extractedPath;
79 if (!ExtractDesignArchive(m_filePath, extractedPath))
80 {
81 logerror("failed to extract archive: (" + m_filePath + ")");
82 return false;
83 }
84
85 m_rootDir = findRootDir(extractedPath);
86 }
87
88 if (is_directory(m_rootDir))
89 {
90 loginfo("Parsing... ");
91
92 if (ParseDesignDirectory(m_rootDir))
93 {
94 timer.stop();
95 auto s = timer.getElapsedSecondsString();
96 loginfo("Successfully parsed. (" + s + "s)");
97
98 return true;
99 }
100 else
101 {
102 logerror("Parsing failed.");
103 throw std::runtime_error("Parsing failed.");
104 }
105 }
106 else
107 {
108 logerror("Failed to find root directory");
109 }
110 }
111 //catch (std::exception& e)
112 //{
113 // logexception(e);
114 // throw e;
115 //}
116
117 return false;
118 }
119
120 bool FileArchive::SaveFileModel(const path& directory)
121 {
122 return SaveFileModel(directory, m_productName);
123 }
124
125 bool FileArchive::SaveFileModel(const path& directory, const std::string& archiveName)
126 {
127 // create directory in /tmp
128 // write dir structure and files to it
129 // gzip with archiveName
130 // move archive to directory
131
132 std::string tempName;
133 if (!CrossPlatform::tmpnam_safe(tempName)) return false;
134
135 auto tempPath = temp_directory_path() / tempName;
136 if (!create_directory(tempPath)) return false;
137
138 auto rootPath = tempPath / archiveName;
139 if (!create_directory(rootPath)) return false;
140 if (!Save(rootPath)) return false;
141
142 // compress the written file structure
143 std::string createdArchivePath;
144 if (! Utils::ArchiveExtractor::CompressDir(rootPath.string(), tempPath.string(), archiveName, createdArchivePath)) return false;
145 if (createdArchivePath.empty()) return false;
146
147 // move the compressed file to the requested save directory
148 path archiveFilename = path(createdArchivePath).filename();
149 path destPath = directory / archiveFilename;
150 std::error_code ec;
151 Utils::fastmove_file(createdArchivePath, destPath, true, ec);
152 if (ec.value() != 0) return false;
153
154 return true;
155 }
156
157 bool FileArchive::Save(const path& directory)
158 {
159 // MiscInfoFile
160 auto miscPath = directory / "misc";
161 if (!create_directory(miscPath)) return false;
162
163 std::ofstream ofs1(miscPath / "info", std::ios::out);
164 if (!m_miscInfoFile.Save(ofs1)) return false;
165 ofs1.close();
166
167 // MiscAttrFile
168 std::ofstream ofs2(miscPath / "sysattr");
169 if (!m_miscAttrListFile.Save(ofs2)) return false;
170 ofs2.close();
171
172 // StandardFontsFile
173 auto fontsPath = directory / "fonts";
174 if (!create_directory(fontsPath)) return false;
175 std::ofstream ofs3(fontsPath / "standard");
176 if (!m_standardFontsFile.Save(ofs3)) return false;
177 ofs3.close();
178
179 // MatrixFile
180 auto matrixPath = directory / "matrix";
181 if (!create_directory(matrixPath)) return false;
182 std::ofstream ofs4(matrixPath / "matrix");
183 if (!m_matrixFile.Save(ofs4)) return false;
184 ofs4.close();
185
186 // Steps
187 const auto stepsDirectory = directory / "steps";
188 if (!create_directory(stepsDirectory)) return false;
189 for (auto& kvStepDirectory : m_stepsByName)
190 {
191 if (!kvStepDirectory.second->Save(stepsDirectory)) return false;
192 }
193
194 // Symbols
195 const auto symbolsDirectory = directory / "symbols";
196 if (!create_directory(symbolsDirectory)) return false;
197 for (auto& kvSymbolsDirectory : m_symbolsDirectoriesByName)
198 {
199 if (!kvSymbolsDirectory.second->Save(symbolsDirectory)) return false;
200 }
201
202 return true;
203 }
204
205 bool FileArchive::ExtractDesignArchive(const path& archivePath, path& extractedPath)
206 {
207 loginfo("Extracting... ");
208
210 {
211 logerror("Unsupported archive type: (" + archivePath.string() + ")");
212 return false;
213 }
214
215 Utils::ArchiveExtractor extractor(archivePath.string());
216 if (!extractor.Extract()) return false;
217
218 auto extracted = path(extractor.GetExtractionDirectory());
219 if (!exists(extracted)) return false;
220
221 extractedPath = extracted;
222
223 loginfo("Successfully extracted.");
224
225 return true;
226
227 }
228
229 /*static*/ std::string FileArchive::findRootDir(const path& extractedPath)
230 {
231 if (pathContainsTopLevelDesignDirs(extractedPath))
232 {
233 return extractedPath.string();
234 }
235 else
236 {
237 for (const auto& p : directory_iterator(extractedPath))
238 {
239 if (is_directory(p))
240 {
241 if (pathContainsTopLevelDesignDirs(p.path()))
242 {
243 return p.path().string();
244 }
245 }
246 }
247
248 return "";
249 }
250 }
251
252 /*static*/ bool FileArchive::pathContainsTopLevelDesignDirs(const path& path)
253 {
254 for (const auto& topLevelRootDirName : TOPLEVEL_DESIGN_DIR_NAMES)
255 {
256 auto rootLevelDirPath = path / topLevelRootDirName;
257 if (!exists(rootLevelDirPath)) return false;
258 }
259 return true;
260 }
261
262 std::unique_ptr<Protobuf::FileArchive> FileArchive::to_protobuf() const
263 {
264 std::unique_ptr<Protobuf::FileArchive> pFileArchiveMessage(new Protobuf::FileArchive);
265 FillProtobufMessage(*pFileArchiveMessage, /*includeSteps=*/true);
266#ifndef NDEBUG
267 // Debug-only assertion: verify all string fields are valid UTF-8 before serialization
268 Odb::Lib::Text::AssertAllStringFieldsAreValidUtf8(*pFileArchiveMessage, "FileArchive");
269#endif
270 return pFileArchiveMessage;
271 }
272
273 std::unique_ptr<Protobuf::FileArchive> FileArchive::to_protobuf_skeleton() const
274 {
275 std::unique_ptr<Protobuf::FileArchive> pFileArchiveMessage(new Protobuf::FileArchive);
276 FillProtobufMessage(*pFileArchiveMessage, /*includeSteps=*/false);
277 return pFileArchiveMessage;
278 }
279
280 void FileArchive::FillProtobufMessage(Protobuf::FileArchive& fileArchiveMessage, bool includeSteps) const
281 {
282 fileArchiveMessage.set_productname(Odb::Lib::Text::ToUtf8(m_productName));
283 fileArchiveMessage.set_filename(Odb::Lib::Text::ToUtf8(m_filename));
284 //fileArchiveMessage.set_filepath(m_filePath);
285 fileArchiveMessage.mutable_matrixfile()->CopyFrom(*m_matrixFile.to_protobuf());
286 fileArchiveMessage.mutable_miscinfofile()->CopyFrom(*m_miscInfoFile.to_protobuf());
287 fileArchiveMessage.mutable_standardfontsfile()->CopyFrom(*m_standardFontsFile.to_protobuf());
288 fileArchiveMessage.mutable_miscattrlistfile()->CopyFrom(*m_miscAttrListFile.to_protobuf());
289
290 if (includeSteps)
291 {
292 for (const auto& kvStepDirectoryRecord : m_stepsByName)
293 {
294 (*fileArchiveMessage.mutable_stepsbyname())[Odb::Lib::Text::ToUtf8(kvStepDirectoryRecord.first)] = *kvStepDirectoryRecord.second->to_protobuf();
295 }
296 }
297
298 for (const auto& kvSymbolsDirectory : m_symbolsDirectoriesByName)
299 {
300 (*fileArchiveMessage.mutable_symbolsdirectoriesbyname())[Odb::Lib::Text::ToUtf8(kvSymbolsDirectory.first)] = *kvSymbolsDirectory.second->to_protobuf();
301 }
302 }
303
304 void FileArchive::from_protobuf(const Protobuf::FileArchive& message)
305 {
306 m_productName = message.productname();
307 m_filename = message.filename();
308 //m_filePath = message.filepath();
309 m_matrixFile.from_protobuf(message.matrixfile());
310 m_miscInfoFile.from_protobuf(message.miscinfofile());
311 m_standardFontsFile.from_protobuf(message.standardfontsfile());
312 m_miscAttrListFile.from_protobuf(message.miscattrlistfile());
313
314 for (const auto& kvStepDirectoryRecord : message.stepsbyname())
315 {
316 auto pStepDirectory = std::make_shared<StepDirectory>("");
317 pStepDirectory->from_protobuf(kvStepDirectoryRecord.second);
318 m_stepsByName[kvStepDirectoryRecord.first] = pStepDirectory;
319 }
320
321 for (const auto& kvSymbolsDirectory : message.symbolsdirectoriesbyname())
322 {
323 auto pSymbolsDirectory = std::make_shared<SymbolsDirectory>("");
324 pSymbolsDirectory->from_protobuf(kvSymbolsDirectory.second);
325 m_symbolsDirectoriesByName[kvSymbolsDirectory.first] = pSymbolsDirectory;
326 }
327 }
328
329 bool FileArchive::ParseDesignDirectory(const path& path)
330 {
331 if (!exists(path)) return false;
332 else if (!is_directory(path)) return false;
333
334 m_productName = path.stem().string();
335
336 if (! ParseStepDirectories(path)) return false;
337 if (! ParseMiscInfoFile(path)) return false;
338 if (! ParseMatrixFile(path)) return false;
339 if (! ParseStandardFontsFile(path)) return false;
340 if (! ParseSymbolsDirectories(path)) return false;
341 if (! ParseMiscAttrListFile(path)) return false;
342
343 return true;
344 }
345
346 bool FileArchive::ParseStepDirectories(const path& path)
347 {
348 loginfo("Parsing steps...");
349
350 auto stepsPath = path / "steps";
351 for (auto& d : directory_iterator(stepsPath))
352 {
353 if (is_directory(d))
354 {
355 auto pStep = std::make_shared<StepDirectory>(d.path());
356 if (pStep->Parse())
357 {
358 m_stepsByName[pStep->GetName()] = pStep;
359 }
360 else
361 {
362 logwarn("Failed to parse step: " + pStep->GetName());
363 //return false;
364 }
365 }
366 }
367
368 loginfo("Parsing steps complete");
369
370 return true;
371 }
372
373 bool FileArchive::ParseMiscInfoFile(const path& path)
374 {
375 loginfo("Parsing misc/info file...");
376
377 auto miscDirectory = path / "misc";
378 if (!exists(miscDirectory))
379 {
380 auto message = "misc directory does not exist: [" + miscDirectory.string() + "]";
381 throw invalid_odb_error(message);
382 }
383 if (!is_directory(miscDirectory))
384 {
385 auto message = "misc path is not a directory: [" + miscDirectory.string() + "]";
386 throw invalid_odb_error(message);
387 }
388
389 if (!m_miscInfoFile.Parse(miscDirectory)) return false;
390
391 loginfo("Parsing misc/info file complete");
392
393 return true;
394 }
395
396 bool FileArchive::ParseMiscAttrListFile(const path& path)
397 {
398 loginfo("Parsing misc/attrlist file...");
399
400 auto miscDirectory = path / "misc";
401 if (!exists(miscDirectory))
402 {
403 auto message = "misc directory does not exist: [" + miscDirectory.string() + "]";
404 throw invalid_odb_error(message);
405 }
406 if (!is_directory(miscDirectory))
407 {
408 auto message = "misc path is not a directory: [" + miscDirectory.string() + "]";
409 throw invalid_odb_error(message);
410 }
411
412 if (!m_miscAttrListFile.Parse(miscDirectory)) return false;
413
414 loginfo("Parsing misc/attrlist file complete");
415
416 return true;
417 }
418
419 bool FileArchive::ParseMatrixFile(const path& path)
420 {
421 loginfo("Parsing matrix/matrix file...");
422
423 auto matrixDir = path / "matrix";
424 if (!exists(matrixDir))
425 {
426 auto message = "matrix directory does not exist: [" + matrixDir.string() + "]";
427 throw invalid_odb_error(message);
428 }
429 if (!is_directory(matrixDir))
430 {
431 auto message = "matrix path is not a directory: [" + matrixDir.string() + "]";
432 throw invalid_odb_error(message);
433 }
434
435 if (!m_matrixFile.Parse(matrixDir)) return false;
436
437 loginfo("Parsing matrix/matrix file complete");
438
439 return true;
440 }
441 bool FileArchive::ParseStandardFontsFile(const path& path)
442 {
443 loginfo("Parsing fonts/standard file...");
444
445 auto fontsDir = path / "fonts";
446 if (!exists(fontsDir))
447 {
448 auto message = "fonts directory does not exist: [" + fontsDir.string() + "]";
449 throw invalid_odb_error(message);
450 }
451 if (!is_directory(fontsDir))
452 {
453 auto message = "fonts path is not a directory: [" + fontsDir.string() + "]";
454 throw invalid_odb_error(message);
455 }
456
457 if (!m_standardFontsFile.Parse(fontsDir)) return false;
458
459 loginfo("Parsing fonts/standard file complete");
460
461 return true;
462 }
463
464 bool FileArchive::ParseSymbolsDirectories(const path& path)
465 {
466 loginfo("Parsing symbols root directory...");
467
468 auto symbolsDirectory = path / "symbols";
469
470 if (!exists(symbolsDirectory))
471 {
472 logwarn("symbols root directory does not exist (" + symbolsDirectory.string() + ")");
473 return true;
474 }
475 else if (!is_directory(symbolsDirectory))
476 {
477 logerror("symbols root directory exists but is a regular file/not a directory (" + symbolsDirectory.string() + ")");
478 return false;
479 }
480
481 for (auto& d : directory_iterator(symbolsDirectory))
482 {
483 if (is_directory(d))
484 {
485 auto pSymbolsDirectory = std::make_shared<SymbolsDirectory>(d.path());
486 if (pSymbolsDirectory->Parse())
487 {
488 //loginfo("Parsing symbols directory: " + pSymbolsDirectory->GetName() + " complete");
489
490 m_symbolsDirectoriesByName[pSymbolsDirectory->GetName()] = pSymbolsDirectory;
491 }
492 else
493 {
494 logerror("Parsing symbol directory: " + pSymbolsDirectory->GetName() + " failed");
495 return false;
496 }
497 }
498 }
499
500 loginfo("Parsing symbols root directory complete");
501
502 return true;
503 }
504
506 {
507 return m_miscInfoFile;
508 }
509
511 {
512 return m_matrixFile;
513 }
514
516 {
517 return m_standardFontsFile;
518 }
519
521 {
522 return m_miscAttrListFile;
523 }
524
525 std::shared_ptr<StepDirectory> FileArchive::GetStepDirectory(const std::string& stepName /*= ""*/) const
526 {
527 std::shared_ptr<FileModel::Design::StepDirectory> pStepDirectory;
528
529 const auto& steps = GetStepsByName();
530 if (!steps.empty())
531 {
532 if (stepName.empty())
533 {
534 // return first step
535 pStepDirectory = steps.begin()->second;
536 }
537 else
538 {
539 auto findIt = steps.find(stepName);
540 if (findIt != steps.end())
541 {
542 pStepDirectory = findIt->second;
543 }
544 }
545 }
546
547 return pStepDirectory;
548 }
549}
#define logerror(s)
Definition Logger.h:17
#define logwarn(s)
Definition Logger.h:20
#define loginfo(s)
Definition Logger.h:14
bool Parse(std::filesystem::path directory)
void from_protobuf(const Odb::Lib::Protobuf::AttrListFile &message) override
std::unique_ptr< Odb::Lib::Protobuf::AttrListFile > to_protobuf() const override
bool Save(std::ostream &os) override
bool SaveFileModel(const std::filesystem::path &directory)
const MiscInfoFile & GetMiscInfoFile() const
std::shared_ptr< StepDirectory > GetStepDirectory(const std::string &stepName="") const
const StandardFontsFile & GetStandardFontsFile() const
const MatrixFile & GetMatrixFile() const
std::unique_ptr< Odb::Lib::Protobuf::FileArchive > to_protobuf() const override
std::unique_ptr< Odb::Lib::Protobuf::FileArchive > to_protobuf_skeleton() const
void from_protobuf(const Odb::Lib::Protobuf::FileArchive &message) override
const StepDirectory::StringMap & GetStepsByName() const
const AttrListFile & GetMiscAttrListFile() const
const SymbolsDirectory::StringMap & GetSymbolsDirectoriesByName() const
bool Save(std::ostream &os) override
std::unique_ptr< Odb::Lib::Protobuf::MatrixFile > to_protobuf() const override
bool Parse(std::filesystem::path path) override
void from_protobuf(const Odb::Lib::Protobuf::MatrixFile &message) override
bool Parse(std::filesystem::path path) override
void from_protobuf(const Odb::Lib::Protobuf::MiscInfoFile &message) override
std::unique_ptr< Odb::Lib::Protobuf::MiscInfoFile > to_protobuf() const override
bool Save(std::ostream &os) override
void from_protobuf(const Odb::Lib::Protobuf::StandardFontsFile &message) override
std::unique_ptr< Odb::Lib::Protobuf::StandardFontsFile > to_protobuf() const override
bool Parse(std::filesystem::path path) override
std::map< std::string, std::shared_ptr< StepDirectory > > StringMap
std::map< std::string, std::shared_ptr< SymbolsDirectory > > StringMap
static bool CompressDir(const std::string &srcDir, const std::string &destDir, const std::string &archiveName, std::string &fileOut, CompressionType type=CompressionType::TarGzip)
static bool IsArchiveTypeSupported(const std::filesystem::path &file)
static bool tmpnam_safe(std::string &tempNameOut)
std::string getElapsedSecondsString(const std::string &suffix="") const
Definition StopWatch.cpp:40
void AssertAllStringFieldsAreValidUtf8(const google::protobuf::Message &msg, std::string_view msgName)
Debug-only assertion to verify all string fields in a message are valid UTF-8.
std::string ToUtf8(std::string_view input)
Converts input to valid UTF-8.
void fastmove_file(const path &source, const path &dest, bool overwriteExisting, std::error_code &ec)
Definition fastmove.cpp:24