Skip to content

Commit

Permalink
refactor(checkpoint): refactor checkpoint file naming logic (#783)
Browse files Browse the repository at this point in the history
* refactor(checkpoint): refactor checkpoint file naming logic

Signed-off-by: jiaxiaoyu <[email protected]>
  • Loading branch information
xyyy1420 authored Jan 22, 2025
1 parent 3869753 commit 95eca21
Showing 1 changed file with 41 additions and 26 deletions.
67 changes: 41 additions & 26 deletions src/checkpoint/serializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,23 +96,25 @@ void Serializer::serializePMem(uint64_t inst_count) {
Log("Put gcpt restorer %s to start of pmem", restorer);
}

string filepath;
string base_file_path;
string memory_file_path;

if (checkpoint_state == SimpointCheckpointing) {
filepath = pathManager.getOutputPath() + "_" + to_string(simpoint2Weights.begin()->first) + "_" +
base_file_path = pathManager.getOutputPath() + "_" + to_string(simpoint2Weights.begin()->first) + "_" +
to_string(simpoint2Weights.begin()->second);
} else {
filepath = pathManager.getOutputPath() + "_" + to_string(inst_count);
base_file_path = pathManager.getOutputPath() + "_" + to_string(inst_count);
}

if (compress_file_format == GZ_FORMAT) {
filepath += "_.gz";
gzFile compressed_mem = gzopen(filepath.c_str(), "wb");
if (compressed_mem == nullptr) {
cerr << "Failed to open " << filepath << endl;
Log("Using GZ format generate checkpoint");
memory_file_path = base_file_path + "_memory_.gz";
gzFile memory_compressed_file = gzopen(memory_file_path.c_str(), "wb");
if (memory_compressed_file == nullptr) {
cerr << "Failed to open " << memory_file_path << endl;
xpanic("Can't open physical memory checkpoint file!\n");
} else {
cout << "Opening " << filepath << " as checkpoint output file" << endl;
cout << "Opening " << memory_file_path << " as checkpoint output file" << endl;
}

uint64_t pass_size = 0;
Expand All @@ -122,39 +124,52 @@ void Serializer::serializePMem(uint64_t inst_count) {
? numeric_limits<int>::max()
: ((int64_t)PMEM_SIZE - (int64_t)written);

if (gzwrite(compressed_mem, pmem + written, (uint32_t)pass_size) != (int)pass_size) {
if (gzwrite(memory_compressed_file, pmem + written, (uint32_t)pass_size) != (int)pass_size) {
xpanic("Write failed on physical memory checkpoint file\n");
}
Log("Written 0x%lx bytes\n", pass_size);
}

if (gzclose(compressed_mem)) {
xpanic("Close failed on physical memory checkpoint file\n");
if (gzclose(memory_compressed_file)) {
xpanic("Close failed on physical checkpoint file\n");
}

} else if (compress_file_format == ZSTD_FORMAT) {
filepath += "_.zstd";
Log("Using ZSTD format generate checkpoint");

memory_file_path += base_file_path + "_memory_.zstd";
Log("Opening %s as memory output file", memory_file_path.c_str());

// zstd compress
size_t const compress_buffer_size = ZSTD_compressBound(PMEM_SIZE);
void *const compress_buffer = malloc(compress_buffer_size);
assert(compress_buffer);
size_t memory_size = PMEM_SIZE;
size_t const memory_compress_buffer_size = ZSTD_compressBound(memory_size);
uint8_t *const memory_compress_buffer = (uint8_t*)malloc(memory_compress_buffer_size);
assert(memory_compress_buffer);


size_t const compress_size = ZSTD_compress(compress_buffer, compress_buffer_size, pmem, PMEM_SIZE, 1);
assert(compress_size <= compress_buffer_size && compress_size != 0);

FILE *compress_file = fopen(filepath.c_str(), "wb");
size_t fw_size = fwrite(compress_buffer, 1, compress_size, compress_file);

if (fw_size != (size_t)compress_size) {
free(compress_buffer);
xpanic("file write error: %s : %s \n", filepath.c_str(), strerror(errno));
// compress flash device memory
size_t memory_compress_size = ZSTD_compress(memory_compress_buffer, memory_compress_buffer_size, pmem, PMEM_SIZE, 1);
assert(memory_compress_size <= memory_compress_buffer_size && memory_compress_size != 0);
Log("pmem compress success, compress size %ld", memory_compress_size);

FILE *memory_compress_file = fopen(memory_file_path.c_str(), "wb");
size_t memory_fw_size = fwrite(memory_compress_buffer, 1, memory_compress_size, memory_compress_file);

if (memory_fw_size != memory_compress_size) {
fclose(memory_compress_file);
free(memory_compress_buffer);
xpanic("file write error: %s : %s\n", memory_file_path.c_str(), strerror(errno));
}

if (fclose(compress_file)) {
free(compress_buffer);
xpanic("file close error: %s : %s \n", filepath.c_str(), strerror(errno));
if (fclose(memory_compress_file)) {
free(memory_compress_buffer);
xpanic("file close error: %s : %s \n", base_file_path.c_str(), strerror(errno));
}

free(compress_buffer);
free(memory_compress_buffer);

} else {
xpanic("You need to specify the compress file format using: --checkpoint-format\n");
}
Expand Down

0 comments on commit 95eca21

Please sign in to comment.