From d8a050b183da37904024953e9d466e9919113e7c Mon Sep 17 00:00:00 2001 From: Michele Rodolfi Date: Mon, 14 Dec 2020 10:20:10 +0100 Subject: [PATCH] Add IO functions to operate on buffer chunks --- src/io.cpp | 101 +++++++++++++++++++++++++++++++++++++---------- src/io.h | 21 ++++++++++ test/io-test.cpp | 21 ++++++++-- test/main.cpp | 14 +++++++ test/test.h | 2 + 5 files changed, 136 insertions(+), 23 deletions(-) diff --git a/src/io.cpp b/src/io.cpp index 7386735..7a644b8 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -14,12 +14,20 @@ FsPlainInput::FsPlainInput(const std::string& path) if (file_size == 0) { throw std::invalid_argument("Input is empty, nothing to encrypt"); } - file_stream = std::ifstream(file_path, std::ios::binary); } raw_data FsPlainInput::read() { - raw_data result(file_size); - file_stream.read(reinterpret_cast(result.data()), file_size); + return read(fs::file_size(file_path)); +// raw_data result(file_size); +// file_stream.open(file_path, std::ios::binary); +// file_stream.read(reinterpret_cast(result.data()), file_size); +// return result; +} +raw_data FsPlainInput::read(size_t size) { + raw_data result(size); + file_stream.open(file_path, std::ios::binary); + file_stream.read(reinterpret_cast(result.data()), size); + result.resize(file_stream.gcount()); return result; } @@ -41,12 +49,31 @@ FsCryptoInput::FsCryptoInput(const std::vector& filenames) { } raw_data FsCryptoInput::read() { - raw_data result(total_size); + // start from the beginning + current_file = 0; + file_stream.close(); + return read(total_size); +// raw_data result(total_size); +// size_t data_read{0}; +// for (auto& f : file_paths) { +// std::ifstream ifstream(f.first, std::ios::binary); +// ifstream.read(reinterpret_cast(result.data()) + data_read, f.second); +// data_read += f.second; +// } +// return result; +} +raw_data FsCryptoInput::read(size_t size) { + raw_data result(size); size_t data_read{0}; - for (auto& f : file_paths) { - std::ifstream ifstream(f.first); - ifstream.read(reinterpret_cast(result.data()) + data_read, f.second); - data_read += f.second; + while (data_read < size && current_file < file_paths.size()) { + if (!file_stream.is_open()) + file_stream.open(file_paths[current_file].first, std::ios::binary); + file_stream.read(reinterpret_cast(result.data()) + data_read, size - data_read); + data_read += file_stream.gcount(); + if(file_stream.eof()) { + file_stream.close(); + ++current_file; + } } return result; } @@ -59,8 +86,14 @@ FsPlainOutput::FsPlainOutput(const std::string& filename) } } size_t FsPlainOutput::write(const raw_data& to_write) { - std::ofstream out(file_path, std::ios::binary); - out.write(reinterpret_cast(to_write.data()), to_write.size()); + //file_stream.open(file_path, std::ios::binary); + //file_stream.write(reinterpret_cast(to_write.data()), to_write.size()); + //return to_write.size(); + return write_chunk(to_write); +} +size_t FsPlainOutput::write_chunk(const raw_data& to_write) { + file_stream.open(file_path, std::ios::binary); + file_stream.write(reinterpret_cast(to_write.data()), to_write.size()); return to_write.size(); } @@ -79,18 +112,46 @@ FsCryptoOutput::FsCryptoOutput(const std::string& folder, const int horcrux_num, size_t FsCryptoOutput::write(const raw_data& to_write) { size = to_write.size() / num; - std::ofstream f; created_files.clear(); + return write_chunk(to_write); +// std::ofstream f; +// created_files.clear(); +// size_t data_written{0}; +// for (int i = 0; i < num; ++i) { +// std::string name = base_name + '.' + std::to_string(i) + ".enc"; +// created_files.emplace_back(folder_path / name); +// f = std::ofstream(created_files.back(), +// std::ios::binary | std::ios::trunc); +// auto chunk_size = i + 1 != num ? size : size + to_write.size() % size; +// f.write(reinterpret_cast(to_write.data()) + data_written, +// chunk_size); +// data_written += chunk_size; +// } +// return data_written; +} +size_t FsCryptoOutput::write_chunk(const raw_data& to_write) { + if (size <= 0) { + throw std::invalid_argument("Invalid horcrux size"); + } size_t data_written{0}; - for (int i = 0; i < num; ++i) { - std::string name = base_name + '.' + std::to_string(i) + ".enc"; - created_files.emplace_back(folder_path / name); - f = std::ofstream(created_files.back(), - std::ios::binary | std::ios::trunc); - auto chunk_size = i + 1 != num ? size : size + to_write.size() % size; - f.write(reinterpret_cast(to_write.data()) + data_written, - chunk_size); - data_written += chunk_size; + std::cout << to_write.size() << " " << data_written << std::endl; + while (data_written < to_write.size()){ + if(!file_stream.is_open()){ + std::string name = base_name + '.' + std::to_string(created_files.size()) + ".enc"; + created_files.emplace_back(folder_path / name); + file_stream.open(created_files.back(), + std::ios::binary | std::ios::trunc); + } + // the last file may contain more data (size % num) + auto writable = std::min(size - file_stream.tellp(), to_write.size() - data_written); + if (num == created_files.size()) { writable = to_write.size() - data_written; } + file_stream.write(reinterpret_cast(to_write.data()) + data_written, + writable); + data_written += writable; + if (file_stream.tellp() >= size){ + file_stream.close(); + } + std::cout << to_write.size() << " " << data_written << std::endl; } return data_written; } diff --git a/src/io.h b/src/io.h index 5d35422..b7d197a 100644 --- a/src/io.h +++ b/src/io.h @@ -17,6 +17,14 @@ public: /** @brief Read from the input * @return new buffer containing all read data */ virtual raw_data read() = 0; + virtual raw_data read(size_t size) = 0; + +// template +// void read_chunks(size_t size, UnaryFunction f) { +// while ((auto buf = read(size)) == size) { +// f(buf); +// } +// } }; /** @brief Base Class for Output @@ -28,6 +36,7 @@ public: * @param data to write * @return written data */ virtual size_t write(const raw_data& data) = 0; + virtual size_t write_chunk(const raw_data& data) = 0; }; /** @brief Input that reads data from a regular file */ @@ -51,6 +60,7 @@ public: /** @brief Read from the input * @return new buffer containing all read data */ raw_data read() override; + raw_data read(size_t size) override; }; /** @brief Input that reads scattered data from multiple files */ @@ -59,6 +69,10 @@ class FsCryptoInput : public Input{ std::vector> file_paths; /// total size of input files size_t total_size{0}; + /// input file stream associated to the current input file + std::ifstream file_stream; + /// iterator pointing to the current input file + int current_file{0}; /// remove default constuctor FsCryptoInput() = delete; @@ -73,12 +87,15 @@ public: * @details basically concatenate the input files. * @return new buffer containing all read data */ raw_data read() override; + raw_data read(size_t size) override; }; /** @brief Output that writes to a single regular file */ class FsPlainOutput : public Output { /// output file path std::filesystem::path file_path; + /// output file stream associated to the output file + std::ofstream file_stream; /// remove default constructor FsPlainOutput() = delete; @@ -93,6 +110,7 @@ public: * @param data to write * @return written data */ size_t write(const raw_data& to_write) override; + size_t write_chunk(const raw_data& to_write) override; }; /** @brief Output that splits the data in equal size chunks and writes them to @@ -108,6 +126,8 @@ class FsCryptoOutput : public Output { int num; /// size of each ouptut file size_t size; + /// output file stream associated to the current output file + std::ofstream file_stream; /// remove default constructor FsCryptoOutput() = delete; @@ -130,6 +150,7 @@ public: * @param data to write * @return written data */ size_t write(const raw_data& to_write) override; + size_t write_chunk(const raw_data& to_write) override; }; }; // namespace horcrux #endif // HORCRUX_SRC_IO_H diff --git a/test/io-test.cpp b/test/io-test.cpp index 86195ba..6fa8a65 100644 --- a/test/io-test.cpp +++ b/test/io-test.cpp @@ -2,33 +2,38 @@ #include "io.h" #include "test.h" + TEST(IoTests, FsPlainInput) { EXPECT_THROW(horcrux::FsPlainInput input{noexist}, std::invalid_argument); EXPECT_THROW(horcrux::FsPlainInput input{folder}, std::invalid_argument); EXPECT_THROW(horcrux::FsPlainInput input{empty}, std::invalid_argument); EXPECT_NO_THROW(horcrux::FsPlainInput input{text}); EXPECT_NO_THROW(horcrux::FsPlainInput input{image}); + check_files_status(); } TEST(IoTests, FsPlainInputReadText) { auto input = horcrux::FsPlainInput{text}; auto buf = generic_read_file(text); EXPECT_EQ(buf, input.read()); + check_files_status(); } TEST(IoTests, FsPlainInputReadImg) { auto input = horcrux::FsPlainInput{image}; auto buf = generic_read_file(image); EXPECT_EQ(buf, input.read()); + check_files_status(); } TEST(IoTests, FsCryptoOutput) { EXPECT_THROW(horcrux::FsCryptoOutput output(noexist, 1), std::invalid_argument); - EXPECT_NO_THROW(horcrux::FsCryptoOutput output(folder, 1)); + EXPECT_NO_THROW(horcrux::FsCryptoOutput output(folder, 1, "test_chunk", 1024)); EXPECT_THROW(horcrux::FsCryptoOutput output(folder, 0), std::invalid_argument); EXPECT_THROW(horcrux::FsCryptoOutput output(empty, 1), std::invalid_argument); EXPECT_THROW(horcrux::FsCryptoOutput output(text, 1), std::invalid_argument); EXPECT_THROW(horcrux::FsCryptoOutput output(image, 1), std::invalid_argument); + check_files_status(); } TEST(IoTests, FsCryptoOutputWriteImage){ @@ -41,6 +46,7 @@ TEST(IoTests, FsCryptoOutputWriteImage){ EXPECT_EQ(10, out.created_files.size()); delete_created_files(out); + check_files_status(); } TEST(IoTests, FsCryptoOutputWriteText){ @@ -52,6 +58,7 @@ TEST(IoTests, FsCryptoOutputWriteText){ EXPECT_EQ(2, out.created_files.size()); delete_created_files(out); + check_files_status(); } TEST(IoTest, FsCryptoInput){ @@ -62,6 +69,7 @@ TEST(IoTest, FsCryptoInput){ EXPECT_THROW(horcrux::FsCryptoInput output({empty, empty}), std::invalid_argument); EXPECT_NO_THROW(horcrux::FsCryptoInput output({empty, empty, text})); EXPECT_NO_THROW(horcrux::FsCryptoInput output({image, empty, text})); + check_files_status(); } TEST(IoTest, FsCryptoInputImage){ @@ -77,11 +85,12 @@ TEST(IoTest, FsCryptoInputImage){ EXPECT_EQ(read, buf); files.push_back(text); - in = horcrux::FsCryptoInput(files); - read = in.read(); + auto in2 = horcrux::FsCryptoInput(files); + read = in2.read(); EXPECT_NE(read, buf); delete_created_files(out); + check_files_status(); } TEST(IoTest, FsPlainOuput){ @@ -91,6 +100,7 @@ TEST(IoTest, FsPlainOuput){ EXPECT_NO_THROW(horcrux::FsPlainOutput output{empty}); EXPECT_NO_THROW(horcrux::FsPlainOutput output{text}); EXPECT_NO_THROW(horcrux::FsPlainOutput output{image}); + check_files_status(); } TEST(IoTest, FsPlainOutputWrite){ @@ -101,6 +111,7 @@ TEST(IoTest, FsPlainOutputWrite){ auto buf2 = generic_read_file(noexist); EXPECT_EQ(buf, buf2); std::filesystem::remove(noexist); + check_files_status(); } TEST(IoTest, PlainToPlain){ @@ -115,6 +126,7 @@ TEST(IoTest, PlainToPlain){ auto buf2 = generic_read_file(noexist); EXPECT_EQ(buf, buf2); std::filesystem::remove(noexist); + check_files_status(); } TEST(IoTest, PlainToCrypto){ auto buf = generic_read_file(image); @@ -130,6 +142,7 @@ TEST(IoTest, PlainToCrypto){ auto buf2 = in_test.read(); EXPECT_EQ(buf, buf2); delete_created_files(out); + check_files_status(); } TEST(IoTest, CryptoToCrypto){ @@ -153,6 +166,7 @@ TEST(IoTest, CryptoToCrypto){ delete_created_files(out_test); delete_created_files(out); + check_files_status(); } TEST(IoTest, CryptoToPlain){ @@ -173,4 +187,5 @@ TEST(IoTest, CryptoToPlain){ EXPECT_EQ(buf, buf2); delete_created_files(out_test); std::filesystem::remove(noexist); + check_files_status(); } diff --git a/test/main.cpp b/test/main.cpp index 148880c..568d4b1 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -36,6 +36,20 @@ std::vector get_encrypted_files(const std::string& folder, return result; } +void check_files_status(){ + const std::filesystem::path p_folder{folder}; + const std::filesystem::path p_noexist{noexist}; + const std::filesystem::path p_empty{empty}; + const std::filesystem::path p_text{text}; + const std::filesystem::path p_image{image}; + + EXPECT_EQ(false, std::filesystem::exists(p_noexist)); + EXPECT_EQ(0, std::filesystem::file_size(p_empty)); + EXPECT_EQ(1388, std::filesystem::file_size(p_text)); + EXPECT_EQ(124106, std::filesystem::file_size(p_image)); + +} + int main(int argc, char *argv[]) { ::testing::InitGoogleTest(&argc, argv); diff --git a/test/test.h b/test/test.h index c16ec7e..415f96c 100644 --- a/test/test.h +++ b/test/test.h @@ -25,6 +25,8 @@ void delete_created_files(const horcrux::FsCryptoOutput& out); std::vector get_encrypted_files(const std::string& folder, const std::string& basename); +void check_files_status(); + /* Crypto Test utils */ /* test command: