Add IO functions to operate on buffer chunks

src/io.cpp

@@ -14,12 +14,20 @@ FsPlainInput::FsPlainInput(const std::string& path)
     if (file_size == 0) {
         throw std::invalid_argument("Input is empty, nothing to encrypt");
     }
-    file_stream = std::ifstream(file_path, std::ios::binary);
 }
 
 raw_data FsPlainInput::read() {
-    raw_data result(file_size);
+    return read(fs::file_size(file_path));
-    file_stream.read(reinterpret_cast<char*>(result.data()), file_size);
+//    raw_data result(file_size);
+//    file_stream.open(file_path, std::ios::binary);
+//    file_stream.read(reinterpret_cast<char*>(result.data()), file_size);
+//    return result;
+}
+raw_data FsPlainInput::read(size_t size) {
+    raw_data result(size);
+    file_stream.open(file_path, std::ios::binary);
+    file_stream.read(reinterpret_cast<char*>(result.data()), size);
+    result.resize(file_stream.gcount());
     return result;
 }
 
@@ -41,12 +49,31 @@ FsCryptoInput::FsCryptoInput(const std::vector<std::string>& filenames) {
 }
 
 raw_data FsCryptoInput::read() {
-    raw_data result(total_size);
+    // start from the beginning
+    current_file = 0;
+    file_stream.close();
+    return read(total_size);
+//    raw_data result(total_size);
+//    size_t data_read{0};
+//    for (auto& f : file_paths) {
+//        std::ifstream ifstream(f.first, std::ios::binary);
+//        ifstream.read(reinterpret_cast<char*>(result.data()) + data_read, f.second);
+//        data_read += f.second;
+//    }
+//    return result;
+}
+raw_data FsCryptoInput::read(size_t size) {
+    raw_data result(size);
     size_t data_read{0};
-    for (auto& f : file_paths) {
+    while (data_read < size && current_file < file_paths.size()) {
-        std::ifstream ifstream(f.first);
+        if (!file_stream.is_open())
-        ifstream.read(reinterpret_cast<char*>(result.data()) + data_read, f.second);
+            file_stream.open(file_paths[current_file].first, std::ios::binary);
-        data_read += f.second;
+        file_stream.read(reinterpret_cast<char*>(result.data()) + data_read, size - data_read);
+        data_read += file_stream.gcount();
+        if(file_stream.eof()) {
+            file_stream.close();
+            ++current_file;
+        }
     }
     return result;
 }
@@ -59,8 +86,14 @@ FsPlainOutput::FsPlainOutput(const std::string& filename)
     }
 }
 size_t FsPlainOutput::write(const raw_data& to_write) {
-    std::ofstream out(file_path, std::ios::binary);
+    //file_stream.open(file_path, std::ios::binary);
-    out.write(reinterpret_cast<const char*>(to_write.data()), to_write.size());
+    //file_stream.write(reinterpret_cast<const char*>(to_write.data()), to_write.size());
+    //return to_write.size();
+    return write_chunk(to_write);
+}
+size_t FsPlainOutput::write_chunk(const raw_data& to_write) {
+    file_stream.open(file_path, std::ios::binary);
+    file_stream.write(reinterpret_cast<const char*>(to_write.data()), to_write.size());
     return to_write.size();
 }
 
@@ -79,18 +112,46 @@ FsCryptoOutput::FsCryptoOutput(const std::string& folder, const int horcrux_num,
 
 size_t FsCryptoOutput::write(const raw_data& to_write) {
     size = to_write.size() / num;
-    std::ofstream f;
     created_files.clear();
+    return write_chunk(to_write);
+//    std::ofstream f;
+//    created_files.clear();
+//    size_t data_written{0};
+//    for (int i = 0; i < num; ++i) {
+//        std::string name = base_name + '.' + std::to_string(i) + ".enc";
+//        created_files.emplace_back(folder_path / name);
+//        f = std::ofstream(created_files.back(),
+//                std::ios::binary | std::ios::trunc);
+//        auto chunk_size = i + 1 != num ? size : size + to_write.size() % size;
+//        f.write(reinterpret_cast<const char*>(to_write.data()) + data_written,
+//                chunk_size);
+//        data_written += chunk_size;
+//    }
+//    return data_written;
+}
+size_t FsCryptoOutput::write_chunk(const raw_data& to_write) {
+    if (size <= 0) {
+        throw std::invalid_argument("Invalid horcrux size");
+    }
     size_t data_written{0};
-    for (int i = 0; i < num; ++i) {
+    std::cout << to_write.size() << " " << data_written << std::endl;
-        std::string name = base_name + '.' + std::to_string(i) + ".enc";
+    while (data_written < to_write.size()){
-        created_files.emplace_back(folder_path / name);
+        if(!file_stream.is_open()){
-        f = std::ofstream(created_files.back(),
+            std::string name = base_name + '.' + std::to_string(created_files.size()) + ".enc";
-                std::ios::binary | std::ios::trunc);
+            created_files.emplace_back(folder_path / name);
-        auto chunk_size = i + 1 != num ? size : size + to_write.size() % size;
+            file_stream.open(created_files.back(),
-        f.write(reinterpret_cast<const char*>(to_write.data()) + data_written,
+                        std::ios::binary | std::ios::trunc);
-                chunk_size);
+        }
-        data_written += chunk_size;
+        // the last file may contain more data (size % num)
+        auto writable = std::min(size - file_stream.tellp(), to_write.size() - data_written);
+        if (num == created_files.size()) { writable = to_write.size() - data_written; }
+        file_stream.write(reinterpret_cast<const char*>(to_write.data()) + data_written,
+                writable);
+        data_written += writable;
+        if (file_stream.tellp() >= size){
+            file_stream.close();
+        }
+        std::cout << to_write.size() << " " << data_written << std::endl;
     }
     return data_written;
 }

src/io.h

@@ -17,6 +17,14 @@ public:
     /** @brief Read from the input
      * @return new buffer containing all read data */
     virtual raw_data read() = 0;
+    virtual raw_data read(size_t size) = 0;
+
+//    template<class UnaryFunction>
+//    void read_chunks(size_t size, UnaryFunction f) {
+//        while ((auto buf = read(size)) == size) {
+//            f(buf);
+//        }
+//    }
 };
 
 /** @brief Base Class for Output
@@ -28,6 +36,7 @@ public:
      * @param data to write
      * @return written data */
     virtual size_t write(const raw_data& data) = 0;
+    virtual size_t write_chunk(const raw_data& data) = 0;
 };
 
 /** @brief Input that reads data from a regular file */
@@ -51,6 +60,7 @@ public:
     /** @brief Read from the input
      * @return new buffer containing all read data */
     raw_data read() override;
+    raw_data read(size_t size) override;
 };
 
 /** @brief Input that reads scattered data from multiple files */
@@ -59,6 +69,10 @@ class FsCryptoInput : public Input{
     std::vector<std::pair<std::filesystem::path, size_t>> file_paths;
     /// total size of input files
     size_t total_size{0};
+    /// input file stream associated to the current input file
+    std::ifstream file_stream;
+    /// iterator pointing to the current input file
+    int current_file{0};
     /// remove default constuctor
     FsCryptoInput() = delete;
 
@@ -73,12 +87,15 @@ public:
      * @details basically concatenate the input files.
      * @return new buffer containing all read data */
     raw_data read() override;
+    raw_data read(size_t size) override;
 };
 
 /** @brief Output that writes to a single regular file */
 class FsPlainOutput : public Output {
     /// output file path
     std::filesystem::path file_path;
+    /// output file stream associated to the output file
+    std::ofstream file_stream;
     /// remove default constructor
     FsPlainOutput() = delete;
 
@@ -93,6 +110,7 @@ public:
      * @param data to write
      * @return written data */
     size_t write(const raw_data& to_write) override;
+    size_t write_chunk(const raw_data& to_write) override;
 };
 
 /** @brief Output that splits the data in equal size chunks and writes them to
@@ -108,6 +126,8 @@ class FsCryptoOutput : public Output {
     int num;
     /// size of each ouptut file
     size_t size;
+    /// output file stream associated to the current output file
+    std::ofstream file_stream;
     /// remove default constructor
     FsCryptoOutput() = delete;
 
@@ -130,6 +150,7 @@ public:
      * @param data to write
      * @return written data */
     size_t write(const raw_data& to_write) override;
+    size_t write_chunk(const raw_data& to_write) override;
 };
 };  // namespace horcrux
 #endif  // HORCRUX_SRC_IO_H

test/io-test.cpp

@@ -2,33 +2,38 @@
 #include "io.h"
 #include "test.h"
 
+
 TEST(IoTests, FsPlainInput) {
     EXPECT_THROW(horcrux::FsPlainInput input{noexist}, std::invalid_argument);
     EXPECT_THROW(horcrux::FsPlainInput input{folder}, std::invalid_argument);
     EXPECT_THROW(horcrux::FsPlainInput input{empty}, std::invalid_argument);
     EXPECT_NO_THROW(horcrux::FsPlainInput input{text});
     EXPECT_NO_THROW(horcrux::FsPlainInput input{image});
+    check_files_status();
 }
 
 TEST(IoTests, FsPlainInputReadText) {
     auto input = horcrux::FsPlainInput{text};
     auto buf = generic_read_file(text);
     EXPECT_EQ(buf, input.read());
+    check_files_status();
 }
 
 TEST(IoTests, FsPlainInputReadImg) {
     auto input = horcrux::FsPlainInput{image};
     auto buf = generic_read_file(image);
     EXPECT_EQ(buf, input.read());
+    check_files_status();
 }
 
 TEST(IoTests, FsCryptoOutput) {
     EXPECT_THROW(horcrux::FsCryptoOutput output(noexist, 1), std::invalid_argument);
-    EXPECT_NO_THROW(horcrux::FsCryptoOutput output(folder, 1));
+    EXPECT_NO_THROW(horcrux::FsCryptoOutput output(folder, 1, "test_chunk", 1024));
     EXPECT_THROW(horcrux::FsCryptoOutput output(folder, 0), std::invalid_argument);
     EXPECT_THROW(horcrux::FsCryptoOutput output(empty, 1), std::invalid_argument);
     EXPECT_THROW(horcrux::FsCryptoOutput output(text, 1), std::invalid_argument);
     EXPECT_THROW(horcrux::FsCryptoOutput output(image, 1), std::invalid_argument);
+    check_files_status();
 }
 
 TEST(IoTests, FsCryptoOutputWriteImage){
@@ -41,6 +46,7 @@ TEST(IoTests, FsCryptoOutputWriteImage){
 
     EXPECT_EQ(10, out.created_files.size());
     delete_created_files(out);
+    check_files_status();
 }
 
 TEST(IoTests, FsCryptoOutputWriteText){
@@ -52,6 +58,7 @@ TEST(IoTests, FsCryptoOutputWriteText){
 
     EXPECT_EQ(2, out.created_files.size());
     delete_created_files(out);
+    check_files_status();
 }
 
 TEST(IoTest, FsCryptoInput){
@@ -62,6 +69,7 @@ TEST(IoTest, FsCryptoInput){
     EXPECT_THROW(horcrux::FsCryptoInput output({empty, empty}), std::invalid_argument);
     EXPECT_NO_THROW(horcrux::FsCryptoInput output({empty, empty, text}));
     EXPECT_NO_THROW(horcrux::FsCryptoInput output({image, empty, text}));
+    check_files_status();
 }
 
 TEST(IoTest, FsCryptoInputImage){
@@ -77,11 +85,12 @@ TEST(IoTest, FsCryptoInputImage){
     EXPECT_EQ(read, buf);
 
     files.push_back(text);
-    in = horcrux::FsCryptoInput(files);
+    auto in2 = horcrux::FsCryptoInput(files);
-    read = in.read();
+    read = in2.read();
     EXPECT_NE(read, buf);
 
     delete_created_files(out);
+    check_files_status();
 }
 
 TEST(IoTest, FsPlainOuput){
@@ -91,6 +100,7 @@ TEST(IoTest, FsPlainOuput){
     EXPECT_NO_THROW(horcrux::FsPlainOutput output{empty});
     EXPECT_NO_THROW(horcrux::FsPlainOutput output{text});
     EXPECT_NO_THROW(horcrux::FsPlainOutput output{image});
+    check_files_status();
 }
 
 TEST(IoTest, FsPlainOutputWrite){
@@ -101,6 +111,7 @@ TEST(IoTest, FsPlainOutputWrite){
     auto buf2 = generic_read_file(noexist);
     EXPECT_EQ(buf, buf2);
     std::filesystem::remove(noexist);
+    check_files_status();
 }
 
 TEST(IoTest, PlainToPlain){
@@ -115,6 +126,7 @@ TEST(IoTest, PlainToPlain){
     auto buf2 = generic_read_file(noexist);
     EXPECT_EQ(buf, buf2);
     std::filesystem::remove(noexist);
+    check_files_status();
 }
 TEST(IoTest, PlainToCrypto){
     auto buf = generic_read_file(image);
@@ -130,6 +142,7 @@ TEST(IoTest, PlainToCrypto){
     auto buf2 = in_test.read();
     EXPECT_EQ(buf, buf2);
     delete_created_files(out);
+    check_files_status();
 }
 
 TEST(IoTest, CryptoToCrypto){
@@ -153,6 +166,7 @@ TEST(IoTest, CryptoToCrypto){
 
     delete_created_files(out_test);
     delete_created_files(out);
+    check_files_status();
 }
 
 TEST(IoTest, CryptoToPlain){
@@ -173,4 +187,5 @@ TEST(IoTest, CryptoToPlain){
     EXPECT_EQ(buf, buf2);
     delete_created_files(out_test);
     std::filesystem::remove(noexist);
+    check_files_status();
 }

test/main.cpp

@@ -36,6 +36,20 @@ std::vector<std::string> get_encrypted_files(const std::string& folder,
     return result;
 }
 
+void check_files_status(){
+    const std::filesystem::path p_folder{folder};
+    const std::filesystem::path p_noexist{noexist};
+    const std::filesystem::path p_empty{empty};
+    const std::filesystem::path p_text{text};
+    const std::filesystem::path p_image{image};
+
+    EXPECT_EQ(false, std::filesystem::exists(p_noexist));
+    EXPECT_EQ(0, std::filesystem::file_size(p_empty));
+    EXPECT_EQ(1388, std::filesystem::file_size(p_text));
+    EXPECT_EQ(124106, std::filesystem::file_size(p_image));
+
+}
+
 
 int main(int argc, char *argv[]) {
     ::testing::InitGoogleTest(&argc, argv);

test/test.h

@@ -25,6 +25,8 @@ void delete_created_files(const horcrux::FsCryptoOutput& out);
 
 std::vector<std::string> get_encrypted_files(const std::string& folder, const std::string& basename);
 
+void check_files_status();
+
 /* Crypto Test utils */
 
 /* test command: