# HG changeset patch # User Sebastien Jodogne # Date 1597828735 -7200 # Node ID b56f3a37a4a1b1ce45cc0d6c44891ac46e6d5eac # Parent 72047b61570f063675cfd74f7e99424fdbb50f65 optimization of ChunkedBuffer if many small chunks are added diff -r 72047b61570f -r b56f3a37a4a1 OrthancFramework/Sources/ChunkedBuffer.cpp --- a/OrthancFramework/Sources/ChunkedBuffer.cpp Tue Aug 18 16:57:20 2020 +0200 +++ b/OrthancFramework/Sources/ChunkedBuffer.cpp Wed Aug 19 11:18:55 2020 +0200 @@ -32,6 +32,7 @@ void ChunkedBuffer::Clear() { numBytes_ = 0; + pendingPos_ = 0; for (Chunks::iterator it = chunks_.begin(); it != chunks_.end(); ++it) @@ -41,8 +42,8 @@ } - void ChunkedBuffer::AddChunk(const void* chunkData, - size_t chunkSize) + void ChunkedBuffer::AddChunkInternal(const void* chunkData, + size_t chunkSize) { if (chunkSize == 0) { @@ -57,6 +58,76 @@ } + void ChunkedBuffer::FlushPendingBuffer() + { + assert(pendingPos_ <= pendingBuffer_.size()); + + if (!pendingBuffer_.empty()) + { + AddChunkInternal(pendingBuffer_.c_str(), pendingPos_); + } + else + { + assert(pendingPos_ == 0); + } + + pendingPos_ = 0; + } + + + ChunkedBuffer::ChunkedBuffer() : + numBytes_(0), + pendingPos_(0) + { + pendingBuffer_.resize(16 * 1024); // Default size of the pending buffer: 16KB + } + + + void ChunkedBuffer::SetPendingBufferSize(size_t size) + { + FlushPendingBuffer(); + pendingBuffer_.resize(size); + } + + + void ChunkedBuffer::AddChunk(const void* chunkData, + size_t chunkSize) + { + if (chunkSize > 0) + { +#if 1 + assert(sizeof(char) == 1); + + // Optimization if Orthanc >= 1.7.3, to speed up in the presence of many small chunks + if (pendingPos_ + chunkSize <= pendingBuffer_.size()) + { + // There remain enough place in the pending buffer + memcpy(&pendingBuffer_[pendingPos_], chunkData, chunkSize); + pendingPos_ += chunkSize; + } + else + { + FlushPendingBuffer(); + + if (!pendingBuffer_.empty() && + chunkSize < pendingBuffer_.size()) + { + memcpy(&pendingBuffer_[0], chunkData, chunkSize); + pendingPos_ = chunkSize; + } + else + { + AddChunkInternal(chunkData, chunkSize); + } + } +#else + // Non-optimized implementation in Orthanc <= 1.7.2 + AddChunkInternal(chunkData, chunkSize); +#endif + } + } + + void ChunkedBuffer::AddChunk(const std::string& chunk) { if (chunk.size() > 0) @@ -80,6 +151,7 @@ void ChunkedBuffer::Flatten(std::string& result) { + FlushPendingBuffer(); result.resize(numBytes_); size_t pos = 0; diff -r 72047b61570f -r b56f3a37a4a1 OrthancFramework/Sources/ChunkedBuffer.h --- a/OrthancFramework/Sources/ChunkedBuffer.h Tue Aug 18 16:57:20 2020 +0200 +++ b/OrthancFramework/Sources/ChunkedBuffer.h Wed Aug 19 11:18:55 2020 +0200 @@ -34,15 +34,21 @@ { private: typedef std::list Chunks; - size_t numBytes_; - Chunks chunks_; + + size_t numBytes_; + Chunks chunks_; + std::string pendingBuffer_; // Buffer to speed up if adding many small chunks + size_t pendingPos_; void Clear(); + void AddChunkInternal(const void* chunkData, + size_t chunkSize); + + void FlushPendingBuffer(); + public: - ChunkedBuffer() : numBytes_(0) - { - } + ChunkedBuffer(); ~ChunkedBuffer() { @@ -51,7 +57,14 @@ size_t GetNumBytes() const { - return numBytes_; + return numBytes_ + pendingPos_; + } + + void SetPendingBufferSize(size_t size); + + size_t GetPendingBufferSize() const + { + return pendingBuffer_.size(); } void AddChunk(const void* chunkData, diff -r 72047b61570f -r b56f3a37a4a1 OrthancFramework/UnitTestsSources/RestApiTests.cpp --- a/OrthancFramework/UnitTestsSources/RestApiTests.cpp Tue Aug 18 16:57:20 2020 +0200 +++ b/OrthancFramework/UnitTestsSources/RestApiTests.cpp Wed Aug 19 11:18:55 2020 +0200 @@ -143,22 +143,34 @@ #endif -TEST(RestApi, ChunkedBuffer) +TEST(ChunkedBuffer, Basic) { - ChunkedBuffer b; - //b.SetTrailingBufferSize(0); // TODO - - ASSERT_EQ(0u, b.GetNumBytes()); + for (unsigned int i = 0; i < 2; i++) + { + ChunkedBuffer b; - b.AddChunk("hello", 5); - ASSERT_EQ(5u, b.GetNumBytes()); + if (i == 0) + { + b.SetPendingBufferSize(0); + ASSERT_EQ(0u, b.GetPendingBufferSize()); + } + else + { + ASSERT_EQ(16u * 1024u, b.GetPendingBufferSize()); + } + + ASSERT_EQ(0u, b.GetNumBytes()); - b.AddChunk("world", 5); - ASSERT_EQ(10u, b.GetNumBytes()); + b.AddChunk("hello", 5); + ASSERT_EQ(5u, b.GetNumBytes()); - std::string s; - b.Flatten(s); - ASSERT_EQ("helloworld", s); + b.AddChunk("world", 5); + ASSERT_EQ(10u, b.GetNumBytes()); + + std::string s; + b.Flatten(s); + ASSERT_EQ("helloworld", s); + } } @@ -883,6 +895,72 @@ } +TEST(ChunkedBuffer, DISABLED_Large) +{ + const size_t LARGE = 60 * 1024 * 1024; + + ChunkedBuffer b; + for (size_t i = 0; i < LARGE; i++) + { + b.AddChunk(boost::lexical_cast(i % 10)); + } + + std::string s; + b.Flatten(s); + ASSERT_EQ(LARGE, s.size()); + ASSERT_EQ(0, b.GetNumBytes()); + + for (size_t i = 0; i < LARGE; i++) + { + ASSERT_EQ('0' + (i % 10), s[i]); + } + + b.Flatten(s); + ASSERT_EQ(0u, s.size()); +} + + +TEST(ChunkedBuffer, Pending) +{ + ChunkedBuffer b; + + for (size_t pendingSize = 0; pendingSize < 16; pendingSize++) + { + b.SetPendingBufferSize(pendingSize); + ASSERT_EQ(pendingSize, b.GetPendingBufferSize()); + + unsigned int pos = 0; + unsigned int iteration = 0; + + while (pos < 1024) + { + size_t chunkSize = (iteration % 17); + + std::string chunk; + chunk.resize(chunkSize); + for (size_t i = 0; i < chunkSize; i++) + { + chunk[i] = '0' + (pos % 10); + pos++; + } + + b.AddChunk(chunk); + + iteration ++; + } + + std::string s; + b.Flatten(s); + ASSERT_EQ(0u, b.GetNumBytes()); + ASSERT_EQ(pos, s.size()); + + for (size_t i = 0; i < s.size(); i++) + { + ASSERT_EQ('0' + (i % 10), s[i]); + } + } +} + namespace @@ -915,7 +993,7 @@ size_t i = 0; while (pos_ < size_ && - i < chunkSize_) + i < chunk.size()) { chunk[i] = '0' + (pos_ % 7); pos_++; @@ -987,9 +1065,9 @@ WebServiceParameters w; w.SetUrl("http://localhost:5000"); - - TotoBody body(600 * 1024 * 1024, 6 * 1024 * 1024 - 17); - //TotoBody body(600 * 1024 * 1024, 1); + + //TotoBody body(600 * 1024 * 1024, 6 * 1024 * 1024 - 17); + TotoBody body(32 * 1024, 1); // This crashes Orthanc 1.6.0 to 1.7.2 HttpClient c(w, "toto"); c.SetMethod(HttpMethod_Post); @@ -1004,20 +1082,3 @@ server.Stop(); } - - - -TEST(Toto, DISABLED_Tata) -{ - boost::posix_time::ptime start = boost::posix_time::microsec_clock::local_time(); - - ChunkedBuffer b; - for (unsigned int i = 0; i < 600 * 1024 * 1024; i++) - { - b.AddChunk("a", 1); - } - - boost::posix_time::ptime end = boost::posix_time::microsec_clock::local_time(); - - printf("time: %d\n", (end-start).total_microseconds()); -}