# HG changeset patch # User Sebastien Jodogne # Date 1622648139 -7200 # Node ID d9942d48fea7aba4235158ce8e0d597c64c2c806 # Parent 42e1f5bde40bb1508a27bbdb83c163df695836da ZipWriter::CancelStream(), ZipWriter::GetArchiveSize() and HttpOutput::AnswerWithoutBuffering() diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/Compression/HierarchicalZipWriter.cpp --- a/OrthancFramework/Sources/Compression/HierarchicalZipWriter.cpp Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/Compression/HierarchicalZipWriter.cpp Wed Jun 02 17:35:39 2021 +0200 @@ -154,9 +154,10 @@ } - HierarchicalZipWriter::HierarchicalZipWriter(ZipWriter::IOutputStream* stream) + HierarchicalZipWriter::HierarchicalZipWriter(ZipWriter::IOutputStream* stream, + bool isZip64) { - writer_.AcquireOutputStream(stream); + writer_.AcquireOutputStream(stream, isZip64); writer_.Open(); } @@ -227,8 +228,24 @@ writer_.Write(data); } - HierarchicalZipWriter* HierarchicalZipWriter::CreateToMemory(std::string& target) + HierarchicalZipWriter* HierarchicalZipWriter::CreateToMemory(std::string& target, + bool isZip64) + { + return new HierarchicalZipWriter(new ZipWriter::MemoryStream(target), isZip64); + } + + void HierarchicalZipWriter::CancelStream() { - return new HierarchicalZipWriter(new ZipWriter::MemoryStream(target)); + writer_.CancelStream(); + } + + void HierarchicalZipWriter::Close() + { + writer_.Close(); + } + + size_t HierarchicalZipWriter::GetArchiveSize() const + { + return writer_.GetArchiveSize(); } } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/Compression/HierarchicalZipWriter.h --- a/OrthancFramework/Sources/Compression/HierarchicalZipWriter.h Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/Compression/HierarchicalZipWriter.h Wed Jun 02 17:35:39 2021 +0200 @@ -83,7 +83,8 @@ public: explicit HierarchicalZipWriter(const char* path); - explicit HierarchicalZipWriter(ZipWriter::IOutputStream* stream); // transfers ownership + HierarchicalZipWriter(ZipWriter::IOutputStream* stream, // transfers ownership + bool isZip64); ~HierarchicalZipWriter(); @@ -112,6 +113,13 @@ void Write(const std::string& data); // The lifetime of the "target" buffer must be larger than that of HierarchicalZipWriter - static HierarchicalZipWriter* CreateToMemory(std::string& target); + static HierarchicalZipWriter* CreateToMemory(std::string& target, + bool isZip64); + + void CancelStream(); + + void Close(); + + size_t GetArchiveSize() const; }; } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/Compression/ZipWriter.cpp --- a/OrthancFramework/Sources/Compression/ZipWriter.cpp Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/Compression/ZipWriter.cpp Wed Jun 02 17:35:39 2021 +0200 @@ -33,8 +33,9 @@ #include #include "../../Resources/ThirdParty/minizip/zip.h" +#include "../Logging.h" #include "../OrthancException.h" -#include "../Logging.h" +#include "../SystemToolbox.h" static void PrepareFileInfo(zip_fileinfo& zfi) @@ -67,7 +68,8 @@ namespace Orthanc { ZipWriter::MemoryStream::MemoryStream(std::string& target) : - target_(target) + target_(target), + archiveSize_(0) { } @@ -75,9 +77,16 @@ void ZipWriter::MemoryStream::Write(const std::string& chunk) { chunked_.AddChunk(chunk); + archiveSize_ += chunk.size(); } + uint64_t ZipWriter::MemoryStream::GetArchiveSize() const + { + return archiveSize_; + } + + void ZipWriter::MemoryStream::Close() { chunked_.Flatten(target_); @@ -272,9 +281,13 @@ { try { - std::string s; - buffer_.Flush(s); - stream_.Write(s); + if (success_) + { + std::string s; + buffer_.Flush(s); + stream_.Write(s); + } + return 0; } catch (...) @@ -296,6 +309,10 @@ { return 0; } + else if (!success_) + { + return 0; // Error + } else { try @@ -317,7 +334,8 @@ try { if (origin == ZLIB_FILEFUNC_SEEK_SET && - offset >= startCurrentFile_) + offset >= startCurrentFile_ && + success_) { ZPOS64_T fullSize = startCurrentFile_ + static_cast(buffer_.GetSize()); assert(offset <= fullSize); @@ -339,7 +357,7 @@ } else { - return 1; // Should never occur + return 1; } } catch (...) @@ -347,6 +365,12 @@ return 1; } } + + + void Cancel() + { + success_ = false; + } static int CloseWrapper(voidpf opaque, @@ -398,12 +422,15 @@ }; - struct ZipWriter::PImpl + struct ZipWriter::PImpl : public boost::noncopyable { zipFile file_; std::unique_ptr streamBuffer_; + uint64_t archiveSize_; - PImpl() : file_(NULL) + PImpl() : + file_(NULL), + archiveSize_(0) { } }; @@ -442,6 +469,7 @@ if (outputStream_.get() != NULL) { outputStream_->Close(); + pimpl_->archiveSize_ = outputStream_->GetArchiveSize(); outputStream_.reset(NULL); } } @@ -542,8 +570,16 @@ void ZipWriter::SetZip64(bool isZip64) { - Close(); - isZip64_ = isZip64; + if (outputStream_.get() == NULL) + { + Close(); + isZip64_ = isZip64; + } + else + { + throw OrthancException(ErrorCode_BadSequenceOfCalls, + "SetZip64() must be given to AcquireOutputStream()"); + } } void ZipWriter::SetCompressionLevel(uint8_t level) @@ -554,9 +590,10 @@ "ZIP compression level must be between 0 (no compression) " "and 9 (highest compression)"); } - - Close(); - compressionLevel_ = level; + else + { + compressionLevel_ = level; + } } uint8_t ZipWriter::GetCompressionLevel() const @@ -657,7 +694,8 @@ } - void ZipWriter::AcquireOutputStream(IOutputStream* stream) + void ZipWriter::AcquireOutputStream(IOutputStream* stream, + bool isZip64) { std::unique_ptr protection(stream); @@ -669,13 +707,47 @@ { Close(); path_.clear(); + isZip64_ = isZip64; outputStream_.reset(protection.release()); } } - void ZipWriter::SetMemoryOutput(std::string& target) + void ZipWriter::SetMemoryOutput(std::string& target, + bool isZip64) + { + AcquireOutputStream(new MemoryStream(target), isZip64); + } + + + void ZipWriter::CancelStream() { - AcquireOutputStream(new MemoryStream(target)); + if (outputStream_.get() == NULL || + pimpl_->streamBuffer_.get() == NULL) + { + throw OrthancException(ErrorCode_BadSequenceOfCalls, "Only applicable after AcquireOutputStream() and Open()"); + } + else + { + pimpl_->streamBuffer_->Cancel(); + } + } + + + size_t ZipWriter::GetArchiveSize() const + { + if (outputStream_.get() != NULL) + { + return outputStream_->GetArchiveSize(); + } + else if (path_.empty()) + { + // This is the case after a call to "Close()" + return pimpl_->archiveSize_; + } + else + { + return SystemToolbox::GetFileSize(path_); + } } } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/Compression/ZipWriter.h --- a/OrthancFramework/Sources/Compression/ZipWriter.h Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/Compression/ZipWriter.h Wed Jun 02 17:35:39 2021 +0200 @@ -65,6 +65,8 @@ virtual void Write(const std::string& chunk) = 0; virtual void Close() = 0; + + virtual uint64_t GetArchiveSize() const = 0; }; @@ -74,6 +76,7 @@ private: std::string& target_; ChunkedBuffer chunked_; + uint64_t archiveSize_; public: MemoryStream(std::string& target); @@ -81,6 +84,8 @@ virtual void Write(const std::string& chunk) ORTHANC_OVERRIDE; virtual void Close() ORTHANC_OVERRIDE; + + virtual uint64_t GetArchiveSize() const ORTHANC_OVERRIDE; }; @@ -162,9 +167,17 @@ void Write(const std::string& data); - void AcquireOutputStream(IOutputStream* stream); // transfers ownership + void AcquireOutputStream(IOutputStream* stream, // transfers ownership + bool isZip64); // The lifetime of the "target" buffer must be larger than that of ZipWriter - void SetMemoryOutput(std::string& target); + void SetMemoryOutput(std::string& target, + bool isZip64); + + void CancelStream(); + + // WARNING: "GetArchiveSize()" only has its final value after + // "Close()" has been called + uint64_t GetArchiveSize() const; }; } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/HttpServer/HttpOutput.cpp --- a/OrthancFramework/Sources/HttpServer/HttpOutput.cpp Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/HttpServer/HttpOutput.cpp Wed Jun 02 17:35:39 2021 +0200 @@ -573,8 +573,7 @@ } - void HttpOutput::StateMachine::StartMultipart(const std::string& subType, - const std::string& contentType) + void HttpOutput::StateMachine::StartStreamInternal(const std::string& contentType) { if (state_ != State_WritingHeader) { @@ -620,22 +619,37 @@ header += "Connection: close\r\n"; } - // Possibly add the cookies - CheckHeadersCompatibilityWithMultipart(); - for (std::list::const_iterator it = headers_.begin(); it != headers_.end(); ++it) { header += *it; } + header += ("Content-Type: " + contentType + "\r\n\r\n"); + + stream_.Send(true, header.c_str(), header.size()); + } + + + void HttpOutput::StateMachine::StartMultipart(const std::string& subType, + const std::string& contentType) + { + CheckHeadersCompatibilityWithMultipart(); + std::string contentTypeHeader; PrepareMultipartMainHeader(multipartBoundary_, contentTypeHeader, subType, contentType); multipartContentType_ = contentType; - header += ("Content-Type: " + contentTypeHeader + "\r\n\r\n"); + + StartStreamInternal(contentTypeHeader); + + state_ = State_WritingMultipart; + } - stream_.Send(true, header.c_str(), header.size()); - state_ = State_WritingMultipart; + + void HttpOutput::StateMachine::StartStream(const std::string& contentType) + { + StartStreamInternal(contentType); + state_ = State_WritingStream; } @@ -736,6 +750,36 @@ } + void HttpOutput::StateMachine::SendStreamItem(const void* data, + size_t size) + { + if (state_ != State_WritingStream) + { + throw OrthancException(ErrorCode_BadSequenceOfCalls); + } + else + { + if (size > 0) + { + stream_.Send(false, data, size); + } + } + } + + + void HttpOutput::StateMachine::CloseStream() + { + if (state_ != State_WritingStream) + { + throw OrthancException(ErrorCode_BadSequenceOfCalls); + } + else + { + state_ = State_Done; + } + } + + static void AnswerStreamAsBuffer(HttpOutput& output, IHttpStreamAnswer& stream) { @@ -860,4 +904,29 @@ chunked.Flatten(body); Answer(body); } + + + void HttpOutput::AnswerWithoutBuffering(IHttpStreamAnswer& stream) + { + std::string contentType = stream.GetContentType(); + if (contentType.empty()) + { + contentType = MIME_BINARY; + } + + std::string filename; + if (stream.HasContentFilename(filename)) + { + stateMachine_.AddHeader("Content-Disposition", "filename=\"" + std::string(filename) + "\""); + } + + stateMachine_.StartStream(contentType.c_str()); + + while (stream.ReadNextChunk()) + { + stateMachine_.SendStreamItem(stream.GetChunkContent(), stream.GetChunkSize()); + } + + stateMachine_.CloseStream(); + } } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/HttpServer/HttpOutput.h --- a/OrthancFramework/Sources/HttpServer/HttpOutput.h Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/HttpServer/HttpOutput.h Wed Jun 02 17:35:39 2021 +0200 @@ -47,7 +47,8 @@ State_WritingHeader, State_WritingBody, State_WritingMultipart, - State_Done + State_Done, + State_WritingStream }; private: @@ -64,6 +65,8 @@ std::string multipartBoundary_; std::string multipartContentType_; + void StartStreamInternal(const std::string& contentType); + public: StateMachine(IHttpOutputStream& stream, bool isKeepAlive); @@ -105,6 +108,13 @@ } void CheckHeadersCompatibilityWithMultipart() const; + + void StartStream(const std::string& contentType); + + void SendStreamItem(const void* data, + size_t size); + + void CloseStream(); }; StateMachine stateMachine_; @@ -185,5 +195,12 @@ const std::vector& parts, const std::vector& sizes, const std::vector*>& headers); + + /** + * Contrarily to "Answer()", this method doesn't bufferizes the + * stream before sending it, which reduces memory but cannot be + * used to handle compression using "Content-Encoding". + **/ + void AnswerWithoutBuffering(IHttpStreamAnswer& stream); }; } diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/RestApi/RestApiOutput.cpp --- a/OrthancFramework/Sources/RestApi/RestApiOutput.cpp Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/RestApi/RestApiOutput.cpp Wed Jun 02 17:35:39 2021 +0200 @@ -76,6 +76,15 @@ alreadySent_ = true; } + + void RestApiOutput::AnswerWithoutBuffering(IHttpStreamAnswer& stream) + { + CheckStatus(); + output_.AnswerWithoutBuffering(stream); + alreadySent_ = true; + } + + void RestApiOutput::AnswerJson(const Json::Value& value) { CheckStatus(); diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/Sources/RestApi/RestApiOutput.h --- a/OrthancFramework/Sources/RestApi/RestApiOutput.h Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/Sources/RestApi/RestApiOutput.h Wed Jun 02 17:35:39 2021 +0200 @@ -66,6 +66,8 @@ void AnswerStream(IHttpStreamAnswer& stream); + void AnswerWithoutBuffering(IHttpStreamAnswer& stream); + void AnswerJson(const Json::Value& value); void AnswerBuffer(const std::string& buffer, diff -r 42e1f5bde40b -r d9942d48fea7 OrthancFramework/UnitTestsSources/ZipTests.cpp --- a/OrthancFramework/UnitTestsSources/ZipTests.cpp Fri May 28 18:44:00 2021 +0200 +++ b/OrthancFramework/UnitTestsSources/ZipTests.cpp Wed Jun 02 17:35:39 2021 +0200 @@ -30,6 +30,7 @@ #include "../Sources/Compression/HierarchicalZipWriter.h" #include "../Sources/Compression/ZipReader.h" #include "../Sources/OrthancException.h" +#include "../Sources/SystemToolbox.h" #include "../Sources/TemporaryFile.h" #include "../Sources/Toolbox.h" @@ -189,10 +190,14 @@ { Orthanc::ZipWriter w; + ASSERT_EQ(0, w.GetArchiveSize()); + w.SetOutputPath(f.GetPath().c_str()); w.Open(); w.OpenFile("world/hello"); w.Write("Hello world"); + + ASSERT_EQ(w.GetArchiveSize(), SystemToolbox::GetFileSize(f.GetPath())); } ASSERT_TRUE(ZipReader::IsZipFile(f.GetPath())); @@ -225,11 +230,29 @@ { { Orthanc::ZipWriter w; - w.SetZip64(i == 0); - w.SetMemoryOutput(memory); + w.SetMemoryOutput(memory, (i == 0) /* ZIP64? */); w.Open(); w.OpenFile("world/hello"); + w.Write("Hello"); + w.CancelStream(); + } + + ASSERT_THROW(ZipReader::CreateFromMemory(memory), Orthanc::OrthancException); + + memory.clear(); + uint64_t archiveSize; + + { + Orthanc::ZipWriter w; + ASSERT_EQ(0, w.GetArchiveSize()); + + w.SetMemoryOutput(memory, (i == 0) /* ZIP64? */); + w.Open(); + + ASSERT_EQ(0, w.GetArchiveSize()); + + w.OpenFile("world/hello"); w.Write(large); w.OpenFile("world/hello2"); w.Write(large); @@ -237,8 +260,21 @@ w.Write("Hello world"); w.OpenFile("world/hello4"); w.Write(large); + + ASSERT_TRUE(memory.empty()); + + uint64_t s1 = w.GetArchiveSize(); + ASSERT_NE(0, s1); + + w.Close(); + archiveSize = w.GetArchiveSize(); + + ASSERT_NE(archiveSize, s1); + ASSERT_EQ(archiveSize, w.GetArchiveSize()); } + ASSERT_EQ(archiveSize, memory.size()); + std::unique_ptr reader(ZipReader::CreateFromMemory(memory)); ASSERT_EQ(4u, reader->GetFilesCount());