Mercurial > hg > orthanc-stone
changeset 2204:e1613509a939 deep-learning tip
integration default->deep-learning, turning deep learning into plugin
| author | Sebastien Jodogne <s.jodogne@gmail.com> |
|---|---|
| date | Sat, 19 Apr 2025 14:46:27 +0200 (43 hours ago) |
| parents | 2795f1ee4a1a (current diff) dcfabb36dc21 (diff) |
| children | |
| files | Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp |
| diffstat | 4 files changed, 179 insertions(+), 394 deletions(-) [+] |
line wrap: on
line diff
--- a/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Sat Apr 19 11:47:20 2025 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Sat Apr 19 14:46:27 2025 +0200 @@ -114,7 +114,7 @@ add_custom_command( COMMAND - ${CLANG_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/ParseWebAssemblyExports.py --libclang=${LIBCLANG} ${CMAKE_SOURCE_DIR}/StoneWebViewer.cpp > ${STONE_WRAPPER} + ${CLANG_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/ParseWebAssemblyExports.py --libclang=${LIBCLANG} ${CMAKE_SOURCE_DIR}/StoneWebViewer.cpp ${CMAKE_SOURCE_DIR}/DeepLearning.cpp > ${STONE_WRAPPER} DEPENDS ${CMAKE_SOURCE_DIR}/StoneWebViewer.cpp ${CMAKE_SOURCE_DIR}/ParseWebAssemblyExports.py @@ -151,6 +151,7 @@ ${ORTHANC_STONE_SOURCES} ${AUTOGENERATED_SOURCES} # Populated by "EmbedResources()" ${AUTOGENERATED_DIR}/DeepLearningWorker.pb.cc + DeepLearning.cpp StoneWebViewer.cpp )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Applications/StoneWebViewer/WebAssembly/IStoneWebViewerContext.h Sat Apr 19 14:46:27 2025 +0200 @@ -0,0 +1,106 @@ +/** + * Stone of Orthanc + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2023 Osimis S.A., Belgium + * Copyright (C) 2021-2025 Sebastien Jodogne, ICTEAM UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +#pragma once + +#include "../../../OrthancStone/Sources/Scene2D/ISceneLayer.h" +#include "../../../OrthancStone/Sources/Toolbox/DicomInstanceParameters.h" + +#include <Images/ImageAccessor.h> + + +#define DISPATCH_JAVASCRIPT_EVENT(name) \ + EM_ASM( \ + const customEvent = document.createEvent("CustomEvent"); \ + customEvent.initCustomEvent(name, false, false, undefined); \ + window.dispatchEvent(customEvent); \ + ); + +#define EXTERN_CATCH_EXCEPTIONS \ + catch (Orthanc::OrthancException& e) \ + { \ + LOG(ERROR) << "OrthancException: " << e.What(); \ + DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ + } \ + catch (OrthancStone::StoneException& e) \ + { \ + LOG(ERROR) << "StoneException: " << e.What(); \ + DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ + } \ + catch (std::exception& e) \ + { \ + LOG(ERROR) << "Runtime error: " << e.what(); \ + DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ + } \ + catch (...) \ + { \ + LOG(ERROR) << "Native exception"; \ + DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ + } + + +// WARNING: This class can be shared by multiple viewports +class ILayerSource : public boost::noncopyable +{ +public: + virtual ~ILayerSource() + { + } + + virtual int GetDepth() const = 0; + + virtual OrthancStone::ISceneLayer* Create(const Orthanc::ImageAccessor& frame, + const OrthancStone::DicomInstanceParameters& instance, + unsigned int frameNumber, + double pixelSpacingX, + double pixelSpacingY, + const OrthancStone::CoordinateSystem3D& plane) = 0; +}; + + +class IStoneWebViewerContext : public boost::noncopyable +{ +public: + virtual ~IStoneWebViewerContext() + { + } + + virtual void RedrawAllViewports() = 0; + + // WARNING: The ImageAccessor will become invalid once leaving the + // JavaScript callback, do not keep a reference! + virtual bool GetSelectedFrame(Orthanc::ImageAccessor& target /* out */, + std::string& sopInstanceUid /* out */, + unsigned int& frameNumber /* out */, + const std::string& canvas /* in */) = 0; +}; + + +class IStoneWebViewerPlugin : public boost::noncopyable +{ +public: + virtual ~IStoneWebViewerPlugin() + { + } + + virtual ILayerSource& GetLayerSource() = 0; +};
--- a/Applications/StoneWebViewer/WebAssembly/ParseWebAssemblyExports.py Sat Apr 19 11:47:20 2025 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/ParseWebAssemblyExports.py Sat Apr 19 14:46:27 2025 +0200 @@ -45,8 +45,8 @@ parser.add_argument('--libclang', default = '', help = 'manually provides the path to the libclang shared library') -parser.add_argument('source', - help = 'Input C++ file') +parser.add_argument('sources', nargs='+', + help = 'Input C++ files') args = parser.parse_args() @@ -57,13 +57,6 @@ index = clang.cindex.Index.create() -# PARSE_SKIP_FUNCTION_BODIES prevents clang from failing because of -# undefined types, which prevents compilation of functions -tu = index.parse(args.source, - [ '-DEMSCRIPTEN_KEEPALIVE=__attribute__((annotate("WebAssembly")))', - '-DSTONE_WEB_VIEWER_EXPORT=__attribute__((annotate("WebAssembly")))'], - options = clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) - TEMPLATE = ''' @@ -197,8 +190,15 @@ for child in node.get_children(): Explore(child) -Explore(tu.cursor) +for source in args.sources: + # PARSE_SKIP_FUNCTION_BODIES prevents clang from failing because of + # undefined types, which prevents compilation of functions + tu = index.parse(source, + [ '-DEMSCRIPTEN_KEEPALIVE=__attribute__((annotate("WebAssembly")))', + '-DSTONE_WEB_VIEWER_EXPORT=__attribute__((annotate("WebAssembly")))'], + options = clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES) + Explore(tu.cursor) print(pystache.render(TEMPLATE, {
--- a/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp Sat Apr 19 11:47:20 2025 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp Sat Apr 19 14:46:27 2025 +0200 @@ -20,41 +20,12 @@ **/ +#include "IStoneWebViewerContext.h" + #include <EmbeddedResources.h> #include <emscripten.h> -#define DISPATCH_JAVASCRIPT_EVENT(name) \ - EM_ASM( \ - const customEvent = document.createEvent("CustomEvent"); \ - customEvent.initCustomEvent(name, false, false, undefined); \ - window.dispatchEvent(customEvent); \ - ); - - -#define EXTERN_CATCH_EXCEPTIONS \ - catch (Orthanc::OrthancException& e) \ - { \ - LOG(ERROR) << "OrthancException: " << e.What(); \ - DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ - } \ - catch (OrthancStone::StoneException& e) \ - { \ - LOG(ERROR) << "StoneException: " << e.What(); \ - DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ - } \ - catch (std::exception& e) \ - { \ - LOG(ERROR) << "Runtime error: " << e.what(); \ - DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ - } \ - catch (...) \ - { \ - LOG(ERROR) << "Native exception"; \ - DISPATCH_JAVASCRIPT_EVENT("StoneException"); \ - } - - // Orthanc framework includes #include <Cache/MemoryObjectCache.h> #include <DicomFormat/DicomArray.h> @@ -107,7 +78,6 @@ static const int LAYER_REFERENCE_LINES = 3; static const int LAYER_ANNOTATIONS_STONE = 5; static const int LAYER_ANNOTATIONS_OSIRIX = 4; -static const int LAYER_DEEP_LEARNING = 6; #if !defined(STONE_WEB_VIEWER_EXPORT) @@ -1790,25 +1760,6 @@ -// WARNING: This class can be shared by multiple viewports -class ILayerSource : public boost::noncopyable -{ -public: - virtual ~ILayerSource() - { - } - - virtual int GetDepth() const = 0; - - virtual OrthancStone::ISceneLayer* Create(const Orthanc::ImageAccessor& frame, - const OrthancStone::DicomInstanceParameters& instance, - unsigned int frameNumber, - double pixelSpacingX, - double pixelSpacingY, - const OrthancStone::CoordinateSystem3D& plane) = 0; -}; - - class LayersHolder : public boost::noncopyable { private: @@ -3927,70 +3878,6 @@ -class DeepLearningSegmentationSource : public ILayerSource -{ -private: - std::unique_ptr<Orthanc::ImageAccessor> mask_; - std::string sopInstanceUid_; - unsigned int frameNumber_; - -public: - DeepLearningSegmentationSource() : - frameNumber_(0) // Dummy initialization - { - } - - virtual int GetDepth() const ORTHANC_OVERRIDE - { - return LAYER_DEEP_LEARNING; - } - - void SetMask(const std::string& sopInstanceUid, - unsigned int frameNumber, - const Orthanc::ImageAccessor& mask) - { - sopInstanceUid_ = sopInstanceUid; - frameNumber_ = frameNumber; - mask_.reset(Orthanc::Image::Clone(mask)); - } - - virtual OrthancStone::ISceneLayer* Create(const Orthanc::ImageAccessor& frame, - const OrthancStone::DicomInstanceParameters& instance, - unsigned int frameNumber, - double pixelSpacingX, - double pixelSpacingY, - const OrthancStone::CoordinateSystem3D& plane) ORTHANC_OVERRIDE - { - if (mask_.get() != NULL && - sopInstanceUid_ == instance.GetSopInstanceUid() && - frameNumber_ == frameNumber) - { - std::unique_ptr<OrthancStone::LookupTableTextureSceneLayer> layer; - - std::vector<uint8_t> lut(4 * 256); - for (unsigned int v = 128; v < 256; v++) - { - lut[4 * v] = 196; - lut[4 * v + 1] = 0; - lut[4 * v + 2] = 0; - lut[4 * v + 3] = 196; - } - - layer.reset(new OrthancStone::LookupTableTextureSceneLayer(*mask_)); - layer->SetLookupTable(lut); - layer->SetPixelSpacing(pixelSpacingX, pixelSpacingY); - - return layer.release(); - } - else - { - return NULL; - } - } -}; - - - typedef std::map<std::string, boost::shared_ptr<ViewerViewport> > Viewports; static Viewports allViewports_; @@ -4001,8 +3888,6 @@ // Orientation markers, new in Stone Web viewer 2.4 static std::unique_ptr<OrientationMarkersSource> orientationMarkersSource_; -static std::unique_ptr<DeepLearningSegmentationSource> deepLearningSegmentationSource_; - static void UpdateReferenceLines() { if (showReferenceLines_) @@ -4321,6 +4206,18 @@ } +IStoneWebViewerPlugin* DeepLearningInitialization(IStoneWebViewerContext& context); + +typedef IStoneWebViewerPlugin* (*PluginInitializer) (IStoneWebViewerContext&); + +static const PluginInitializer pluginsInitializers_[] = { + DeepLearningInitialization, + NULL +}; + +static std::list< boost::shared_ptr<IStoneWebViewerPlugin> > plugins_; + + static boost::shared_ptr<ViewerViewport> GetViewport(const std::string& canvas) { Viewports::iterator found = allViewports_.find(canvas); @@ -4334,7 +4231,10 @@ viewport->AddLayerSource(*osiriXLayerSource_); viewport->AddLayerSource(*orientationMarkersSource_); - viewport->AddLayerSource(*deepLearningSegmentationSource_); + for (std::list< boost::shared_ptr<IStoneWebViewerPlugin> >::iterator it = plugins_.begin(); it != plugins_.end(); ++it) + { + viewport->AddLayerSource((*it)->GetLayerSource()); + } allViewports_[canvas] = viewport; return viewport; @@ -4346,233 +4246,50 @@ } -#include <emscripten/fetch.h> -#include <DeepLearningWorker.pb.h> - -enum DeepLearningState -{ - DeepLearningState_Waiting, - DeepLearningState_Pending, - DeepLearningState_Running -}; - -static DeepLearningState deepLearningState_ = DeepLearningState_Waiting; -static worker_handle deepLearningWorker_; -static std::string deepLearningPendingSopInstanceUid_; -static unsigned int deepLearningPendingFrameNumber_; - -// Forward declaration -static void DeepLearningCallback(char* data, - int size, - void* payload); - -static void SendRequestToWebWorker(const OrthancStone::Messages::Request& request) -{ - std::string s; - if (request.SerializeToString(&s) && - !s.empty()) - { - emscripten_call_worker(deepLearningWorker_, "Execute", &s[0], s.size(), DeepLearningCallback, NULL); - } - else - { - throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, - "Cannot send command to the Web worker"); - } -} - -static void DeepLearningSchedule(const std::string& sopInstanceUid, - unsigned int frameNumber) -{ - if (deepLearningState_ == DeepLearningState_Waiting) - { - LOG(WARNING) << "Starting deep learning on: " << sopInstanceUid << " / " << frameNumber; - - FramesCache::Accessor accessor(*framesCache_, sopInstanceUid, frameNumber); - if (accessor.IsValid() && - accessor.GetImage().GetFormat() == Orthanc::PixelFormat_Float32) - { - const Orthanc::ImageAccessor& image = accessor.GetImage(); - - OrthancStone::Messages::Request request; - request.set_type(OrthancStone::Messages::RequestType::LOAD_IMAGE); - request.mutable_load_image()->set_sop_instance_uid(sopInstanceUid); - request.mutable_load_image()->set_frame_number(frameNumber); - request.mutable_load_image()->set_width(image.GetWidth()); - request.mutable_load_image()->set_height(image.GetHeight()); - - const unsigned int height = image.GetHeight(); - const unsigned int width = image.GetWidth(); - for (unsigned int y = 0; y < height; y++) - { - const float* p = reinterpret_cast<const float*>(image.GetConstRow(y)); - for (unsigned int x = 0; x < width; x++, p++) - { - request.mutable_load_image()->mutable_values()->Add(*p); - } - } - - deepLearningState_ = DeepLearningState_Running; - SendRequestToWebWorker(request); - } - else - { - LOG(ERROR) << "Cannot access the frame content, maybe a color image?"; - - EM_ASM({ - const customEvent = document.createEvent("CustomEvent"); - customEvent.initCustomEvent("DeepLearningStep", false, false, - { "progress" : "0" }); - window.dispatchEvent(customEvent); - }); - } - } - else - { - deepLearningState_ = DeepLearningState_Pending; - deepLearningPendingSopInstanceUid_ = sopInstanceUid; - deepLearningPendingFrameNumber_ = frameNumber; - } -} - -static void DeepLearningNextStep() +class StoneWebViewerContext : public IStoneWebViewerContext { - switch (deepLearningState_) - { - case DeepLearningState_Pending: - deepLearningState_ = DeepLearningState_Waiting; - DeepLearningSchedule(deepLearningPendingSopInstanceUid_, deepLearningPendingFrameNumber_); - break; - - case DeepLearningState_Running: - { - OrthancStone::Messages::Request request; - request.set_type(OrthancStone::Messages::RequestType::EXECUTE_STEP); - SendRequestToWebWorker(request); - break; - } - - default: - throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, "Bad state for deep learning"); - } -} - -static void DeepLearningCallback(char* data, - int size, - void* payload) -{ - try - { - OrthancStone::Messages::Response response; - if (response.ParseFromArray(data, size)) - { - switch (response.type()) +public: + static StoneWebViewerContext& GetInstance() + { + static StoneWebViewerContext instance; + return instance; + } + + virtual void RedrawAllViewports() ORTHANC_OVERRIDE + { + for (Viewports::iterator it = allViewports_.begin(); it != allViewports_.end(); ++it) + { + assert(it->second != NULL); + it->second->Redraw(); + } + } + + virtual bool GetSelectedFrame(Orthanc::ImageAccessor& target /* out */, + std::string& sopInstanceUid /* out */, + unsigned int& frameNumber /* out */, + const std::string& canvas /* in */) ORTHANC_OVERRIDE + { + boost::shared_ptr<ViewerViewport> viewport = GetViewport(canvas); + + if (viewport->GetCurrentFrame(sopInstanceUid, frameNumber)) + { + FramesCache::Accessor accessor(*framesCache_, sopInstanceUid, frameNumber); + if (accessor.IsValid()) { - case OrthancStone::Messages::ResponseType::INITIALIZED: - DISPATCH_JAVASCRIPT_EVENT("DeepLearningInitialized"); - break; - - case OrthancStone::Messages::ResponseType::PARSED_MODEL: - LOG(WARNING) << "Number of steps in the model: " << response.parse_model().number_of_steps(); - DISPATCH_JAVASCRIPT_EVENT("DeepLearningModelReady"); - break; - - case OrthancStone::Messages::ResponseType::LOADED_IMAGE: - DeepLearningNextStep(); - break; - - case OrthancStone::Messages::ResponseType::STEP_DONE: - { - EM_ASM({ - const customEvent = document.createEvent("CustomEvent"); - customEvent.initCustomEvent("DeepLearningStep", false, false, - { "progress" : $0 }); - window.dispatchEvent(customEvent); - }, - response.step().progress() - ); - - if (response.step().done()) - { - deepLearningState_ = DeepLearningState_Waiting; - - const unsigned int height = response.step().mask().height(); - const unsigned int width = response.step().mask().width(); - - LOG(WARNING) << "SUCCESS! Mask: " << width << "x" << height << " for frame " - << response.step().mask().sop_instance_uid() << " / " - << response.step().mask().frame_number(); - - Orthanc::Image mask(Orthanc::PixelFormat_Grayscale8, width, height, false); - - size_t pos = 0; - for (unsigned int y = 0; y < height; y++) - { - uint8_t* p = reinterpret_cast<uint8_t*>(mask.GetRow(y)); - for (unsigned int x = 0; x < width; x++, p++, pos++) - { - *p = response.step().mask().values(pos) ? 255 : 0; - } - } - - deepLearningSegmentationSource_->SetMask(response.step().mask().sop_instance_uid(), - response.step().mask().frame_number(), mask); - - for (Viewports::iterator it = allViewports_.begin(); it != allViewports_.end(); ++it) - { - assert(it->second != NULL); - it->second->Redraw(); - } - } - else - { - DeepLearningNextStep(); - } - - break; - } - - default: - LOG(ERROR) << "Unsupported response type from the deep learning worker"; + accessor.GetImage().GetReadOnlyAccessor(target); + return true; + } + else + { + return false; } } else { - LOG(ERROR) << "Bad response received from the deep learning worker"; - } - } - EXTERN_CATCH_EXCEPTIONS; -} - -static void DeepLearningModelLoaded(emscripten_fetch_t *fetch) -{ - try - { - LOG(WARNING) << "Deep learning model loaded: " << fetch->numBytes; - - OrthancStone::Messages::Request request; - request.set_type(OrthancStone::Messages::RequestType::PARSE_MODEL); - request.mutable_parse_model()->mutable_content()->assign(fetch->data, fetch->numBytes); - - emscripten_fetch_close(fetch); // Don't use "fetch" below - SendRequestToWebWorker(request); - } - EXTERN_CATCH_EXCEPTIONS; -} - - -static void DeepLearningInitialization() -{ - deepLearningWorker_ = emscripten_create_worker("../stone-deep-learning/DeepLearningWorker.js"); - emscripten_call_worker(deepLearningWorker_, "Initialize", NULL, 0, DeepLearningCallback, NULL); -} - - -typedef void (*PluginInitializer) (); - -static const PluginInitializer pluginsInitializers_[] = { - DeepLearningInitialization, - NULL + LOG(WARNING) << "No active frame"; + return false; + } + } }; @@ -4596,11 +4313,13 @@ osiriXLayerSource_.reset(new OsiriXLayerSource); orientationMarkersSource_.reset(new OrientationMarkersSource); - deepLearningSegmentationSource_.reset(new DeepLearningSegmentationSource); - for (size_t i = 0; pluginsInitializers_[i] != NULL; i++) { - pluginsInitializers_[i] (); + std::unique_ptr<IStoneWebViewerPlugin> plugin(pluginsInitializers_[i] (StoneWebViewerContext::GetInstance())); + if (plugin.get() != NULL) + { + plugins_.push_back(boost::shared_ptr<IStoneWebViewerPlugin>(plugin.release())); + } } DISPATCH_JAVASCRIPT_EVENT("StoneInitialized"); @@ -4608,47 +4327,6 @@ EMSCRIPTEN_KEEPALIVE - void LoadDeepLearningModel(const char* uri) - { - try - { - LOG(WARNING) << "Loading deep learning model: " << uri; - - emscripten_fetch_attr_t attr; - emscripten_fetch_attr_init(&attr); - strcpy(attr.requestMethod, "GET"); - attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; - attr.onsuccess = DeepLearningModelLoaded; - attr.onerror = NULL; - emscripten_fetch(&attr, uri); - } - EXTERN_CATCH_EXCEPTIONS; - } - - - EMSCRIPTEN_KEEPALIVE - void ApplyDeepLearningModel(const char* canvas) - { - try - { - boost::shared_ptr<ViewerViewport> viewport = GetViewport(canvas); - - std::string sopInstanceUid; - unsigned int frameNumber; - if (viewport->GetCurrentFrame(sopInstanceUid, frameNumber)) - { - DeepLearningSchedule(sopInstanceUid, frameNumber); - } - else - { - LOG(WARNING) << "No active frame"; - } - } - EXTERN_CATCH_EXCEPTIONS; - } - - - EMSCRIPTEN_KEEPALIVE void SetDicomWebRoot(const char* uri, int useRendered) {