Mercurial > hg > orthanc-stone

changeset 1950:6ffd65cfd2ee deep-learning

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new branch: deep-learning
author Sebastien Jodogne <s.jodogne@gmail.com>
date Fri, 12 Aug 2022 15:48:35 +0200
parents d3ae6858e77b
children 060d61913e39
files Applications/StoneWebViewer/WebApplication/app.js Applications/StoneWebViewer/WebApplication/index.html Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp
diffstat 4 files changed, 197 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/Applications/StoneWebViewer/WebApplication/app.js	Fri Jul 15 16:19:18 2022 +0200
+++ b/Applications/StoneWebViewer/WebApplication/app.js	Fri Aug 12 15:48:35 2022 +0200
@@ -463,7 +463,10 @@
       series: [],
       studies: [],
       seriesIndex: {},  // Maps "SeriesInstanceUID" to "index in this.series"
-      virtualSeriesThumbnails: {}
+      virtualSeriesThumbnails: {},
+
+      deepLearningReady: false,
+      deepLearningProgress: 0  // Floating-point number in the range [0..1]
     }
   },
   computed: {
@@ -1102,6 +1105,11 @@
           alert('Cannot find the study in Orthanc');
         });
       
+    },
+
+    ApplyDeepLearning: function()
+    {
+      stone.ApplyDeepLearningModel(this.GetActiveCanvas());
     }
   },
   
@@ -1358,3 +1366,12 @@
     }
   }
 });
+
+
+window.addEventListener('DeepLearningInitialized', function() {
+  stone.LoadDeepLearningModel('model.message');
+});
+
+window.addEventListener('DeepLearningReady', function() {
+  app.deepLearningReady = true;
+});
--- a/Applications/StoneWebViewer/WebApplication/index.html	Fri Jul 15 16:19:18 2022 +0200
+++ b/Applications/StoneWebViewer/WebApplication/index.html	Fri Aug 12 15:48:35 2022 +0200
@@ -303,11 +303,23 @@
 
             </div>        
             <div class="wvLayoutLeft__contentBottom">
+              <div v-if="deepLearningReady">
+                <div style="width:100%;padding:10px;text-align:center;">
+                  <button class="btn btn-primary" @click="ApplyDeepLearning()">Apply deep learning</button>
+                </div>
+                
+                <div style="padding: 10px; position: relative; width:100%;">
+                  <div style="background-color: #007000; position: relative; height: 10px;">    
+                    <div v-bind:style="{ 'background-color': '#00ff00', position: 'absolute', height: '100%', width: (deepLearningProgress*100) + '%' }"></div>
+                  </div>
+                </div>
+              </div>
+                
               <div style="width:100%;padding:10px;text-align:center;"
                    v-if="globalConfiguration.InstitutionLogo != ''">
                 <img style="max-width:100%" v-bind:src="globalConfiguration.InstitutionLogo" />
               </div>
-            </div>        
+            </div>
           </div>
         </div>
         <div class="wvLayout__main"
--- a/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt	Fri Jul 15 16:19:18 2022 +0200
+++ b/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt	Fri Aug 12 15:48:35 2022 +0200
@@ -24,6 +24,8 @@
 project(OrthancStone)
 include(${CMAKE_SOURCE_DIR}/../Version.cmake)
 
+include(${CMAKE_SOURCE_DIR}/deep-learning/WebAssembly/Protobuf.cmake)  # TODO
+
 set(ORTHANC_STONE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/../../../wasm-binaries/StoneWebViewer" CACHE PATH "Where to put the WebAssembly binaries")
 
 
@@ -131,6 +133,8 @@
 add_executable(StoneWebViewer
   ${ORTHANC_STONE_SOURCES}
   ${AUTOGENERATED_SOURCES}
+  ${PROTOBUF_SOURCES}  # TODO
+  ${CMAKE_SOURCE_DIR}/deep-learning/WebAssembly/Worker.pb.cc  # TODO
   StoneWebViewer.cpp
   )
 
--- a/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp	Fri Jul 15 16:19:18 2022 +0200
+++ b/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp	Fri Aug 12 15:48:35 2022 +0200
@@ -2281,7 +2281,7 @@
     {
       const size_t cursorIndex = cursor_->GetCurrentIndex();
       const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex);
-      const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
+      const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
 
       // Only change the scene if the loaded frame still corresponds to the current cursor
       if (instance.GetSopInstanceUid() == loadedSopInstanceUid &&
@@ -2605,7 +2605,7 @@
       {
         const size_t cursorIndex = cursor_->GetCurrentIndex();
         const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex);
-        const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
+        const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
 
         StoneAnnotationsRegistry::GetInstance().Save(instance.GetSopInstanceUid(), frameNumber, *stoneAnnotations_);
 
@@ -2800,7 +2800,7 @@
       const size_t cursorIndex = cursor_->GetCurrentIndex();
 
       const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex);
-      const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
+      const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
 
       FramesCache::Accessor accessor(*framesCache_, instance.GetSopInstanceUid(), frameNumber);
       if (accessor.IsValid())
@@ -3302,7 +3302,7 @@
     {
       const size_t cursorIndex = cursor_->GetCurrentIndex();
       const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex);
-      const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
+      const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
 
       if (instance.GetSopInstanceUid() == sopInstanceUid &&
           frameNumber == frame)
@@ -3317,6 +3317,25 @@
       }
     }    
   }
+
+
+  bool GetCurrentFrame(std::string& sopInstanceUid,
+                       unsigned int& frameNumber) const
+  {
+    if (cursor_.get() != NULL &&
+        frames_.get() != NULL)
+    {
+      const size_t cursorIndex = cursor_->GetCurrentIndex();
+      const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex);
+      sopInstanceUid = instance.GetSopInstanceUid();
+      frameNumber = frames_->GetFrameNumberInInstance(cursorIndex);
+      return true;
+    }
+    else
+    {
+      return false;
+    }
+  }
 };
 
 
@@ -3646,6 +3665,70 @@
 }
 
 
+#include <emscripten/fetch.h>
+#include "deep-learning/WebAssembly/Worker.pb.h"
+
+static void DeepLearningCallback(char* data,
+                                 int size,
+                                 void* payload)
+{
+  OrthancStone::Messages::Response response;
+  if (response.ParseFromArray(data, size))
+  {
+    switch (response.type())
+    {
+      case OrthancStone::Messages::ResponseType::INITIALIZED:
+        DISPATCH_JAVASCRIPT_EVENT("DeepLearningInitialized");
+        break;
+
+      case OrthancStone::Messages::ResponseType::PARSED_MODEL:
+        LOG(WARNING) << "Number of steps in the model: " << response.parse_model().number_of_steps();
+        DISPATCH_JAVASCRIPT_EVENT("DeepLearningReady");
+        break;
+
+      default:
+        LOG(ERROR) << "Unsupported response type from the deep learning worker";
+    }
+  }
+  else
+  {
+    LOG(ERROR) << "Bad response received from the deep learning worker";
+  }
+}
+
+static worker_handle deepLearningWorker_;
+
+static void SendRequestToWebWorker(const OrthancStone::Messages::Request& request)
+{
+  std::string s;
+  if (request.SerializeToString(&s) &&
+      !s.empty())
+  {
+    emscripten_call_worker(deepLearningWorker_, "Execute", &s[0], s.size(), DeepLearningCallback, NULL);
+  }
+  else
+  {
+    throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError,
+                                    "Cannot send command to the Web worker");
+  }
+}
+
+static void DeepLearningModelLoaded(emscripten_fetch_t *fetch)
+{
+  try
+  {
+    LOG(WARNING) << "Deep learning model loaded: " << fetch->numBytes;
+
+    OrthancStone::Messages::Request request;
+    request.set_type(OrthancStone::Messages::RequestType::PARSE_MODEL);
+    request.mutable_parse_model()->mutable_content()->assign(fetch->data, fetch->numBytes);
+    
+    emscripten_fetch_close(fetch);  // Don't use "fetch" below
+    SendRequestToWebWorker(request);
+  }
+  EXTERN_CATCH_EXCEPTIONS;
+}
+
 extern "C"
 {
   int main(int argc, char const *argv[]) 
@@ -3661,11 +3744,86 @@
     framesCache_.reset(new FramesCache);
     osiriXAnnotations_.reset(new OrthancStone::OsiriX::CollectionOfAnnotations);
 
+    deepLearningWorker_ = emscripten_create_worker("DeepLearningWorker.js");
+    emscripten_call_worker(deepLearningWorker_, "Initialize", NULL, 0, DeepLearningCallback, NULL);
+
     DISPATCH_JAVASCRIPT_EVENT("StoneInitialized");
   }
 
 
   EMSCRIPTEN_KEEPALIVE
+  void LoadDeepLearningModel(const char* uri)
+  {
+    try
+    {
+      LOG(WARNING) << "Loading deep learning model: " << uri;
+
+      emscripten_fetch_attr_t attr;
+      emscripten_fetch_attr_init(&attr);
+      strcpy(attr.requestMethod, "GET");
+      attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY;
+      attr.onsuccess = DeepLearningModelLoaded;
+      attr.onerror = NULL;
+      emscripten_fetch(&attr, uri);
+    }
+    EXTERN_CATCH_EXCEPTIONS;
+  }
+
+
+  EMSCRIPTEN_KEEPALIVE
+  void ApplyDeepLearningModel(const char* canvas)
+  {
+    try
+    {
+      boost::shared_ptr<ViewerViewport> viewport = GetViewport(canvas);
+
+      std::string sopInstanceUid;
+      unsigned int frameNumber;
+      if (viewport->GetCurrentFrame(sopInstanceUid, frameNumber))
+      {
+        LOG(ERROR) << "OK: " << sopInstanceUid << " / " << frameNumber;
+        
+        FramesCache::Accessor accessor(*framesCache_, sopInstanceUid, frameNumber);
+        if (accessor.IsValid() &&
+            accessor.GetImage().GetFormat() == Orthanc::PixelFormat_Float32)
+        {
+          const Orthanc::ImageAccessor& image = accessor.GetImage();
+          
+          OrthancStone::Messages::Request request;
+          request.set_type(OrthancStone::Messages::RequestType::LOAD_IMAGE);
+          request.mutable_load_image()->set_sop_instance_uid(sopInstanceUid);
+          request.mutable_load_image()->set_frame_number(frameNumber);
+          request.mutable_load_image()->set_width(image.GetWidth());
+          request.mutable_load_image()->set_height(image.GetHeight());
+
+          const unsigned int height = image.GetHeight();
+          const unsigned int width = image.GetWidth();
+          for (unsigned int y = 0; y < height; y++)
+          {
+            const float* p = reinterpret_cast<const float*>(image.GetConstRow(y));
+            for (unsigned int x = 0; x < width; x++, p++)
+            {
+              request.mutable_load_image()->mutable_values()->Add(*p);
+            }
+          }
+          
+          SendRequestToWebWorker(request);
+        }
+        else
+        {
+          LOG(WARNING) << "Cannot access graylevel frame";
+        }
+      }
+      else
+      {
+        LOG(WARNING) << "No active frame";
+      }
+    }
+    EXTERN_CATCH_EXCEPTIONS;
+  }
+
+
+  EMSCRIPTEN_KEEPALIVE
   void SetDicomWebRoot(const char* uri,
                        int useRendered)
   {