Mercurial > hg > orthanc-stone
changeset 2053:0d814292a17e deep-learning
integration mainline->deep-learning
| author | Sebastien Jodogne <s.jodogne@gmail.com> |
|---|---|
| date | Thu, 30 Mar 2023 17:20:26 +0200 |
| parents | 85ab86f10d01 (diff) d77fea5934fb (current diff) |
| children | d4e769a0961f |
| files | Applications/StoneWebViewer/WebApplication/app.js |
| diffstat | 14 files changed, 713 insertions(+), 70 deletions(-) [+] |
line wrap: on
line diff
--- a/Applications/Samples/WebAssembly/CMakeLists.txt Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/Samples/WebAssembly/CMakeLists.txt Thu Mar 30 17:20:26 2023 +0200 @@ -45,12 +45,14 @@ set(WASM_FLAGS "${WASM_FLAGS} -s SAFE_HEAP=1") endif() +set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ENVIRONMENT=web") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]'") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ERROR_ON_UNDEFINED_SYMBOLS=1") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_MEMORY=268435456") # 256MB + resize set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1") add_definitions( -DDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 + -DORTHANC_WEBGL2_HEAP_COMPAT=0 )
--- a/Applications/StoneWebViewer/WebApplication/app.js Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/StoneWebViewer/WebApplication/app.js Thu Mar 30 17:20:26 2023 +0200 @@ -603,7 +603,11 @@ series: [], studies: [], seriesIndex: {}, // Maps "SeriesInstanceUID" to "index in this.series" - virtualSeriesThumbnails: {} + virtualSeriesThumbnails: {}, + + deepLearningReady: false, + deepLearningProgress: 0, // Floating-point number in the range [0..1] + deepLearningStartTime: null } }, computed: { @@ -1301,6 +1305,11 @@ }); }, + ApplyDeepLearning: function() { + stone.ApplyDeepLearningModel(this.GetActiveCanvas()); + app.deepLearningStartTime = performance.now(); + }, + ChangeActiveSeries: function(offset) { var seriesTags = this.GetActiveViewportSeriesTags(); if (seriesTags !== null) { @@ -1726,3 +1735,18 @@ } } }); + + +window.addEventListener('DeepLearningInitialized', function() { + stone.LoadDeepLearningModel('model.message'); +}); + +window.addEventListener('DeepLearningModelReady', function() { + app.deepLearningReady = true; + app.deepLearningProgress = 0; +}); + +window.addEventListener('DeepLearningStep', function(args) { + app.deepLearningProgress = args.detail.progress; + console.log('Elapsed time: ' + Math.round(performance.now() - app.deepLearningStartTime) + 'ms'); +});
--- a/Applications/StoneWebViewer/WebApplication/index.html Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/StoneWebViewer/WebApplication/index.html Thu Mar 30 17:20:26 2023 +0200 @@ -316,11 +316,23 @@ </div> <div class="wvLayoutLeft__contentBottom"> + <div v-if="deepLearningReady"> + <div style="width:100%;padding:10px;text-align:center;"> + <button class="btn btn-primary" @click="ApplyDeepLearning()">Apply deep learning</button> + </div> + + <div style="padding: 10px; position: relative; width:100%;"> + <div style="background-color: #007000; position: relative; height: 10px;"> + <div v-bind:style="{ 'background-color': '#00ff00', position: 'absolute', height: '100%', width: (deepLearningProgress*100) + '%' }"></div> + </div> + </div> + </div> + <div style="width:100%;padding:10px;text-align:center;" v-if="globalConfiguration.InstitutionLogo != ''"> <img style="max-width:100%" v-bind:src="globalConfiguration.InstitutionLogo" /> </div> - </div> + </div> </div> </div> <div class="wvLayout__main"
--- a/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/CMakeLists.txt Thu Mar 30 17:20:26 2023 +0200 @@ -24,6 +24,8 @@ project(OrthancStone) include(${CMAKE_SOURCE_DIR}/../Version.cmake) +include(${CMAKE_SOURCE_DIR}/deep-learning/WebAssembly/Protobuf.cmake) # TODO + set(ORTHANC_STONE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/../../../wasm-binaries/StoneWebViewer" CACHE PATH "Where to put the WebAssembly binaries") @@ -34,14 +36,17 @@ set(WASM_FLAGS "${WASM_FLAGS} -s WASM=1 -s FETCH=1 -s ASSERTIONS=1 -s DISABLE_EXCEPTION_CATCHING=0") if (CMAKE_BUILD_TYPE STREQUAL "Debug") set(WASM_FLAGS "${WASM_FLAGS} -s SAFE_HEAP=1") + set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s GL_DEBUG=1 -s GL_ASSERTIONS=1 -s TRACE_WEBGL_CALLS=1") endif() +set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ENVIRONMENT=web") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]'") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ERROR_ON_UNDEFINED_SYMBOLS=1") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_MEMORY=268435456") # 256MB + resize set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1") add_definitions( -DDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 + -DORTHANC_WEBGL2_HEAP_COMPAT=0 ) @@ -131,6 +136,8 @@ add_executable(StoneWebViewer ${ORTHANC_STONE_SOURCES} ${AUTOGENERATED_SOURCES} + ${PROTOBUF_SOURCES} # TODO + ${CMAKE_SOURCE_DIR}/Worker.pb.cc # TODO StoneWebViewer.cpp )
--- a/Applications/StoneWebViewer/WebAssembly/StoneModule/CMakeLists.txt Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/StoneModule/CMakeLists.txt Thu Mar 30 17:20:26 2023 +0200 @@ -46,6 +46,7 @@ set(WASM_FLAGS "${WASM_FLAGS} -s SAFE_HEAP=1") endif() +set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ENVIRONMENT=web") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]'") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ERROR_ON_UNDEFINED_SYMBOLS=1") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ASSERTIONS=1 -s DISABLE_EXCEPTION_CATCHING=0") @@ -53,6 +54,7 @@ set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1") add_definitions( -DDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 + -DORTHANC_WEBGL2_HEAP_COMPAT=0 )
--- a/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp Thu Mar 30 17:20:01 2023 +0200 +++ b/Applications/StoneWebViewer/WebAssembly/StoneWebViewer.cpp Thu Mar 30 17:20:26 2023 +0200 @@ -1606,10 +1606,11 @@ private: static const int LAYER_TEXTURE = 0; static const int LAYER_OVERLAY = 1; - static const int LAYER_ORIENTATION_MARKERS = 2; - static const int LAYER_REFERENCE_LINES = 3; - static const int LAYER_ANNOTATIONS_OSIRIX = 4; - static const int LAYER_ANNOTATIONS_STONE = 5; + static const int LAYER_DEEP_LEARNING = 2; + static const int LAYER_ORIENTATION_MARKERS = 3; + static const int LAYER_REFERENCE_LINES = 4; + static const int LAYER_ANNOTATIONS_OSIRIX = 5; + static const int LAYER_ANNOTATIONS_STONE = 6; class ICommand : public Orthanc::IDynamicObject @@ -2025,9 +2026,12 @@ // coordinates of the current texture, with (0,0) corresponding to // the center of the top-left pixel boost::shared_ptr<OrthancStone::AnnotationsSceneLayer> stoneAnnotations_; - + bool linearInterpolation_; + boost::shared_ptr<Orthanc::ImageAccessor> deepLearningMask_; + std::string deepLearningSopInstanceUid_; + unsigned int deepLearningFrameNumber_; void ScheduleNextPrefetch() { @@ -2246,6 +2250,26 @@ } } + std::unique_ptr<OrthancStone::LookupTableTextureSceneLayer> deepLearningLayer; + + if (deepLearningMask_.get() != NULL && + deepLearningSopInstanceUid_ == instance.GetSopInstanceUid() && + deepLearningFrameNumber_ == frameIndex) + { + std::vector<uint8_t> lut(4 * 256); + for (unsigned int v = 128; v < 256; v++) + { + lut[4 * v] = 196; + lut[4 * v + 1] = 0; + lut[4 * v + 2] = 0; + lut[4 * v + 3] = 196; + } + + deepLearningLayer.reset(new OrthancStone::LookupTableTextureSceneLayer(*deepLearningMask_)); + deepLearningLayer->SetLookupTable(lut); + deepLearningLayer->SetPixelSpacing(pixelSpacingX, pixelSpacingY); + } + StoneAnnotationsRegistry::GetInstance().Load(*stoneAnnotations_, instance.GetSopInstanceUid(), frameIndex); // Orientation markers, new in Stone Web viewer 2.4 @@ -2322,6 +2346,15 @@ scene.DeleteLayer(LAYER_ORIENTATION_MARKERS); } + if (deepLearningLayer.get() != NULL) + { + scene.SetLayer(LAYER_DEEP_LEARNING, deepLearningLayer.release()); + } + else + { + scene.DeleteLayer(LAYER_DEEP_LEARNING); + } + stoneAnnotations_->Render(scene); // Necessary for "FitContent()" to work if (fitNextContent_) @@ -2348,7 +2381,7 @@ { const size_t cursorIndex = cursor_->GetCurrentIndex(); const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex); - const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); + const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); // Only change the scene if the loaded frame still corresponds to the current cursor if (instance.GetSopInstanceUid() == loadedSopInstanceUid && @@ -2654,7 +2687,7 @@ { const size_t cursorIndex = cursor_->GetCurrentIndex(); const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex); - const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); + const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); StoneAnnotationsRegistry::GetInstance().Save(instance.GetSopInstanceUid(), frameNumber, *stoneAnnotations_); @@ -2880,7 +2913,7 @@ const size_t cursorIndex = cursor_->GetCurrentIndex(); const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex); - const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); + const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); FramesCache::Accessor accessor(*framesCache_, instance.GetSopInstanceUid(), frameNumber); if (accessor.IsValid()) @@ -3454,7 +3487,7 @@ { const size_t cursorIndex = cursor_->GetCurrentIndex(); const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex); - const size_t frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); + const unsigned int frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); if (instance.GetSopInstanceUid() == sopInstanceUid && frameNumber == frame) @@ -3470,6 +3503,7 @@ } } + void SetLinearInterpolation(bool linearInterpolation) { if (linearInterpolation_ != linearInterpolation) @@ -3479,6 +3513,7 @@ } } + void AddTextAnnotation(const std::string& label, const OrthancStone::ScenePoint2D& pointedPosition, const OrthancStone::ScenePoint2D& labelPosition) @@ -3488,6 +3523,43 @@ } + bool GetCurrentFrame(std::string& sopInstanceUid /* out */, + unsigned int& frameNumber /* out */) const + { + if (cursor_.get() != NULL && + frames_.get() != NULL) + { + const size_t cursorIndex = cursor_->GetCurrentIndex(); + const OrthancStone::DicomInstanceParameters& instance = frames_->GetInstanceOfFrame(cursorIndex); + sopInstanceUid = instance.GetSopInstanceUid(); + frameNumber = frames_->GetFrameNumberInInstance(cursorIndex); + return true; + } + else + { + return false; + } + } + + + void SetDeepLearningMask(const std::string& sopInstanceUid, + unsigned int frameNumber, + const Orthanc::ImageAccessor& mask) + { + std::string currentSopInstanceUid; + unsigned int currentFrameNumber; + if (GetCurrentFrame(currentSopInstanceUid, currentFrameNumber) && + sopInstanceUid == currentSopInstanceUid && + frameNumber == currentFrameNumber) + { + deepLearningSopInstanceUid_ = sopInstanceUid; + deepLearningFrameNumber_ = frameNumber; + deepLearningMask_.reset(Orthanc::Image::Clone(mask)); + Redraw(); + } + } + + void SignalSynchronizedBrowsing() { if (synchronizationEnabled_ && @@ -3854,6 +3926,218 @@ } +#include <emscripten/fetch.h> +#include "Worker.pb.h" + +enum DeepLearningState +{ + DeepLearningState_Waiting, + DeepLearningState_Pending, + DeepLearningState_Running +}; + +static DeepLearningState deepLearningState_ = DeepLearningState_Waiting; +static worker_handle deepLearningWorker_; +static std::string deepLearningPendingSopInstanceUid_; +static unsigned int deepLearningPendingFrameNumber_; + +// Forward declaration +static void DeepLearningCallback(char* data, + int size, + void* payload); + +static void SendRequestToWebWorker(const OrthancStone::Messages::Request& request) +{ + std::string s; + if (request.SerializeToString(&s) && + !s.empty()) + { + emscripten_call_worker(deepLearningWorker_, "Execute", &s[0], s.size(), DeepLearningCallback, NULL); + } + else + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "Cannot send command to the Web worker"); + } +} + +static void DeepLearningSchedule(const std::string& sopInstanceUid, + unsigned int frameNumber) +{ + if (deepLearningState_ == DeepLearningState_Waiting) + { + LOG(WARNING) << "Starting deep learning on: " << sopInstanceUid << " / " << frameNumber; + + FramesCache::Accessor accessor(*framesCache_, sopInstanceUid, frameNumber); + if (accessor.IsValid() && + accessor.GetImage().GetFormat() == Orthanc::PixelFormat_Float32) + { + const Orthanc::ImageAccessor& image = accessor.GetImage(); + + OrthancStone::Messages::Request request; + request.set_type(OrthancStone::Messages::RequestType::LOAD_IMAGE); + request.mutable_load_image()->set_sop_instance_uid(sopInstanceUid); + request.mutable_load_image()->set_frame_number(frameNumber); + request.mutable_load_image()->set_width(image.GetWidth()); + request.mutable_load_image()->set_height(image.GetHeight()); + + const unsigned int height = image.GetHeight(); + const unsigned int width = image.GetWidth(); + for (unsigned int y = 0; y < height; y++) + { + const float* p = reinterpret_cast<const float*>(image.GetConstRow(y)); + for (unsigned int x = 0; x < width; x++, p++) + { + request.mutable_load_image()->mutable_values()->Add(*p); + } + } + + deepLearningState_ = DeepLearningState_Running; + SendRequestToWebWorker(request); + } + else + { + LOG(ERROR) << "Cannot access the frame content, maybe a color image?"; + + EM_ASM({ + const customEvent = document.createEvent("CustomEvent"); + customEvent.initCustomEvent("DeepLearningStep", false, false, + { "progress" : "0" }); + window.dispatchEvent(customEvent); + }); + } + } + else + { + deepLearningState_ = DeepLearningState_Pending; + deepLearningPendingSopInstanceUid_ = sopInstanceUid; + deepLearningPendingFrameNumber_ = frameNumber; + } +} + +static void DeepLearningNextStep() +{ + switch (deepLearningState_) + { + case DeepLearningState_Pending: + deepLearningState_ = DeepLearningState_Waiting; + DeepLearningSchedule(deepLearningPendingSopInstanceUid_, deepLearningPendingFrameNumber_); + break; + + case DeepLearningState_Running: + { + OrthancStone::Messages::Request request; + request.set_type(OrthancStone::Messages::RequestType::EXECUTE_STEP); + SendRequestToWebWorker(request); + break; + } + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, "Bad state for deep learning"); + } +} + +static void DeepLearningCallback(char* data, + int size, + void* payload) +{ + try + { + OrthancStone::Messages::Response response; + if (response.ParseFromArray(data, size)) + { + switch (response.type()) + { + case OrthancStone::Messages::ResponseType::INITIALIZED: + DISPATCH_JAVASCRIPT_EVENT("DeepLearningInitialized"); + break; + + case OrthancStone::Messages::ResponseType::PARSED_MODEL: + LOG(WARNING) << "Number of steps in the model: " << response.parse_model().number_of_steps(); + DISPATCH_JAVASCRIPT_EVENT("DeepLearningModelReady"); + break; + + case OrthancStone::Messages::ResponseType::LOADED_IMAGE: + DeepLearningNextStep(); + break; + + case OrthancStone::Messages::ResponseType::STEP_DONE: + { + EM_ASM({ + const customEvent = document.createEvent("CustomEvent"); + customEvent.initCustomEvent("DeepLearningStep", false, false, + { "progress" : $0 }); + window.dispatchEvent(customEvent); + }, + response.step().progress() + ); + + if (response.step().done()) + { + deepLearningState_ = DeepLearningState_Waiting; + + const unsigned int height = response.step().mask().height(); + const unsigned int width = response.step().mask().width(); + + LOG(WARNING) << "SUCCESS! Mask: " << width << "x" << height << " for frame " + << response.step().mask().sop_instance_uid() << " / " + << response.step().mask().frame_number(); + + Orthanc::Image mask(Orthanc::PixelFormat_Grayscale8, width, height, false); + + size_t pos = 0; + for (unsigned int y = 0; y < height; y++) + { + uint8_t* p = reinterpret_cast<uint8_t*>(mask.GetRow(y)); + for (unsigned int x = 0; x < width; x++, p++, pos++) + { + *p = response.step().mask().values(pos) ? 255 : 0; + } + } + + for (Viewports::iterator it = allViewports_.begin(); it != allViewports_.end(); ++it) + { + assert(it->second != NULL); + it->second->SetDeepLearningMask(response.step().mask().sop_instance_uid(), + response.step().mask().frame_number(), mask); + } + } + else + { + DeepLearningNextStep(); + } + + break; + } + + default: + LOG(ERROR) << "Unsupported response type from the deep learning worker"; + } + } + else + { + LOG(ERROR) << "Bad response received from the deep learning worker"; + } + } + EXTERN_CATCH_EXCEPTIONS; +} + +static void DeepLearningModelLoaded(emscripten_fetch_t *fetch) +{ + try + { + LOG(WARNING) << "Deep learning model loaded: " << fetch->numBytes; + + OrthancStone::Messages::Request request; + request.set_type(OrthancStone::Messages::RequestType::PARSE_MODEL); + request.mutable_parse_model()->mutable_content()->assign(fetch->data, fetch->numBytes); + + emscripten_fetch_close(fetch); // Don't use "fetch" below + SendRequestToWebWorker(request); + } + EXTERN_CATCH_EXCEPTIONS; +} + extern "C" { int main(int argc, char const *argv[]) @@ -3871,11 +4155,55 @@ framesCache_.reset(new FramesCache); osiriXAnnotations_.reset(new OrthancStone::OsiriX::CollectionOfAnnotations); + deepLearningWorker_ = emscripten_create_worker("DeepLearningWorker.js"); + emscripten_call_worker(deepLearningWorker_, "Initialize", NULL, 0, DeepLearningCallback, NULL); + DISPATCH_JAVASCRIPT_EVENT("StoneInitialized"); } EMSCRIPTEN_KEEPALIVE + void LoadDeepLearningModel(const char* uri) + { + try + { + LOG(WARNING) << "Loading deep learning model: " << uri; + + emscripten_fetch_attr_t attr; + emscripten_fetch_attr_init(&attr); + strcpy(attr.requestMethod, "GET"); + attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; + attr.onsuccess = DeepLearningModelLoaded; + attr.onerror = NULL; + emscripten_fetch(&attr, uri); + } + EXTERN_CATCH_EXCEPTIONS; + } + + + EMSCRIPTEN_KEEPALIVE + void ApplyDeepLearningModel(const char* canvas) + { + try + { + boost::shared_ptr<ViewerViewport> viewport = GetViewport(canvas); + + std::string sopInstanceUid; + unsigned int frameNumber; + if (viewport->GetCurrentFrame(sopInstanceUid, frameNumber)) + { + DeepLearningSchedule(sopInstanceUid, frameNumber); + } + else + { + LOG(WARNING) << "No active frame"; + } + } + EXTERN_CATCH_EXCEPTIONS; + } + + + EMSCRIPTEN_KEEPALIVE void SetDicomWebRoot(const char* uri, int useRendered) {
--- a/OrthancStone/Resources/WebAssemblySharedLibrary/CMakeLists.txt Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Resources/WebAssemblySharedLibrary/CMakeLists.txt Thu Mar 30 17:20:26 2023 +0200 @@ -54,6 +54,7 @@ set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s DISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1") add_definitions( -DDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 + -DORTHANC_WEBGL2_HEAP_COMPAT=0 )
--- a/OrthancStone/Resources/WebAssemblyUnitTests/CMakeLists.txt Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Resources/WebAssemblyUnitTests/CMakeLists.txt Thu Mar 30 17:20:26 2023 +0200 @@ -39,6 +39,7 @@ set(WASM_FLAGS "${WASM_FLAGS} -s SAFE_HEAP=1") endif() +set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ENVIRONMENT=web") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]'") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ERROR_ON_UNDEFINED_SYMBOLS=1") set(WASM_LINKER_FLAGS "${WASM_LINKER_FLAGS} -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_MEMORY=268435456") # 256MB + resize @@ -46,6 +47,7 @@ add_definitions( -DDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR=1 -DORTHANC_BUILD_UNIT_TESTS=1 + -DORTHANC_WEBGL2_HEAP_COMPAT=0 )
--- a/OrthancStone/Sources/OpenGL/OpenGLProgram.cpp Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/OpenGL/OpenGLProgram.cpp Thu Mar 30 17:20:26 2023 +0200 @@ -86,6 +86,12 @@ { //ORTHANC_OPENGL_TRACE_CURRENT_CONTEXT("About to call glUseProgram"); glUseProgram(program_); + if (glGetError() != GL_NO_ERROR) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "Cannot use successfully compiled OpenGL shader"); + } + ORTHANC_OPENGL_CHECK("glUseProgram"); }
--- a/OrthancStone/Sources/OpenGL/OpenGLShader.cpp Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/OpenGL/OpenGLShader.cpp Thu Mar 30 17:20:26 2023 +0200 @@ -55,8 +55,10 @@ glCompileShader(shader); ORTHANC_OPENGL_CHECK("glCompileShader"); + GLenum error = glGetError(); + // Check if there were errors - int infoLen = 0; + GLint infoLen = 0; glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); ORTHANC_OPENGL_CHECK("glGetShaderiv"); @@ -65,18 +67,24 @@ std::string infoLog; infoLog.resize(infoLen + 1); glGetShaderInfoLog(shader, infoLen, NULL, &infoLog[0]); - ORTHANC_OPENGL_CHECK("glGetShaderInfoLog"); - ORTHANC_OPENGL_TRACE_CURRENT_CONTEXT("About to call glDeleteShader"); - glDeleteShader(shader); - ORTHANC_OPENGL_CHECK("glDeleteShader"); - throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, - "Error while creating an OpenGL shader: " + infoLog); + if (error) + { + ORTHANC_OPENGL_CHECK("glGetShaderInfoLog"); + ORTHANC_OPENGL_TRACE_CURRENT_CONTEXT("About to call glDeleteShader"); + glDeleteShader(shader); + ORTHANC_OPENGL_CHECK("glDeleteShader"); + + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "Error while creating an OpenGL shader: " + infoLog); + } + else + { + LOG(WARNING) << "Warning while creating an OpenGL shader: " << infoLog; + } } - else - { - return shader; - } + + return shader; } }
--- a/OrthancStone/Sources/OpenGL/OpenGLTexture.cpp Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/OpenGL/OpenGLTexture.cpp Thu Mar 30 17:20:26 2023 +0200 @@ -24,26 +24,37 @@ #include "OpenGLTexture.h" #include "IOpenGLContext.h" +#include <Images/Image.h> #include <Logging.h> #include <OrthancException.h> +#if defined(__EMSCRIPTEN__) +# if !defined(ORTHANC_WEBGL2_HEAP_COMPAT) +# error The macro ORTHANC_WEBGL2_HEAP_COMPAT must be defined +# endif +#endif + namespace OrthancStone { namespace OpenGL { - OpenGLTexture::OpenGLTexture(OpenGL::IOpenGLContext& context) - : width_(0) - , height_(0) - , context_(context) + OpenGLTexture::OpenGLTexture(OpenGL::IOpenGLContext& context) : + texture_(0), + width_(0), + height_(0), + format_(Orthanc::PixelFormat_Grayscale8), + context_(context) { if (!context_.IsContextLost()) { // Generate a texture object glGenTextures(1, &texture_); + ORTHANC_OPENGL_CHECK("glGenTextures()"); + if (texture_ == 0) { throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, - "Cannot create an OpenGL program"); + "Cannot create an OpenGL texture"); } } } @@ -80,56 +91,89 @@ } } - void OpenGLTexture::Load(const Orthanc::ImageAccessor& image, - bool isLinearInterpolation) + void OpenGLTexture::Setup(Orthanc::PixelFormat format, + unsigned int width, + unsigned int height, + bool isLinearInterpolation, + const void* data) { - if (!context_.IsContextLost()) + if (context_.IsContextLost()) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "OpenGL context has been lost"); + } + else { glPixelStorei(GL_UNPACK_ALIGNMENT, 1); // Disable byte-alignment restriction - if (image.GetPitch() != image.GetBytesPerPixel() * image.GetWidth()) - { - throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented, - "Unsupported non-zero padding"); - } - // Bind it glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, texture_); - GLenum sourceFormat, internalFormat; - - switch (image.GetFormat()) - { - case Orthanc::PixelFormat_Grayscale8: - sourceFormat = GL_RED; - internalFormat = GL_RED; - break; - - case Orthanc::PixelFormat_RGB24: - sourceFormat = GL_RGB; - internalFormat = GL_RGB; - break; + GLenum sourceFormat, internalFormat, pixelType; + ConvertToOpenGLFormats(sourceFormat, internalFormat, pixelType, format); - case Orthanc::PixelFormat_RGBA32: - sourceFormat = GL_RGBA; - internalFormat = GL_RGBA; - break; - - default: - throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented, - "No support for this format in OpenGL textures: " + - std::string(EnumerationToString(image.GetFormat()))); - } - - width_ = image.GetWidth(); - height_ = image.GetHeight(); + format_ = format; + width_ = width; + height_ = height; GLint interpolation = (isLinearInterpolation ? GL_LINEAR : GL_NEAREST); // Load the texture from the image buffer - glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, image.GetWidth(), image.GetHeight(), - 0, sourceFormat, GL_UNSIGNED_BYTE, image.GetConstBuffer()); + +#if defined(__EMSCRIPTEN__) && (ORTHANC_WEBGL2_HEAP_COMPAT == 1) + /** + * This compatibility implementation seems to be necessary + * with WebGL2, at least in Web workers. In such a situation, + * the calls that are referred to as the "new garbage-free + * entry points" in the Emscripten source file + * "upstream/emscripten/src/library_webgl.js" seem to fail, + * because the "Uint8Array" and "Float32Array" seem to be + * incorrectly created. This compatibility reverts to the + * WebGL1 behavior of "library_webgl.js", which requires the + * function "emscriptenWebGLGetTexPixelData" that is defined + * in "upstream/emscripten/src/library_webgl.js" to be + * exported in the linker using option + * "EXTRA_EXPORTED_RUNTIME_METHODS" or + * "EXPORTED_RUNTIME_METHODS". + **/ + EM_ASM({ + var ptr = $0 ? emscriptenWebGLGetTexPixelData($5, $4, $2, $3, $0, $1) : null; + GLctx.texImage2D(GLctx.TEXTURE_2D, 0, $1, $2, $3, 0, $4, $5, ptr); + }, + data, // $0 + internalFormat, // $1 + width, // $2 + height, // $3 + sourceFormat, // $4 + pixelType); // $5 +#else + glTexImage2D(GL_TEXTURE_2D, 0, internalFormat, width, height, + 0, sourceFormat, pixelType, data); +#endif + + ORTHANC_OPENGL_CHECK("glTexImage2D()"); + +#if !defined(__EMSCRIPTEN__) + /** + * glGetTexLevelParameteriv() was introduced in OpenGL ES 3.1, + * but WebGL 2 only supports OpenGL ES 3.0, so it is not + * available in WebAssembly: + * https://registry.khronos.org/OpenGL-Refpages/es3.1/html/glGetTexLevelParameter.xhtml + **/ + GLint w, h; + glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &w); + glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &h); + if (width != static_cast<unsigned int>(w) || + height != static_cast<unsigned int>(h)) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "Your GPU cannot create a texture of size " + + boost::lexical_cast<std::string>(width) + " x " + + boost::lexical_cast<std::string>(height)); + } +#endif + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, interpolation); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, interpolation); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); @@ -137,12 +181,143 @@ } } + void OpenGLTexture::Load(const Orthanc::ImageAccessor& image, + bool isLinearInterpolation) + { + if (image.GetPitch() != image.GetBytesPerPixel() * image.GetWidth()) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented, + "Pitch is not the same as the row size"); + } + else + { + Setup(image.GetFormat(), image.GetWidth(), image.GetHeight(), + isLinearInterpolation, image.GetConstBuffer()); + } + } - void OpenGLTexture::Bind(GLint location) + + void OpenGLTexture::Bind(GLint location) const { glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, texture_); glUniform1i(location, 0 /* texture unit */); } + + + void OpenGLTexture::BindAsTextureUnit(GLint location, + unsigned int unit) const + { + if (unit >= 32) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); + } + + assert(GL_TEXTURE0 + 1 == GL_TEXTURE1 && + GL_TEXTURE0 + 31 == GL_TEXTURE31); + + glActiveTexture(GL_TEXTURE0 + unit); + glBindTexture(GL_TEXTURE_2D, texture_); + glUniform1i(location, unit /* texture unit */); + } + + + Orthanc::ImageAccessor* OpenGLTexture::Download(Orthanc::PixelFormat format) const + { + if (context_.IsContextLost()) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError, + "OpenGL context is lost"); + } + + std::unique_ptr<Orthanc::ImageAccessor> target(new Orthanc::Image(format, width_, height_, true)); + assert(target->GetPitch() == width_ * Orthanc::GetBytesPerPixel(format)); + +#if defined(__EMSCRIPTEN__) + /** + * The "glGetTexImage()" is unavailable in WebGL, it is + * necessary to use a framebuffer: + * https://stackoverflow.com/a/15064957 + **/ + throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented); + +#else + glBindTexture(GL_TEXTURE_2D, texture_); + + switch (format) + { + case Orthanc::PixelFormat_Grayscale8: + glGetTexImage(GL_TEXTURE_2D, 0 /* base level */, GL_RED, GL_UNSIGNED_BYTE, target->GetBuffer()); + break; + + case Orthanc::PixelFormat_RGB24: + glGetTexImage(GL_TEXTURE_2D, 0 /* base level */, GL_RGB, GL_UNSIGNED_BYTE, target->GetBuffer()); + break; + + case Orthanc::PixelFormat_RGBA32: + glGetTexImage(GL_TEXTURE_2D, 0 /* base level */, GL_RGBA, GL_UNSIGNED_BYTE, target->GetBuffer()); + break; + + case Orthanc::PixelFormat_Float32: + glGetTexImage(GL_TEXTURE_2D, 0 /* base level */, GL_RED, GL_FLOAT, target->GetBuffer()); + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented); + } +#endif + + return target.release(); + } + + + void OpenGLTexture::SetClampingToZero() + { + glBindTexture(GL_TEXTURE_2D, texture_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER); + + GLfloat colorfv[4] = { 0, 0, 0, 0 }; + glTextureParameterfv(texture_, GL_TEXTURE_BORDER_COLOR, colorfv); + } + + + void OpenGLTexture::ConvertToOpenGLFormats(GLenum& sourceFormat, + GLenum& internalFormat, + GLenum& pixelType, + Orthanc::PixelFormat format) + { + switch (format) + { + case Orthanc::PixelFormat_Grayscale8: + sourceFormat = GL_RED; + internalFormat = GL_RED; + pixelType = GL_UNSIGNED_BYTE; + break; + + case Orthanc::PixelFormat_RGB24: + sourceFormat = GL_RGB; + internalFormat = GL_RGB; + pixelType = GL_UNSIGNED_BYTE; + break; + + case Orthanc::PixelFormat_RGBA32: + sourceFormat = GL_RGBA; + internalFormat = GL_RGBA; + pixelType = GL_UNSIGNED_BYTE; + break; + + case Orthanc::PixelFormat_Float32: + sourceFormat = GL_RED; + internalFormat = GL_R32F; // Don't use "GL_RED" here, as it clamps to [0,1] + pixelType = GL_FLOAT; + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_NotImplemented, + "No support for this format in OpenGL textures: " + + std::string(EnumerationToString(format))); + } + } } }
--- a/OrthancStone/Sources/OpenGL/OpenGLTexture.h Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/OpenGL/OpenGLTexture.h Thu Mar 30 17:20:26 2023 +0200 @@ -24,6 +24,7 @@ #pragma once #include "OpenGLIncludes.h" +#include "IOpenGLContext.h" #include <Images/ImageAccessor.h> @@ -34,21 +35,41 @@ { namespace OpenGL { - class IOpenGLContext; - class OpenGLTexture : public boost::noncopyable { private: GLuint texture_; unsigned int width_; unsigned int height_; + + Orthanc::PixelFormat format_; OpenGL::IOpenGLContext& context_; + void Setup(Orthanc::PixelFormat format, + unsigned int width, + unsigned int height, + bool isLinearInterpolation, + const void* data); + public: explicit OpenGLTexture(OpenGL::IOpenGLContext& context); ~OpenGLTexture(); + /** + * Returns the low-level OpenGL handle of the texture. Beware to + * never change the size of the texture using this handle! + **/ + GLuint GetId() const + { + return texture_; + } + + Orthanc::PixelFormat GetFormat() const + { + return format_; + } + unsigned int GetWidth() const { return width_; @@ -59,10 +80,34 @@ return height_; } + void Setup(Orthanc::PixelFormat format, + unsigned int width, + unsigned int height, + bool isLinearInterpolation) + { + Setup(format, width, height, isLinearInterpolation, NULL); + } + void Load(const Orthanc::ImageAccessor& image, bool isLinearInterpolation); - void Bind(GLint location); + void Bind(GLint location) const; + + void BindAsTextureUnit(GLint location, + unsigned int unit) const; + + Orthanc::ImageAccessor* Download(Orthanc::PixelFormat format) const; + + /** + * By default, textures are mirrored at the borders. This + * function will set out-of-image access to zero. + **/ + void SetClampingToZero(); + + static void ConvertToOpenGLFormats(GLenum& sourceFormat, + GLenum& internalFormat, + GLenum& pixelType, + Orthanc::PixelFormat format); }; } }
--- a/OrthancStone/Sources/Platforms/WebAssembly/WebAssemblyOpenGLContext.cpp Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/Platforms/WebAssembly/WebAssemblyOpenGLContext.cpp Thu Mar 30 17:20:26 2023 +0200 @@ -45,7 +45,8 @@ bool isContextLost_; public: - explicit PImpl(const std::string& canvasSelector) : + explicit PImpl(const std::string& canvasSelector, + Version version) : canvasSelector_(canvasSelector), isContextLost_(false) { @@ -53,6 +54,20 @@ EmscriptenWebGLContextAttributes attr; emscripten_webgl_init_context_attributes(&attr); + switch (version) + { + case Version_WebGL1: + break; + + case Version_WebGL2: + attr.majorVersion = 2; + attr.minorVersion = 0; + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); + } + // The next line might be necessary to print using // WebGL. Sometimes, if set to "false" (the default value), // the canvas was rendered as a fully white or black @@ -162,7 +177,13 @@ WebAssemblyOpenGLContext::WebAssemblyOpenGLContext(const std::string& canvasSelector) : - pimpl_(new PImpl(canvasSelector)) + pimpl_(new PImpl(canvasSelector, Version_WebGL1)) + { + } + + WebAssemblyOpenGLContext::WebAssemblyOpenGLContext(const std::string& canvasSelector, + Version version) : + pimpl_(new PImpl(canvasSelector, version)) { }
--- a/OrthancStone/Sources/Platforms/WebAssembly/WebAssemblyOpenGLContext.h Thu Mar 30 17:20:01 2023 +0200 +++ b/OrthancStone/Sources/Platforms/WebAssembly/WebAssemblyOpenGLContext.h Thu Mar 30 17:20:26 2023 +0200 @@ -51,6 +51,13 @@ { class WebAssemblyOpenGLContext : public OpenGL::IOpenGLContext { + public: + enum Version + { + Version_WebGL1, + Version_WebGL2 + }; + private: class PImpl; boost::shared_ptr<PImpl> pimpl_; @@ -58,6 +65,9 @@ public: explicit WebAssemblyOpenGLContext(const std::string& canvasSelector); + explicit WebAssemblyOpenGLContext(const std::string& canvasSelector, + Version version); + virtual bool IsContextLost() ORTHANC_OVERRIDE; virtual void MakeCurrent() ORTHANC_OVERRIDE;