# HG changeset patch # User Alain Mazy # Date 1687957815 -7200 # Node ID 39b7ccaa6dfc98db9eac9f2a00de8f5a8641632c # Parent 13d932c8a743258ba311c11cd20035210f7c2870 using multiple threads for studies/metadata diff -r 13d932c8a743 -r 39b7ccaa6dfc Plugin/WadoRs.cpp --- a/Plugin/WadoRs.cpp Wed Jun 28 14:40:23 2023 +0200 +++ b/Plugin/WadoRs.cpp Wed Jun 28 15:10:15 2023 +0200 @@ -1188,45 +1188,6 @@ } -void RetrieveStudyMetadata(OrthancPluginRestOutput* output, - const char* url, - const OrthancPluginHttpRequest* request) -{ - bool isXml; - AcceptMetadata(request, isXml); - - const OrthancPlugins::MetadataMode mode = - OrthancPlugins::Configuration::GetMetadataMode(Orthanc::ResourceType_Study); - - MainDicomTagsCache cache; - - std::string studyOrthancId, studyInstanceUid; - if (LocateStudy(output, studyOrthancId, studyInstanceUid, request)) - { - OrthancPlugins::DicomWebFormatter::HttpWriter writer(output, isXml); - - std::list series; - std::string studyDicomUid; - GetChildrenIdentifiers(series, studyDicomUid, Orthanc::ResourceType_Study, studyOrthancId); - - for (std::list::const_iterator s = series.begin(); s != series.end(); ++s) - { - std::list instances; - std::string seriesDicomUid; - GetChildrenIdentifiers(instances, seriesDicomUid, Orthanc::ResourceType_Series, *s); - - for (std::list::const_iterator i = instances.begin(); i != instances.end(); ++i) - { - WriteInstanceMetadata(writer, mode, cache, *i, studyInstanceUid, seriesDicomUid, - OrthancPlugins::Configuration::GetBasePublicUrl(request)); - } - } - - writer.Send(); - } -} - - static const char* EXIT_WORKER_MESSAGE = "exit"; class InstanceToLoad : public Orthanc::IDynamicObject @@ -1317,10 +1278,101 @@ } } +void RetrieveSeriesMetadataInternal(OrthancPluginRestOutput* output, + OrthancPlugins::DicomWebFormatter::HttpWriter& writer, + MainDicomTagsCache& cache, + const OrthancPlugins::MetadataMode& mode, + bool isXml, + const std::string& seriesOrthancId, + const std::string& studyInstanceUid, + const std::string& seriesInstanceUid, + const std::string& wadoBase) +{ + ChildrenMainDicomMaps instancesDicomMaps; + std::list instancesIds; + std::string seriesDicomUid; + + size_t threadCount = 4; + bool oneLargeQuery = false; // we keep this code here for future use once we'll have optimized Orthanc API /series/.../instances?full to minimize the SQL queries + // right now, it is faster to call /instances/..?full in each worker but, later, it should be more efficient with a large SQL query in Orthanc + + if (oneLargeQuery) + { + GetChildrenMainDicomTags(instancesDicomMaps, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); + } + else + { + GetChildrenIdentifiers(instancesIds, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); + } + + if (threadCount > 1) + { + // span a few workers to get the tags from the core and serialize them + Orthanc::SharedMessageQueue instancesQueue; + std::vector > instancesWorkers; + boost::mutex writerMutex; + std::vector > instancesWorkersData; + + for (size_t t = 0; t < threadCount; t++) + { + InstanceWorkerData* threadData = new InstanceWorkerData(boost::lexical_cast(t), &instancesQueue, wadoBase); + instancesWorkersData.push_back(boost::shared_ptr(threadData)); + instancesWorkers.push_back(boost::shared_ptr(new boost::thread(InstanceWorkerThread, threadData))); + } + + if (oneLargeQuery) + { + for (ChildrenMainDicomMaps::const_iterator i = instancesDicomMaps.begin(); i != instancesDicomMaps.end(); ++i) + { + std::string bulkRoot = (wadoBase + + "studies/" + studyInstanceUid + + "/series/" + seriesInstanceUid + + "/instances/" + i->second->GetStringValue(Orthanc::DICOM_TAG_SOP_INSTANCE_UID, "", false) + "/bulk"); + + instancesQueue.Enqueue(new InstanceToLoad(i->first, bulkRoot, writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); + } + } + else + { + for (std::list::const_iterator i = instancesIds.begin(); i != instancesIds.end(); ++i) + { + instancesQueue.Enqueue(new InstanceToLoad(*i, "", writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); + } + } + + // send a dummy "exit" message to all workers such that they stop waiting for messages on the queue + for (size_t i = 0; i < instancesWorkers.size(); i++) + { + instancesQueue.Enqueue(new InstanceToLoad(EXIT_WORKER_MESSAGE, "", writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); + } + + for (size_t i = 0; i < instancesWorkers.size(); i++) + { + if (instancesWorkers[i]->joinable()) + { + instancesWorkers[i]->join(); + } + } + + instancesWorkers.clear(); + } + else + { // old single threaded code + std::list instances; + std::string seriesDicomUid; // not used + + GetChildrenIdentifiers(instances, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); + + for (std::list::const_iterator i = instances.begin(); i != instances.end(); ++i) + { + WriteInstanceMetadata(writer, mode, cache, *i, studyInstanceUid, seriesInstanceUid, wadoBase); + } + } +} void RetrieveSeriesMetadata(OrthancPluginRestOutput* output, - const char* url, + const char* /*url*/, const OrthancPluginHttpRequest* request) { bool isXml; @@ -1330,94 +1382,50 @@ OrthancPlugins::Configuration::GetMetadataMode(Orthanc::ResourceType_Series); MainDicomTagsCache cache; + OrthancPlugins::DicomWebFormatter::HttpWriter writer(output, isXml); std::string seriesOrthancId, studyInstanceUid, seriesInstanceUid; if (LocateSeries(output, seriesOrthancId, studyInstanceUid, seriesInstanceUid, request)) { + std::string wadoBase = OrthancPlugins::Configuration::GetBasePublicUrl(request); + RetrieveSeriesMetadataInternal(output, writer, cache, mode, isXml, seriesOrthancId, studyInstanceUid, seriesInstanceUid, wadoBase); + } + + writer.Send(); +} + + +void RetrieveStudyMetadata(OrthancPluginRestOutput* output, + const char* url, + const OrthancPluginHttpRequest* request) +{ + bool isXml; + AcceptMetadata(request, isXml); + + const OrthancPlugins::MetadataMode mode = + OrthancPlugins::Configuration::GetMetadataMode(Orthanc::ResourceType_Study); + + MainDicomTagsCache cache; + + std::string studyOrthancId, studyInstanceUid; + if (LocateStudy(output, studyOrthancId, studyInstanceUid, request)) + { OrthancPlugins::DicomWebFormatter::HttpWriter writer(output, isXml); - ChildrenMainDicomMaps instancesDicomMaps; - std::list instancesIds; - std::string seriesDicomUid; - - size_t threadCount = 4; - bool oneLargeQuery = false; // we keep this code here for future use once we'll have optimized Orthanc API /series/.../instances?full to minimize the SQL queries - // right now, it is faster to call /instances/..?full in each worker but, later, it should be more efficient with a large SQL query in Orthanc - - if (oneLargeQuery) - { - GetChildrenMainDicomTags(instancesDicomMaps, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); - } - else - { - GetChildrenIdentifiers(instancesIds, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); - } - - if (threadCount > 1) - { - // span a few workers to get the tags from the core and serialize them - Orthanc::SharedMessageQueue instancesQueue; - std::vector > instancesWorkers; - boost::mutex writerMutex; - std::vector > instancesWorkersData; - std::string wadoBase = OrthancPlugins::Configuration::GetBasePublicUrl(request); - - for (size_t t; t < threadCount; t++) - { - InstanceWorkerData* threadData = new InstanceWorkerData(boost::lexical_cast(t), &instancesQueue, wadoBase); - instancesWorkersData.push_back(boost::shared_ptr(threadData)); - instancesWorkers.push_back(boost::shared_ptr(new boost::thread(InstanceWorkerThread, threadData))); - } + std::string wadoBase = OrthancPlugins::Configuration::GetBasePublicUrl(request); + + std::list series; + std::string studyDicomUid; + GetChildrenIdentifiers(series, studyDicomUid, Orthanc::ResourceType_Study, studyOrthancId); - if (oneLargeQuery) - { - for (ChildrenMainDicomMaps::const_iterator i = instancesDicomMaps.begin(); i != instancesDicomMaps.end(); ++i) - { - std::string bulkRoot = (wadoBase + - "studies/" + studyInstanceUid + - "/series/" + seriesInstanceUid + - "/instances/" + i->second->GetStringValue(Orthanc::DICOM_TAG_SOP_INSTANCE_UID, "", false) + "/bulk"); - - instancesQueue.Enqueue(new InstanceToLoad(i->first, bulkRoot, writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); - } - } - else - { - for (std::list::const_iterator i = instancesIds.begin(); i != instancesIds.end(); ++i) - { - instancesQueue.Enqueue(new InstanceToLoad(*i, "", writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); - } - } + for (std::list::const_iterator s = series.begin(); s != series.end(); ++s) + { + std::list instances; + std::string seriesDicomUid; + GetChildrenIdentifiers(instances, seriesDicomUid, Orthanc::ResourceType_Series, *s); - // send a dummy "exit" message to all workers such that they stop waiting for messages on the queue - for (size_t i = 0; i < instancesWorkers.size(); i++) - { - instancesQueue.Enqueue(new InstanceToLoad(EXIT_WORKER_MESSAGE, "", writerMutex, writer, isXml, OrthancPluginDicomWebBinaryMode_BulkDataUri, studyInstanceUid, seriesInstanceUid)); - } - - for (size_t i = 0; i < instancesWorkers.size(); i++) - { - if (instancesWorkers[i]->joinable()) - { - instancesWorkers[i]->join(); - } - } - - instancesWorkers.clear(); - } - else - { // old single threaded code - std::list instances; - std::string seriesDicomUid; // not used - - GetChildrenIdentifiers(instances, seriesDicomUid, Orthanc::ResourceType_Series, seriesOrthancId); - - for (std::list::const_iterator i = instances.begin(); i != instances.end(); ++i) - { - WriteInstanceMetadata(writer, mode, cache, *i, studyInstanceUid, seriesInstanceUid, - OrthancPlugins::Configuration::GetBasePublicUrl(request)); - } + RetrieveSeriesMetadataInternal(output, writer, cache, mode, isXml, *s, studyDicomUid, seriesDicomUid, wadoBase); } writer.Send(); diff -r 13d932c8a743 -r 39b7ccaa6dfc TODO --- a/TODO Wed Jun 28 14:40:23 2023 +0200 +++ b/TODO Wed Jun 28 15:10:15 2023 +0200 @@ -26,4 +26,9 @@ - note that all measurements have been performed on a DB with a single series ! We should repeat - that with a more realistic DB \ No newline at end of file + that with a more realistic DB + + with a 3 series study (11 + 1233 + 598 instances) + time curl http://localhost:8043/dicom-web/studies/1.2.276.0.7230010.3.1.2.1215942821.4756.1664826045.3529/metadata > /dev/null + -> 1.355ms in Full mode before using worker threads + -> 700ms in Full mode with 4 workers