Mercurial > hg > orthanc-object-storage

changeset 83:431ab61b5760

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
/move-storage when HybridMode is enabled
author Alain Mazy <am@osimis.io>
date Thu, 20 Oct 2022 15:14:39 +0200
parents f30b9acf80f0
children 8a9207933297
files Aws/CMakeLists.txt Azure/CMakeLists.txt Common/MoveStorageJob.cpp Common/MoveStorageJob.h Common/StoragePlugin.cpp Google/CMakeLists.txt NEWS README.md TODO
diffstat 9 files changed, 350 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/Aws/CMakeLists.txt	Mon Oct 17 15:17:33 2022 +0200
+++ b/Aws/CMakeLists.txt	Thu Oct 20 15:14:39 2022 +0200
@@ -131,6 +131,8 @@
   ${CMAKE_SOURCE_DIR}/../Common/EncryptionConfigurator.h
   ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.h
   ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.cpp
+  ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.h
+  ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.cpp
   ${CMAKE_SOURCE_DIR}/../Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.cpp
 
   ${AWS_SOURCES}
--- a/Azure/CMakeLists.txt	Mon Oct 17 15:17:33 2022 +0200
+++ b/Azure/CMakeLists.txt	Thu Oct 20 15:14:39 2022 +0200
@@ -95,6 +95,8 @@
     ${CMAKE_SOURCE_DIR}/../Common/EncryptionConfigurator.h
     ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.h
     ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.cpp
+    ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.h
+    ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.cpp
     ${ORTHANC_FRAMEWORK_ROOT}/../../OrthancServer/Plugins/Samples/Common/OrthancPluginCppWrapper.cpp
 
     ${ORTHANC_CORE_SOURCES}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Common/MoveStorageJob.cpp	Thu Oct 20 15:14:39 2022 +0200
@@ -0,0 +1,151 @@
+/**
+ * Cloud storage plugins for Orthanc
+ * Copyright (C) 2020-2021 Osimis S.A., Belgium
+ *
+ * This program is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ **/
+
+#include "MoveStorageJob.h"
+#include "Logging.h"
+
+
+MoveStorageJob::MoveStorageJob(const std::string& targetStorage,
+                               const std::vector<std::string>& instances,
+                               const Json::Value& resourceForJobContent,
+                               bool cryptoEnabled)
+  : OrthancPlugins::OrthancJob("MoveStorage"),
+    targetStorage_(targetStorage),
+    instances_(instances),
+    processedInstancesCount_(0),
+    resourceForJobContent_(resourceForJobContent),
+    fileSystemStorage_(NULL),
+    objectStorage_(NULL),
+    cryptoEnabled_(cryptoEnabled)
+{
+}
+
+void MoveStorageJob::SetStorages(IStorage* fileSystemStorage, IStorage* objectStorage)
+{
+  fileSystemStorage_ = fileSystemStorage;
+  objectStorage_ = objectStorage;
+}
+
+static bool MoveAttachment(const std::string& uuid, int type, IStorage* sourceStorage, IStorage* targetStorage, bool cryptoEnabled)
+{
+  std::vector<char> buffer;
+  
+  // read from source storage
+  try
+  {
+    OrthancPlugins::LogInfo("Move attachment: " + sourceStorage->GetNameForLogs() + ": reading attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type));
+    std::unique_ptr<IStorage::IReader> reader(sourceStorage->GetReaderForObject(uuid.c_str(), static_cast<OrthancPluginContentType>(type), cryptoEnabled));
+
+    size_t fileSize = reader->GetSize();
+    buffer.resize(fileSize);
+
+    reader->ReadWhole(buffer.data(), fileSize);
+  }
+  catch (StoragePluginException& ex)
+  {
+    OrthancPlugins::LogInfo("Move attachment: " + sourceStorage->GetNameForLogs() + ": error while reading attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type) + ", this likely means that the file is already on the right storage");
+    return true;
+  }
+
+  // write to target storage
+  if (buffer.size() > 0)
+  {
+    try
+    {
+      std::unique_ptr<IStorage::IWriter> writer(targetStorage->GetWriterForObject(uuid.c_str(), static_cast<OrthancPluginContentType>(type), cryptoEnabled));
+
+      writer->Write(buffer.data(), buffer.size());
+    }
+    catch (StoragePluginException& ex)
+    {
+      OrthancPlugins::LogError("Move attachment: " + targetStorage->GetNameForLogs() + ": error while writing attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type) + ": " + ex.what());
+      return false;
+    }
+  }
+
+  // everything went well so fare, we can delete from source storage
+  if (buffer.size() > 0)
+  {
+    try
+    {
+      sourceStorage->DeleteObject(uuid.c_str(), static_cast<OrthancPluginContentType>(type), cryptoEnabled);
+    }
+    catch (StoragePluginException& ex)
+    {
+      OrthancPlugins::LogError("Move attachment: " + sourceStorage->GetNameForLogs() + ": error while deleting attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type) + ": " + ex.what());
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool MoveInstance(const std::string& instanceId, IStorage* sourceStorage, IStorage* targetStorage, bool cryptoEnabled)
+{
+  LOG(INFO) << "Moving instance from " << sourceStorage->GetNameForLogs() << " to " << targetStorage->GetNameForLogs();
+
+  Json::Value attachmentsList;
+  OrthancPlugins::RestApiGet(attachmentsList, std::string("/instances/") + instanceId + "/attachments?full", false);
+
+  Json::Value::Members attachmentsMembers = attachmentsList.getMemberNames();
+  bool success = true;
+
+  for (size_t i = 0; i < attachmentsMembers.size(); i++)
+  {
+    int attachmentId = attachmentsList[attachmentsMembers[i]].asInt();
+
+    Json::Value attachmentInfo;
+    OrthancPlugins::RestApiGet(attachmentInfo, std::string("/instances/") + instanceId + "/attachments/" + boost::lexical_cast<std::string>(attachmentId) + "/info", false);
+
+    std::string attachmentUuid = attachmentInfo["Uuid"].asString();
+
+    // now we have the uuid and type.  We actually don't know where the file is but we'll try to move it anyway to the requested target
+    success &= MoveAttachment(attachmentUuid, attachmentId, sourceStorage, targetStorage, cryptoEnabled);
+  }
+
+  return success;
+}
+
+OrthancPluginJobStepStatus MoveStorageJob::Step()
+{
+  if (processedInstancesCount_ < instances_.size())
+  {
+    IStorage* sourceStorage = (targetStorage_ == "file-system" ? objectStorage_ : fileSystemStorage_);
+    IStorage* targetStorage = (targetStorage_ == "file-system" ? fileSystemStorage_ : objectStorage_);
+
+    if (MoveInstance(instances_[processedInstancesCount_], sourceStorage, targetStorage, cryptoEnabled_))
+    {
+      processedInstancesCount_++;
+      return OrthancPluginJobStepStatus_Continue;
+    }
+    else
+    {
+      return OrthancPluginJobStepStatus_Failure;
+    }
+  }
+
+  return OrthancPluginJobStepStatus_Success;
+}
+
+void MoveStorageJob::Stop(OrthancPluginJobStopReason reason)
+{
+}
+    
+void MoveStorageJob::Reset()
+{
+  processedInstancesCount_ = 0;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Common/MoveStorageJob.h	Thu Oct 20 15:14:39 2022 +0200
@@ -0,0 +1,54 @@
+/**
+ * Cloud storage plugins for Orthanc
+ * Copyright (C) 2020-2021 Osimis S.A., Belgium
+ *
+ * This program is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ **/
+
+
+#pragma once
+
+
+#include <orthanc/OrthancCPlugin.h>
+#include <OrthancPluginCppWrapper.h>
+#include <json/json.h>
+#include "IStorage.h"
+
+#include <vector>
+
+class MoveStorageJob : public OrthancPlugins::OrthancJob
+{
+  std::string targetStorage_;
+  std::vector<std::string> instances_;
+  size_t processedInstancesCount_;
+  Json::Value resourceForJobContent_;
+  IStorage* fileSystemStorage_;
+  IStorage* objectStorage_;
+  bool cryptoEnabled_;
+
+public:
+  MoveStorageJob(const std::string& targetStorage,
+                  const std::vector<std::string>& instances,
+                  const Json::Value& resourceForJobContent,
+                  bool cryptoEnabled);
+
+  virtual OrthancPluginJobStepStatus Step();
+
+  virtual void Stop(OrthancPluginJobStopReason reason);
+  
+  virtual void Reset();
+
+  void SetStorages(IStorage* fileSystemStorage, IStorage* objectStorage);
+
+};
\ No newline at end of file
--- a/Common/StoragePlugin.cpp	Mon Oct 17 15:17:33 2022 +0200
+++ b/Common/StoragePlugin.cpp	Thu Oct 20 15:14:39 2022 +0200
@@ -44,6 +44,7 @@
 
 #include <Logging.h>
 #include <SystemToolbox.h>
+#include "MoveStorageJob.h"
 
 static std::unique_ptr<IStorage> primaryStorage;
 static std::unique_ptr<IStorage> secondaryStorage;
@@ -315,6 +316,131 @@
 }
 
 
+static void AddResourceForJobContent(Json::Value& resourcesForJobContent /* out */, Orthanc::ResourceType resourceType, const std::string& resourceId)
+{
+  const char* resourceGroup = Orthanc::GetResourceTypeText(resourceType, true, true);
+
+  if (!resourcesForJobContent.isMember(resourceGroup))
+  {
+    resourcesForJobContent[resourceGroup] = Json::arrayValue;
+  }
+  
+  resourcesForJobContent[resourceGroup].append(resourceId);
+}
+
+void MoveStorage(OrthancPluginRestOutput* output,
+                 const char* /*url*/,
+                 const OrthancPluginHttpRequest* request)
+{
+  OrthancPluginContext* context = OrthancPlugins::GetGlobalContext();
+
+  if (request->method != OrthancPluginHttpMethod_Post)
+  {
+    OrthancPluginSendMethodNotAllowed(context, output, "POST");
+    return;
+  }
+
+  Json::Value requestPayload;
+
+  if (!OrthancPlugins::ReadJson(requestPayload, request->body, request->bodySize))
+  {
+    throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat, "A JSON payload was expected");
+  }
+
+  std::vector<std::string> instances;
+  Json::Value resourcesForJobContent;
+
+  static const char* RESOURCES = "Resources";
+  static const char* TARGET_STORAGE = "TargetStorage";
+
+  if (requestPayload.type() != Json::objectValue ||
+      !requestPayload.isMember(RESOURCES) ||
+      requestPayload[RESOURCES].type() != Json::arrayValue)
+  {
+    throw Orthanc::OrthancException(
+      Orthanc::ErrorCode_BadFileFormat,
+      "A request to the move-storage endpoint must provide a JSON object "
+      "with the field \"" + std::string(RESOURCES) + 
+      "\" containing an array of resources to be sent");
+  }
+
+  if (!requestPayload.isMember(TARGET_STORAGE)
+      || requestPayload[TARGET_STORAGE].type() != Json::stringValue
+      || (requestPayload[TARGET_STORAGE].asString() != "file-system" && requestPayload[TARGET_STORAGE].asString() != "object-storage"))
+  {
+    throw Orthanc::OrthancException(
+      Orthanc::ErrorCode_BadFileFormat,
+      "A request to the move-storage endpoint must provide a JSON object "
+      "with the field \"" + std::string(TARGET_STORAGE) + 
+      "\" set to \"file-system\" or \"object-storage\"");
+  }
+
+  const std::string& targetStorage = requestPayload[TARGET_STORAGE].asString();
+  const Json::Value& resources = requestPayload[RESOURCES];
+
+  // Extract information about all the child instances
+  for (Json::Value::ArrayIndex i = 0; i < resources.size(); i++)
+  {
+    if (resources[i].type() != Json::stringValue)
+    {
+      throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat);
+    }
+
+    std::string resource = resources[i].asString();
+    if (resource.empty())
+    {
+      throw Orthanc::OrthancException(Orthanc::ErrorCode_UnknownResource);
+    }
+
+    // Test whether this resource is an instance
+    Json::Value tmpResource;
+    Json::Value tmpInstances;
+    if (OrthancPlugins::RestApiGet(tmpResource, "/instances/" + resource, false))
+    {
+      instances.push_back(resource);
+      AddResourceForJobContent(resourcesForJobContent, Orthanc::ResourceType_Instance, resource);
+    }
+    // This was not an instance, successively try with series/studies/patients
+    else if ((OrthancPlugins::RestApiGet(tmpResource, "/series/" + resource, false) &&
+              OrthancPlugins::RestApiGet(tmpInstances, "/series/" + resource + "/instances", false)) ||
+             (OrthancPlugins::RestApiGet(tmpResource, "/studies/" + resource, false) &&
+              OrthancPlugins::RestApiGet(tmpInstances, "/studies/" + resource + "/instances", false)) ||
+             (OrthancPlugins::RestApiGet(tmpResource, "/patients/" + resource, false) &&
+              OrthancPlugins::RestApiGet(tmpInstances, "/patients/" + resource + "/instances", false)))
+    {
+      if (tmpInstances.type() != Json::arrayValue)
+      {
+        throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError);
+      }
+
+      for (Json::Value::ArrayIndex j = 0; j < tmpInstances.size(); j++)
+      {
+        instances.push_back(tmpInstances[j]["ID"].asString());
+        AddResourceForJobContent(resourcesForJobContent, Orthanc::StringToResourceType(tmpResource["Type"].asString().c_str()), resource);
+      }
+    }
+    else
+    {
+      throw Orthanc::OrthancException(Orthanc::ErrorCode_UnknownResource);
+    }   
+  }
+
+  OrthancPlugins::LogInfo("Moving " + boost::lexical_cast<std::string>(instances.size()) + " instances to " + targetStorage);
+
+  std::unique_ptr<MoveStorageJob> job(new MoveStorageJob(targetStorage, instances, resourcesForJobContent, cryptoEnabled));
+
+  if (hybridMode == HybridMode_WriteToFileSystem)
+  {
+    job->SetStorages(primaryStorage.get(), secondaryStorage.get());
+  }
+  else
+  {
+    job->SetStorages(secondaryStorage.get(), primaryStorage.get());
+  }
+
+  OrthancPlugins::OrthancJob::SubmitFromRestApiPost(output, requestPayload, job.release());
+}
+
 extern "C"
 {
   ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* context)
@@ -459,6 +585,11 @@
         }
 
 
+        if (IsHybridModeEnabled())
+        {
+          OrthancPlugins::RegisterRestCallback<MoveStorage>("/move-storage", true);
+        }
+
       }
 
       if (cryptoEnabled)
--- a/Google/CMakeLists.txt	Mon Oct 17 15:17:33 2022 +0200
+++ b/Google/CMakeLists.txt	Thu Oct 20 15:14:39 2022 +0200
@@ -69,6 +69,8 @@
     ${CMAKE_SOURCE_DIR}/../Common/EncryptionConfigurator.h
     ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.h
     ${CMAKE_SOURCE_DIR}/../Common/FileSystemStorage.cpp
+    ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.h
+    ${CMAKE_SOURCE_DIR}/../Common/MoveStorageJob.cpp
     ${ORTHANC_FRAMEWORK_ROOT}/../../OrthancServer/Plugins/Samples/Common/OrthancPluginCppWrapper.cpp
 
     ${ORTHANC_CORE_SOURCES}
--- a/NEWS	Mon Oct 17 15:17:33 2022 +0200
+++ b/NEWS	Thu Oct 20 15:14:39 2022 +0200
@@ -2,6 +2,8 @@
   from both storages.  Allowed values: "WriteToFileSystem", "WriteToObjectStorage", "Disabled".
   If disabled (default), the plugin writes all data to the object storage and tries to read them only from the
   object storage.
+* new /move-storage route when "HybridMode" is enabled.  This allows moving a study from one-storage to the other
+  to, e.g, have recent studies on the file-system and older studies on a cloud object storage.
 
 2022-08-30 - v 2.0.0
 ====================
--- a/README.md	Mon Oct 17 15:17:33 2022 +0200
+++ b/README.md	Thu Oct 20 15:14:39 2022 +0200
@@ -176,3 +176,6 @@
   - there should be only one file in the disk storage
   - there should be only one file in the S3 bucket
 
+test moving a study to file-system storage
+curl http://localhost:8043/move-storage -d '{"Resources": ["737c0c8d-ea890b4d-e36a43bb-fb8c8d41-aa0ed0a8"], "TargetStorage" : "file-system"}'
+curl http://localhost:8043/move-storage -d '{"Resources": ["737c0c8d-ea890b4d-e36a43bb-fb8c8d41-aa0ed0a8"], "TargetStorage" : "object-storage"}'
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/TODO	Thu Oct 20 15:14:39 2022 +0200
@@ -0,0 +1,3 @@
+- test serialize/unserialize MoveStorageJob
+
+- /move-storage shall also move the attachments at Study/Series/Patient level, not only the Instances
\ No newline at end of file