Mercurial > hg > orthanc

changeset 6597:81746b9484c4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
added Toolbox::IsValidUtf8()
author Sebastien Jodogne <s.jodogne@gmail.com>
date Fri, 20 Feb 2026 16:40:13 +0100
parents de1b2b661ec3
children 24f00121619f
files OrthancFramework/Sources/Toolbox.cpp OrthancFramework/Sources/Toolbox.h OrthancFramework/UnitTestsSources/FrameworkTests.cpp OrthancFramework/UnitTestsSources/FromDcmtkTests.cpp OrthancFramework/UnitTestsSources/ToolboxTests.cpp
diffstat 5 files changed, 176 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/OrthancFramework/Sources/Toolbox.cpp	Fri Feb 20 16:03:16 2026 +0100
+++ b/OrthancFramework/Sources/Toolbox.cpp	Fri Feb 20 16:40:13 2026 +0100
@@ -2825,6 +2825,139 @@
     }
   }
 
+
+  bool Toolbox::IsValidUtf8(const std::string& s)
+  {
+    if (s.empty())
+    {
+      return true;
+    }
+
+    const unsigned char* bytes = reinterpret_cast<const unsigned char*>(s.data());
+    size_t len = s.size();
+
+    size_t i = 0;
+    while (i < len)
+    {
+      unsigned char b1 = bytes[i];
+
+      if (b1 <= 0x7fu)
+      {
+        // 1-byte (ASCII): 0xxxxxxx
+        i++;
+      }
+      else if ((b1 >> 5) == 0x06u)
+      {
+        // 2-byte sequence: 110xxxxx 10xxxxxx
+        if (i + 1 >= len)
+        {
+          return false;
+        }
+
+        unsigned char b2 = bytes[i + 1];
+        if ((b2 >> 6) != 0x02u)
+        {
+          return false;
+        }
+
+        uint32_t codepoint =
+          ((b1 & 0x1fu) << 6) |
+          (b2 & 0x3fu);
+
+        // Overlong encoding check
+        if (codepoint < 0x80u)
+        {
+          return false;
+        }
+
+        i += 2;
+      }
+      else if ((b1 >> 4) == 0x0eu)
+      {
+        // 3-byte sequence: 1110xxxx 10xxxxxx 10xxxxxx
+        if (i + 2 >= len)
+        {
+          return false;
+        }
+
+        unsigned char b2 = bytes[i + 1];
+        unsigned char b3 = bytes[i + 2];
+
+        if ((b2 >> 6) != 0x02u ||
+            (b3 >> 6) != 0x02u)
+        {
+          return false;
+        }
+
+        uint32_t codepoint =
+          ((b1 & 0x0fu) << 12) |
+          ((b2 & 0x3fu) << 6) |
+          (b3 & 0x3fu);
+
+        // Overlong encoding check
+        if (codepoint < 0x0800u)
+        {
+          return false;
+        }
+
+        // Surrogate range check (U+D800 to U+DFFF)
+        if (codepoint >= 0xd800u &&
+            codepoint <= 0xdfffu)
+        {
+          return false;
+        }
+
+        i += 3;
+      }
+      else if ((b1 >> 3) == 0x1eu)
+      {
+        // 4-byte sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+        if (i + 3 >= len)
+        {
+          return false;
+        }
+
+        unsigned char b2 = bytes[i + 1];
+        unsigned char b3 = bytes[i + 2];
+        unsigned char b4 = bytes[i + 3];
+
+        if ((b2 >> 6) != 0x02u ||
+            (b3 >> 6) != 0x02u ||
+            (b4 >> 6) != 0x02u)
+        {
+          return false;
+        }
+
+        uint32_t codepoint =
+          ((b1 & 0x07u) << 18) |
+          ((b2 & 0x3fu) << 12) |
+          ((b3 & 0x3fu) << 6) |
+          (b4 & 0x3fu);
+
+        // Overlong encoding check
+        if (codepoint < 0x00010000u)
+        {
+          return false;
+        }
+
+        // Maximum valid Unicode code point
+        if (codepoint > 0x0010ffffu)
+        {
+          return false;
+        }
+
+        i += 4;
+      }
+      else
+      {
+        return false; // Invalid leading byte
+      }
+    }
+
+    return true;
+  }
+
+
   Toolbox::ElapsedTimer::ElapsedTimer()
   {
     Restart();
--- a/OrthancFramework/Sources/Toolbox.h	Fri Feb 20 16:03:16 2026 +0100
+++ b/OrthancFramework/Sources/Toolbox.h	Fri Feb 20 16:40:13 2026 +0100
@@ -405,6 +405,8 @@
 
     static void RemoveSurroundingQuotes(std::string& value);
 
+    static bool IsValidUtf8(const std::string& s);
+
     class ORTHANC_PUBLIC ElapsedTimer : public boost::noncopyable
     {
     private:
--- a/OrthancFramework/UnitTestsSources/FrameworkTests.cpp	Fri Feb 20 16:03:16 2026 +0100
+++ b/OrthancFramework/UnitTestsSources/FrameworkTests.cpp	Fri Feb 20 16:40:13 2026 +0100
@@ -504,6 +504,7 @@
   
   std::string s(reinterpret_cast<const char*>(&data[0]), 10);
   ASSERT_EQ("&abc", Toolbox::ConvertToAscii(s));
+  ASSERT_FALSE(Toolbox::IsValidUtf8(s));
 
   // Open in Emacs, then save with UTF-8 encoding, then "hexdump -C"
   std::string utf8 = Toolbox::ConvertToUtf8(s, Encoding_Latin1, false, false);
@@ -523,6 +524,7 @@
   ASSERT_EQ(0x62, static_cast<unsigned char>(utf8[12]));
   ASSERT_EQ(0x63, static_cast<unsigned char>(utf8[13]));
   ASSERT_EQ(0x00, static_cast<unsigned char>(utf8[14]));  // Null-terminated string
+  ASSERT_TRUE(Toolbox::IsValidUtf8(utf8));
 }
 
 
@@ -532,9 +534,12 @@
   const unsigned char latin1[] = { 0x63, 0x72, 0xe2, 0x6e, 0x65 };
 
   std::string s(reinterpret_cast<const char*>(&latin1[0]), sizeof(latin1) / sizeof(char));
+  ASSERT_FALSE(Toolbox::IsValidUtf8(s));
 
-  ASSERT_EQ(s, Toolbox::ConvertFromUtf8(Toolbox::ConvertToUtf8(s, Encoding_Latin1, false, false), Encoding_Latin1));
+  std::string utf8 = Toolbox::ConvertToUtf8(s, Encoding_Latin1, false, false);
+  ASSERT_EQ(s, Toolbox::ConvertFromUtf8(utf8, Encoding_Latin1));
   ASSERT_EQ("cre", Toolbox::ConvertToUtf8(s, Encoding_Utf8, false, false));
+  ASSERT_TRUE(Toolbox::IsValidUtf8(utf8));
 }
 
 
--- a/OrthancFramework/UnitTestsSources/FromDcmtkTests.cpp	Fri Feb 20 16:03:16 2026 +0100
+++ b/OrthancFramework/UnitTestsSources/FromDcmtkTests.cpp	Fri Feb 20 16:40:13 2026 +0100
@@ -265,6 +265,7 @@
     std::string source(testEncodingsEncoded[i]);
     std::string expected(testEncodingsExpected[i]);
     std::string s = Toolbox::ConvertToUtf8(source, testEncodings[i], false, false);
+    ASSERT_TRUE(Toolbox::IsValidUtf8(s));
     //std::cout << EnumerationToString(testEncodings[i]) << std::endl;
     EXPECT_EQ(expected, s);
   }
@@ -335,6 +336,7 @@
       f.SetEncoding(testEncodings[i]);
 
       std::string s = Toolbox::ConvertToUtf8(testEncodingsEncoded[i], testEncodings[i], false, false);
+      ASSERT_TRUE(Toolbox::IsValidUtf8(s));
       f.Insert(DICOM_TAG_PATIENT_NAME, s, false, "");
       f.SaveToMemoryBuffer(dicom);
     }
@@ -1173,6 +1175,7 @@
         std::string encoded = Toolbox::ConvertFromUtf8(testEncodingsExpected[i], testEncodings[i]);
         ASSERT_STREQ(testEncodingsEncoded[i], encoded.c_str());
         std::string decoded = Toolbox::ConvertToUtf8(encoded, testEncodings[i], false, false);
+        ASSERT_TRUE(Toolbox::IsValidUtf8(decoded));
         ASSERT_STREQ(testEncodingsExpected[i], decoded.c_str());
 
         if (testEncodings[i] != Encoding_Chinese)
@@ -1182,6 +1185,7 @@
 
           const std::string tmp = Toolbox::ConvertToUtf8(
             Toolbox::ConvertFromUtf8(utf8, testEncodings[i]), testEncodings[i], false, false);
+          ASSERT_TRUE(Toolbox::IsValidUtf8(tmp));
           ASSERT_STREQ(testEncodingsExpected[i], tmp.c_str());
         }
       }
@@ -1612,7 +1616,8 @@
     0xea, 0xb8, 0xb8, 0xeb, 0x8f, 0x99
   };
 
-  std::string utf8(reinterpret_cast<const char*>(utf8raw), sizeof(utf8raw));
+  const std::string utf8(reinterpret_cast<const char*>(utf8raw), sizeof(utf8raw));
+  ASSERT_TRUE(Toolbox::IsValidUtf8(utf8));
 
   ParsedDicomFile dicom(false);
   dicom.ReplacePlainString(DICOM_TAG_SPECIFIC_CHARACTER_SET, "\\ISO 2022 IR 149");
@@ -1709,7 +1714,8 @@
     0x8d, 0xe3, 0x81, 0x86
   };
 
-  std::string utf8(reinterpret_cast<const char*>(utf8raw), sizeof(utf8raw));
+  const std::string utf8(reinterpret_cast<const char*>(utf8raw), sizeof(utf8raw));
+  ASSERT_TRUE(Toolbox::IsValidUtf8(utf8));
 
   ParsedDicomFile dicom(false);
   dicom.ReplacePlainString(DICOM_TAG_SPECIFIC_CHARACTER_SET, "\\ISO 2022 IR 87");
@@ -1914,7 +1920,10 @@
 
   std::string value;
   ASSERT_TRUE(dicom.GetTagValue(value, DICOM_TAG_PATIENT_NAME));
-  ASSERT_EQ(value, std::string(reinterpret_cast<const char*>(utf8), sizeof(utf8)));
+
+  const std::string tmp(reinterpret_cast<const char*>(utf8), sizeof(utf8));
+  ASSERT_EQ(value, tmp);
+  ASSERT_TRUE(Toolbox::IsValidUtf8(tmp));
 }
 
 
--- a/OrthancFramework/UnitTestsSources/ToolboxTests.cpp	Fri Feb 20 16:03:16 2026 +0100
+++ b/OrthancFramework/UnitTestsSources/ToolboxTests.cpp	Fri Feb 20 16:40:13 2026 +0100
@@ -436,3 +436,26 @@
   s = Orthanc::Toolbox::ConvertDicomStringToUtf8("ORIGINAL\\PRIMARY", Encoding_Latin1, false, ValueRepresentation_ShortText);
   ASSERT_EQ("ORIGINAL\\PRIMARY", s);
 }
+
+TEST(Toolbox, IsValidUtf8)
+{
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("Hello, world!"));          // ASCII
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8(""));                       // Empty string is valid
+
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("\xC2\xA2"));               // U+00A2 2-byte
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("\xE2\x82\xAC"));           // U+20AC 3-byte
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("\xF0\x9F\x8C\x8D"));       // U+1F30D 4-byte
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("Hello \xE2\x82\xAC \xF0\x9F\x8C\x8D"));   // Mixed
+  ASSERT_TRUE(Orthanc::Toolbox::IsValidUtf8("\xF4\x8F\xBF\xBF"));       // U+10FFFF (max valid)
+
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xC2\x20"));              // Invalid continuation
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xE2\x82"));              // Missing continuation
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\x80"));                  // Lone continuation byte
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xC0\xAF"));              // Overlong 2-byte
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xE0\x80\xAF"));          // Overlong 3-byte
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xED\xA0\x80"));          // Surrogate (U+D800)
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xF4\x90\x80\x80"));      // > U+10FFFF
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xF0\x9F\x8C"));          // Truncated 4-byte
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xFF"));                  // Invalid leading byte
+  ASSERT_FALSE(Orthanc::Toolbox::IsValidUtf8("\xF0\x28\x8C\x28"));
+}