--- a/.hgignore	Wed Apr 27 08:46:26 2022 +0200
+++ b/.hgignore	Sat Apr 30 19:38:34 2022 +0200
@@ -8,3 +8,5 @@
 PerfsDb/Results/
 .vscode/
 *~
+NewTests/storages/**
+NewTests/configurations/*.json

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/Housekeeper/test_housekeeper.py	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,124 @@
+import unittest
+import time
+from helpers import OrthancTestCase, Helpers
+
+from orthanc_api_client import OrthancApiClient, generate_test_dicom_file
+
+import pathlib
+here = pathlib.Path(__file__).parent.resolve()
+
+
+class TestHousekeeper(OrthancTestCase):
+
+    @classmethod
+    def prepare(cls):
+        print('-------------- preparing TestHousekeeper tests')
+
+        cls.clear_storage(storage_name="housekeeper")
+
+        cls.launch_orthanc_to_prepare_db(
+            config_name="housekeeper_preparation",
+            storage_name="housekeeper",
+            config={
+                "StorageCompression": False,
+                "Housekeeper": {
+                    "Enable": False
+                }
+            },
+            plugins=Helpers.plugins
+        )
+
+        # upload a study and keep track of data before housekeeper runs
+        cls.o.upload_folder(here / "../../Database/Knix/Loc")
+
+        cls.instance_before, cls.series_before, cls.study_before, cls.patient_before = cls.get_infos()
+
+        cls.kill_orthanc()
+
+        # generate config for orthanc-under-tests (change StorageCompression and add ExtraMainDicomTags)
+        config_path = cls.generate_configuration(
+            config_name="housekeeper_under_test",
+            storage_name="housekeeper",
+            config={
+                "StorageCompression": True,
+                "ExtraMainDicomTags": {
+                    "Patient" : ["PatientWeight", "PatientAge"],
+                    "Study": ["NameOfPhysiciansReadingStudy"],
+                    "Series": ["ScanOptions"],
+                    "Instance": ["Rows", "Columns"]
+                },
+                "Housekeeper": {
+                    "Enable": True
+                }
+            },
+            plugins=Helpers.plugins
+        )
+
+        print('-------------- prepared TestHousekeeper tests')
+        if Helpers.break_after_preparation:
+            print(f"++++ It is now time to start your Orthanc under tests with configuration file '{config_path}' +++++")
+            input("Press Enter to continue")
+        else:
+            print('-------------- launching TestHousekeeper tests')
+            cls.launch_orthanc(
+                exe_path=Helpers.orthanc_under_tests_exe,
+                config_path=config_path
+            )
+
+        print('-------------- waiting for orthanc-under-tests to be available')
+        cls.o.wait_started()
+        
+        completed = False
+        while not completed:
+            print('-------------- waiting for housekeeper to finish processing')
+            time.sleep(1)
+            housekeeper_status = cls.o.get_json("/housekeeper/status")
+            completed = (housekeeper_status["LastProcessedConfiguration"]["StorageCompressionEnabled"] == True) \
+                        and (housekeeper_status["LastChangeToProcess"] == housekeeper_status["LastProcessedChange"])
+
+
+    @classmethod
+    def get_infos(cls):
+        instance_id = cls.o.lookup(
+            needle="1.2.840.113619.2.176.2025.1499492.7040.1171286241.704",
+            filter="Instance"
+        )[0]
+
+        instance_info = cls.o.get_json(relative_url=f"/instances/{instance_id}")
+        
+        series_id = instance_info["ParentSeries"]
+        series_info = cls.o.get_json(relative_url=f"/series/{series_id}")
+        
+        study_id = series_info["ParentStudy"]
+        study_info = cls.o.get_json(relative_url=f"/studies/{study_id}")
+
+        patient_id = study_info["ParentPatient"]
+        patient_info = cls.o.get_json(relative_url=f"/patients/{patient_id}")
+
+        return instance_info, series_info, study_info, patient_info
+
+
+
+    def test_before_after_reconstruction(self):
+
+        # make sure it has run once !
+        housekeeper_status = self.o.get_json("/housekeeper/status")
+        self.assertIsNotNone(housekeeper_status["LastTimeStarted"])
+
+        instance_after, series_after, study_after, patient_after = self.get_infos()
+
+        # extra tags were not in DB before reconstruction
+        self.assertNotIn("Rows", self.instance_before["MainDicomTags"])
+        self.assertNotIn("ScanOptions", self.series_before["MainDicomTags"])
+        self.assertNotIn("NameOfPhysiciansReadingStudy", self.study_before["MainDicomTags"])
+        self.assertNotIn("PatientWeight", self.patient_before["MainDicomTags"])
+
+        # extra tags are in  DB after reconstruction
+        self.assertIn("Rows", instance_after["MainDicomTags"])
+        self.assertIn("ScanOptions", series_after["MainDicomTags"])
+        self.assertIn("NameOfPhysiciansReadingStudy", study_after["MainDicomTags"])
+        self.assertIn("PatientWeight", patient_after["MainDicomTags"])
+
+        # storage has been compressed during reconstruction
+        self.assertTrue(self.instance_before["FileSize"] > instance_after["FileSize"]) 
+        self.assertNotEqual(self.instance_before["FileUuid"], instance_after["FileUuid"]) # files ID have changed

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/README	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,61 @@
+This is a new set of tests directly written in python3.  They should be able to handle 
+more complex scenarios like upgrades or change of configurations.
+
+Prerequisites:
+=============
+
+These tests use python3 and require some modules define in requirements.txt.  Therefore, you need
+to first execute 
+
+pip3 install -r requirements.txt 
+
+Introduction:
+============
+
+You may use these tests to debug Orthanc on your machine.  In this case, there is 
+usually a `preparation` phase and  `execution` phase.  You are usually able to 
+interrupt the tests between these 2 phases such that you can start your debugger.
+Use the `--break_after_preparation` option to do so.
+As well, you may skip the preperation phase thanks to the `--skip_preparation` option.
+
+The orthanc that is being tested is called the `orthanc-under-tests`.
+
+Examples:
+========
+
+All-tests:
+---------
+
+python3 NewTests/main.py --pattern=* \
+                         --orthanc_under_tests_exe=/home/alain/o/build/orthanc/Orthanc \
+                         --orthanc_under_tests_http_port=8043 \
+                         --plugin=/home/alain/o/build/orthanc/libHousekeeper.so \
+                         --plugin=/home/alain/o/build/orthanc-gdcm/libOrthancGdcm.so
+
+
+Housekeeper:
+-----------
+
+Run the Housekeeper tests with your locally build version and break between preparation
+and execution to allow you to start your debugger.
+
+python3 NewTests/main.py --pattern=Housekeeper.test_housekeeper.TestHousekeeper.test_before_after_reconstruction \
+                         --orthanc_under_tests_exe=/home/alain/o/build/orthanc/Orthanc \
+                         --orthanc_under_tests_http_port=8043 \
+                         --plugin=/home/alain/o/build/orthanc/libHousekeeper.so \
+                         --break_after_preparation
+
+The test script will:
+- generate 2 configuration file in the `configurations` folder,
+- start your local Orthanc version to prepare the db with one of the configuration file, 
+- drive this Orthanc to prepare the DB
+- interrupt and instruct you how to start your own version, pointing to the configuration file to use
+- execute tests
+
+
+
+TODO: implement and document usage with Docker !!!!
+
+
+
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/configurations/.keep	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,1 @@
+just a file to force directory creation !

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/helpers.py	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,146 @@
+import unittest
+from orthanc_api_client import OrthancApiClient
+import subprocess
+import json
+import time
+import typing
+import shutil
+from threading import Thread
+
+
+import pathlib
+here = pathlib.Path(__file__).parent.resolve()
+
+
+default_base_config = {
+    "AuthenticationEnabled": False,
+    "RemoteAccessAllowed": True
+}
+
+class Helpers:
+
+    orthanc_under_tests_hostname: str = 'localhost'
+    orthanc_under_tests_http_port: int = 8042
+    orthanc_under_tests_exe: str = None
+    orthanc_previous_version_exe: str = None
+    orthanc_under_tests_docker_image: str = None
+    skip_preparation: bool = False
+    break_after_preparation: bool = False
+    plugins: typing.List[str] = []
+
+    @classmethod
+    def get_orthanc_url(cls):
+        return f"http://{cls.orthanc_under_tests_hostname}:{cls.orthanc_under_tests_http_port}"
+
+class OrthancTestCase(unittest.TestCase):
+
+    o: OrthancApiClient = None  # the orthanc under tests api client
+    _orthanc_process = None
+    _orthanc_is_running = False
+    _orthanc_logger_thread = None
+
+    @classmethod
+    def setUpClass(cls):
+
+        cls.o = OrthancApiClient(Helpers.get_orthanc_url())
+        cls._prepare()
+
+    @classmethod
+    def tearDownClass(cls):
+        if not Helpers.break_after_preparation:
+            cls.kill_orthanc()
+
+    @classmethod
+    def prepare(cls):
+        pass # to override
+
+    @classmethod
+    def _prepare(cls):
+        if not Helpers.skip_preparation:
+            cls.prepare()
+
+    @classmethod
+    def get_storage_path(cls, storage_name: str):
+        return str(here / "storages" / f"{storage_name}")
+
+    @classmethod
+    def generate_configuration(cls, config_name: str, config: object, storage_name: str, plugins = []):
+        
+        # add plugins and default storge directory
+        config["Plugins"] = plugins
+
+        if not "StorageDirectory" in config:
+            config["StorageDirectory"] = cls.get_storage_path(storage_name=storage_name)
+
+        if not "Name" in config:
+            config["Name"] = config_name
+
+        if not "HttpPort" in config:
+            config["HttpPort"] = Helpers.orthanc_under_tests_http_port
+
+        # copy the values from the base config
+        for k, v in default_base_config.items():
+            if not k in config:
+                config[k] = v
+
+        # save to disk
+        path = str(here / "configurations" / f"{config_name}.json")
+        with open(path, "w") as f:
+            json.dump(config, f, indent=4)
+
+        return path
+
+    @classmethod
+    def clear_storage(cls, storage_name: str):
+        storage_path = cls.get_storage_path(storage_name=storage_name)
+        shutil.rmtree(storage_path)
+
+
+    @classmethod
+    def launch_orthanc_to_prepare_db(cls, config_name: str, config: object, storage_name: str, plugins = []):
+        # generate the configuration file
+        config_path = cls.generate_configuration(
+            config_name=config_name,
+            storage_name=storage_name,
+            config=config,
+            plugins=plugins
+            )
+
+        # run orthanc
+        if Helpers.orthanc_previous_version_exe:
+            cls.launch_orthanc(
+                exe_path=Helpers.orthanc_previous_version_exe,
+                config_path=config_path
+            )
+        else:
+            raise RuntimeError("No orthanc_previous_version_exe defined, can not launch Orthanc")
+
+    @classmethod
+    def launch_orthanc(cls, exe_path: str, config_path: str):
+            cls._orthanc_process = subprocess.Popen(
+                [exe_path, "--verbose", config_path],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+
+            cls.o.wait_started(10)
+            if not cls.o.is_alive():
+                output = cls.get_orthanc_process_output()
+                print("Orthanc output\n" + output)
+
+                raise RuntimeError(f"Orthanc failed to start '{exe_path}', conf = '{config_path}'.  Check output above")
+
+    @classmethod
+    def kill_orthanc(cls):
+        cls._orthanc_process.kill()
+        output = cls.get_orthanc_process_output()
+        print("Orthanc output\n" + output)
+        cls._orthanc_process = None
+
+    @classmethod
+    def get_orthanc_process_output(cls):
+        outputs = cls._orthanc_process.communicate()
+        output = ""
+        for o in outputs:
+            output += o.decode('utf-8')
+        return output

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/main.py	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,54 @@
+import argparse
+import unittest
+import os
+import sys
+import argparse
+from helpers import Helpers
+import pathlib
+# python3 main.py --orthanc_under_tests_exe=/home/alain/o/build/orthanc/Orthanc --pattern=Housekeeper.test_housekeeper.TestHousekeeper.*test_is* --plugin=/home/alain/o/build/orthanc/libHousekeeper.so
+
+here = pathlib.Path(__file__).parent.resolve()
+
+
+def load_tests(loader=None, tests=None, pattern='test_*.py'):
+    this_dir = os.path.dirname(__file__)
+    package_tests = loader.discover(start_dir=this_dir, pattern=pattern)
+    return package_tests
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Executes Orthanc integration tests.')
+    parser.add_argument('-k', '--pattern', dest='test_name_patterns', action='append', type=str, help='a test pattern (ex: Housekeeper.toto')
+    parser.add_argument('--orthanc_under_tests_hostname', type=str, default="localhost", help="orthanc under tests hostname")
+    parser.add_argument('--orthanc_under_tests_http_port', type=int, default=8042, help="orthanc under tests HTTP port")
+    parser.add_argument('--orthanc_under_tests_exe', type=str, default=None, help="path to the orthanc executable (if it must be launched by this script)")
+    parser.add_argument('--orthanc_previous_version_exe', type=str, default=None, help="path to the orthanc executable used to prepare previous version of storage/db (if it must be launched by this script and if different from orthanc_under_tests_exe)")
+    parser.add_argument('--orthanc_under_tests_docker_image', type=str, default=None, help="tag of the Docker image of the orthanc under tests (if it must be launched by this script)")
+    parser.add_argument('--skip_preparation', action='store_true', help="if this is a multi stage tests with preparations, skip the preparation")
+    parser.add_argument('--break_after_preparation', action='store_true', help="if this is a multi stage tests with preparations, pause after the preparation (such that you can start your own orthanc-under-tests in your debugger)")
+    parser.add_argument('-p', '--plugin', dest='plugins', action='append', type=str, help='path to a plugin to add to configuration')
+
+
+    args = parser.parse_args()
+
+    loader = unittest.TestLoader()
+    loader.testNamePatterns = args.test_name_patterns
+
+    Helpers.orthanc_under_tests_hostname = args.orthanc_under_tests_hostname
+    Helpers.orthanc_under_tests_http_port = args.orthanc_under_tests_http_port
+    Helpers.orthanc_under_tests_exe = args.orthanc_under_tests_exe
+    Helpers.plugins = args.plugins
+    if args.orthanc_previous_version_exe:
+        Helpers.orthanc_previous_version_exe = args.orthanc_previous_version_exe
+    else:
+        Helpers.orthanc_previous_version_exe = args.orthanc_under_tests_exe
+
+    Helpers.orthanc_under_tests_docker_image = args.orthanc_under_tests_docker_image
+    if args.skip_preparation:
+        Helpers.skip_preparation = True
+    if args.break_after_preparation:
+        Helpers.break_after_preparation = True
+
+    result = unittest.TextTestRunner(verbosity=2).run(load_tests(loader=loader))
+    if not result.wasSuccessful():
+        sys.exit(1)

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/requirements.txt	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,1 @@
+orthanc-api-client==0.3.5
\ No newline at end of file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/storages/.keep	Sat Apr 30 19:38:34 2022 +0200
@@ -0,0 +1,1 @@
+just a file to force directory creation !
\ No newline at end of file

--- a/README	Wed Apr 27 08:46:26 2022 +0200
+++ b/README	Sat Apr 30 19:38:34 2022 +0200
@@ -156,6 +156,12 @@
 and the orthanc package as well.
 
 
+NewTests
+========
+
+Check the README in the NewTests folder for more complex scenarios
+using python3 and a new test framework.
+
 
 Licensing
 =========

--- a/Tests/Tests.py	Wed Apr 27 08:46:26 2022 +0200
+++ b/Tests/Tests.py	Sat Apr 30 19:38:34 2022 +0200
@@ -3779,12 +3779,27 @@
         self.assertEqual('1.2.840.10008.5.1.4.1.1.4', DoGet(_REMOTE, '/instances/%s/metadata/SopClassUid' % a).strip())
         self.assertEqual('test', DoGet(_REMOTE, '/instances/%s/metadata/SopClassUid' % b).strip())
 
+        if IsOrthancVersionAbove(_REMOTE, 1, 11, 0):
+            # metadata before reconstruct
+            mba = DoGet(_REMOTE, '/instances/%s/metadata?expand' % a)
+            mbb = DoGet(_REMOTE, '/instances/%s/metadata?expand' % a)
+
+        # reconstruct by taking the new instance as the reference -> should repopulate study fields from this instance tags
         DoPost(_REMOTE, '/instances/%s/reconstruct' % b, {})
 
         CompareMainDicomTag('hello', a, 'study', 'StudyDescription')
         CompareMainDicomTag('world', a, 'series', 'SeriesDescription')
         CompareMainDicomTag('1.2.840.113619.2.176.2025.1499492.7040.1171286242.109', a, '', 'SOPInstanceUID')
 
+        if IsOrthancVersionAbove(_REMOTE, 1, 11, 0):
+            # metadata after reconstruct should have been preserved
+            maa = DoGet(_REMOTE, '/instances/%s/metadata?expand' % a)
+            mab = DoGet(_REMOTE, '/instances/%s/metadata?expand' % a)
+
+            self.assertEqual(mba, maa)
+            self.assertEqual(mbb, mab)
+
+
     def test_httpClient_lua(self):
         retries = 3
         result = ''