Mercurial > hg > orthanc-tests
changeset 931:1e29700a920f
non regression perfs: implemented retries
| author | Alain Mazy <am@orthanc.team> |
|---|---|
| date | Tue, 16 Jun 2026 11:31:07 +0200 |
| parents | 101f1b3b9643 |
| children | d7cc1ed1ad65 |
| files | NewTests/NonRegressionPerfs/test_non_regression_perfs.py |
| diffstat | 1 files changed, 37 insertions(+), 28 deletions(-) [+] |
line wrap: on
line diff
--- a/NewTests/NonRegressionPerfs/test_non_regression_perfs.py Mon Jun 15 11:44:41 2026 +0200 +++ b/NewTests/NonRegressionPerfs/test_non_regression_perfs.py Tue Jun 16 11:31:07 2026 +0200 @@ -42,40 +42,49 @@ parallel_count: Optional[int] = None, tolerance_pct = 0.25) -> None: + retry_counter = 0 + max_retries = 3 + has_succeeded = False + test_results[test_name] = {} - for (test_config_name, test_config) in test_configs.items(): - o = OrthancApiClient(orthanc_root_url=test_config["orthanc-url"]) + while not has_succeeded and retry_counter < max_retries: + for (test_config_name, test_config) in test_configs.items(): + o = OrthancApiClient(orthanc_root_url=test_config["orthanc-url"]) - start_time = time.perf_counter() - for i in range(0, reapeat_count): - if not parallel_count: # execute with a single thread - perform_test(o) - else: # execute in parallel + wait they all complete - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit(perform_test, o) for i in range(parallel_count)] - results = [future.result() for future in concurrent.futures.as_completed(futures)] + start_time = time.perf_counter() + for i in range(0, reapeat_count): + if not parallel_count: # execute with a single thread + perform_test(o) + else: # execute in parallel + wait they all complete + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(perform_test, o) for i in range(parallel_count)] + results = [future.result() for future in concurrent.futures.as_completed(futures)] - end_time = time.perf_counter() - test_results[test_name][test_config_name] = (end_time - start_time) * 1000 + end_time = time.perf_counter() + test_results[test_name][test_config_name] = (end_time - start_time) * 1000 + + ref_time = test_results[test_name]["ref"] + new_time = test_results[test_name]["new"] - ref_time = test_results[test_name]["ref"] - new_time = test_results[test_name]["new"] + delta_perf = new_time / ref_time - 1 + + if (delta_perf) > tolerance_pct: + retry_counter += 1 + if retry_counter < max_retries: + delta_text = f"+ {(delta_perf*100):>8.1f} % (FAILED - will retry)" + else: + delta_text = f"+ {(delta_perf*100):>8.1f} % (FAILED)" + elif (delta_perf) > 0: + delta_text = f"+ {(delta_perf*100):>8.1f} %" + has_succeeded = True + else: + delta_text = f"- {abs((delta_perf*100)):>8.1f} %" + has_succeeded = True - delta_perf = new_time / ref_time - 1 - - if (delta_perf) > tolerance_pct: - delta_text = f"+ {(delta_perf*100):>8.1f} % (FAILED)" - failed = True - elif (delta_perf) > 0: - delta_text = f"+ {(delta_perf*100):>8.1f} %" - failed = False - else: - delta_text = f"- {abs((delta_perf*100)):>8.1f} %" - failed = False - - print(f"{test_name:<60} | {ref_time:>20.3f} | {new_time:>20.3f} | {delta_text:>20}") - test_results[test_name]["success"] = not failed + print(f"{test_name:<60} | {ref_time:>20.3f} | {new_time:>20.3f} | {delta_text:>20}") + + test_results[test_name]["success"] = has_succeeded def test_non_regression_s3(self):
