Mercurial > hg > orthanc-databases
diff NOTES @ 436:f16faa1fdc46 pg-transactions
InsertOrUpdateMetadata function + UpdateAndGetStatistics
author | Alain Mazy <am@osimis.io> |
---|---|
date | Fri, 15 Dec 2023 17:11:26 +0100 |
parents | 326f8304daa1 |
children | d979f25e60cf |
line wrap: on
line diff
--- a/NOTES Thu Dec 14 09:25:45 2023 +0100 +++ b/NOTES Fri Dec 15 17:11:26 2023 +0100 @@ -169,32 +169,102 @@ In debug, no verbose logs Orthanc 1.12.1 + PG 5.1 (serializable mode) : test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 43.957 s -Orthanc mainline + PG mainline (serializable mode) : test_concurrent_anonymize_same_study with 4 workers and 10x repeat: XX s -Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 19.861 s -Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 20.027 s (with temporary tables) - test_concurrent_anonymize_same_study deletion took: 28.4 s +Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 15.744 s + test_concurrent_anonymize_same_study deletion took: 18.8 s Orthanc 1.12.1 + PG 5.1 (serializable mode) : test_concurrent_uploads_same_study with 20 workers and 1x repeat: 21.214 s -Orthanc mainline + PG mainline (serializable mode) : test_concurrent_uploads_same_study with 20 workers and 1x repeat: 23.010 s -Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_uploads_same_study with 20 workers and 1x repeat: 1.71 s (with RemainingAncestor.id) -Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_uploads_same_study with 20 workers and 1x repeat: 2.97 s (with temporary tables) +Orthanc mainline + PG mainline (read-committed mode) : test_concurrent_uploads_same_study with 20 workers and 1x repeat: 6.57 s (with temporary tables) Orthanc 1.12.1 + PG 5.1 (serializable mode) : test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 23.016 s -Orthanc mainline + PG mainline (read-committed mode) : test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 8.788 s -Orthanc mainline + PG mainline (read-committed mode) : test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 11.872 s (with temporary tables) +Orthanc mainline + PG mainline (read-committed mode) : test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 7.129 s + +With Docker with 10 connections SQL: +osimis/orthanc:23.11.0: TIMING test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 25.047 s +osimis/orthanc:current: TIMING test_concurrent_anonymize_same_study with 4 workers and 10x repeat: 8.649 s + +osimis/orthanc:23.11.0: TIMING test_concurrent_anonymize_same_study deletion took: 11.807 s +osimis/orthanc:current: TIMING test_concurrent_anonymize_same_study deletion took: 10.620 s + +osimis/orthanc:23.11.0: TIMING test_concurrent_uploads_same_study with 20 workers and 1x repeat: 10.736 s +osimis/orthanc:current: TIMING test_concurrent_uploads_same_study with 20 workers and 1x repeat: 3.465 s + +osimis/orthanc:23.11.0: TIMING test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 11.966 s +osimis/orthanc:current: TIMING test_upload_delete_same_study_from_multiple_threads with 5 workers and 30x repeat: 5.092 s + TODO: -- assert((statement.Next(), statement.IsDone())) commented out -> create temporary tables ? - try again temporary tables (with different names ? Have a switch in the code ?) +- check RETURNS SET OF ! + CREATE FUNCTION getfoo(int) RETURNS SETOF foo AS $$ + SELECT * FROM foo WHERE fooid = $1; + $$ LANGUAGE SQL; - have a separate "thread" to increment/decrement statistics because everybody is fighting to modify the GlobalIntegers rows + ExecuteSetResourcesContentTags(manager, "DicomIdentifiers", "i", -> make unique + handle on conflicts + countIdentifierTags, identifierTags); + + ExecuteSetResourcesContentTags(manager, "MainDicomTags", "t", -> make unique + handle on conflicts + countMainDicomTags, mainDicomTags); + + ExecuteSetResourcesContentMetadata(manager, HasRevisionsSupport(), countMetadata, metadata); handle on conflicts + +CREATE OR REPLACE FUNCTION InsertOrReplaceMetadata(VARIADIC params SETOF metadata) +RETURNS VOID AS $$ +BEGIN + + FOR i IN 1..array_length(my_params, 1) LOOP + my_sum := my_sum + my_params[i].my_column; + END LOOP; + RETURN my_sum; +END; +$$ LANGUAGE plpgsql; + +#include <postgresql/libpq-fe.h> + +int main() { + PGconn *conn = PQconnectdb("dbname=mydb user=myuser password=mypassword"); + if (PQstatus(conn) != CONNECTION_OK) { + std::cerr << "Connection to database failed: " << PQerrorMessage(conn) << std::endl; + PQfinish(conn); + return 1; + } + + const char *function_name = "my_function"; + int num_params = 3; + int params[] = {1, 2, 3}; + + char *query = PQescapeLiteral(conn, function_name, strlen(function_name)); + for (int i = 0; i < num_params; i++) { + char *param = PQescapeLiteral(conn, (const char *)¶ms[i], sizeof(int)); + query = (char *)realloc(query, strlen(query) + strlen(param) + 1); + strcat(query, ","); + strcat(query, param); + PQfreemem(param); + } + strcat(query, ")"); + + PGresult *res = PQexec(conn, query); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + std::cerr << "Function call failed: " << PQerrorMessage(conn) << std::endl; + PQfreemem(query); + PQclear(res); + PQfinish(conn); + return 1; + } + + int result = atoi(PQgetvalue(res, 0, 0)); + std::cout << "Result: " << result << std::endl; + + PQfreemem(query); + PQclear(res); + PQfinish(conn); + + - test events generation StableSeries .... -- test RemainingAncestor response in integration tests: OK -- disable MySQL ODBC plugin - run tests with docker localy + in CI - check https://discourse.orthanc-server.org/t/image-insert-are-too-slow-databse-performance-too-poor-when-using-mysql-mariadb/3820 - PatientAddedFunc contains an IF - validate upgrade DB from previous Orthanc and from scratch - test with older version of PG +- In Docker images, re-enable MySQL & ODBC plugins + tests DONE: - force the create/update DB transaction to be serializable (not needed: this is handled by POSTGRESQL_LOCK_DATABASE_SETUP)