Mercurial > hg > orthanc-java
changeset 28:43923934e934
added sample: deep learning for mammography
| author | Sebastien Jodogne <s.jodogne@gmail.com> |
|---|---|
| date | Wed, 12 Jun 2024 13:58:29 +0200 |
| parents | 4a750ca9461e |
| children | 118adbad648f |
| files | .hgignore .reuse/dep5 Samples/MammographyDeepLearning/configuration.json Samples/MammographyDeepLearning/pom.xml Samples/MammographyDeepLearning/src/main/java/Detection.java Samples/MammographyDeepLearning/src/main/java/DicomCode.java Samples/MammographyDeepLearning/src/main/java/DicomToolbox.java Samples/MammographyDeepLearning/src/main/java/ImageProcessing.java Samples/MammographyDeepLearning/src/main/java/Main.java Samples/MammographyDeepLearning/src/main/java/OrthancConnection.java Samples/MammographyDeepLearning/src/main/java/Rectangle.java Samples/MammographyDeepLearning/src/main/java/RetinaNet.java Samples/MammographyDeepLearning/src/main/resources/OrthancExplorer.js |
| diffstat | 13 files changed, 1737 insertions(+), 1 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Tue Jun 11 19:14:51 2024 +0200 +++ b/.hgignore Wed Jun 12 13:58:29 2024 +0200 @@ -19,4 +19,12 @@ Samples/Dcm4Che/target/ Samples/FHIR/.idea/ Samples/FHIR/OrthancStorage/ +Samples/FHIR/libOrthancDicomWeb.so Samples/FHIR/target/ +Samples/MammographyDeepLearning/.idea/ +Samples/MammographyDeepLearning/2024-03-08-retina_res50_trained_08_03.torchscript +Samples/MammographyDeepLearning/2024-03-15-StoneWebViewer-DICOM-SR.zip +Samples/MammographyDeepLearning/OrthancStorage/ +Samples/MammographyDeepLearning/dependency-reduced-pom.xml +Samples/MammographyDeepLearning/libOrthancDicomWeb.so +Samples/MammographyDeepLearning/target/
--- a/.reuse/dep5 Tue Jun 11 19:14:51 2024 +0200 +++ b/.reuse/dep5 Wed Jun 12 13:58:29 2024 +0200 @@ -3,7 +3,7 @@ Upstream-Contact: Sebastien Jodogne <s.jodogne@gmail.com> Source: https://orthanc.uclouvain.be/ -Files: NEWS README Samples/Basic/NOTES.txt Samples/Dcm4Che/NOTES.txt Samples/FHIR/NOTES.txt Samples/FHIR/configuration.json +Files: NEWS README Samples/Basic/NOTES.txt Samples/Dcm4Che/NOTES.txt Samples/FHIR/NOTES.txt Samples/FHIR/configuration.json Samples/MammographyDeepLearning/NOTES.txt Samples/MammographyDeepLearning/configuration.json Copyright: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium License: CC0-1.0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/configuration.json Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,11 @@ +{ + "Plugins" : [ + "../../Plugin/Build", + "./libOrthancDicomWeb.so" + ], + "Java" : { + "Enabled" : true, + "Classpath" : "target/OrthancMammographyDeepLearning-mainline.jar", + "InitializationClass" : "Main" + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/pom.xml Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,162 @@ +<?xml version='1.0' encoding='UTF-8' ?> + +<!-- + + SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + SPDX-License-Identifier: GPL-3.0-or-later + +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifest> + <mainClass>Main</mainClass> + </manifest> + </archive> + </configuration> + </plugin> + + <!-- Maven Shade plugin - for creating the uberjar / fatjar --> + <!-- see http://maven.apache.org/plugins/maven-shade-plugin/index.html for details --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <transformers> + <transformer + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + </transformers> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + + <!-- Include the Orthanc Java SDK --> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.basedir}/../../JavaSDK/be/uclouvain/orthanc</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + </build> + + <groupId>OrthancMammographyDeepLearning</groupId> + <artifactId>OrthancMammographyDeepLearning</artifactId> + <version>mainline</version> + + <properties> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> + <djl.version>0.26.0</djl.version> + <exec.mainClass>Main</exec.mainClass> + </properties> + + <repositories> + <repository> + <id>djl.ai</id> + <url>https://oss.sonatype.org/content/repositories/snapshots/</url> + </repository> + </repositories> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>ai.djl</groupId> + <artifactId>bom</artifactId> + <version>${djl.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + + <dependencies> + <dependency> + <groupId>ai.djl</groupId> + <artifactId>api</artifactId> + </dependency> + <dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-engine</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-jni</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-model-zoo</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>1.7.36</version> + <scope>runtime</scope> + </dependency> + + <!-- Include PyTorch native library for platforms --> + <dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-native-cpu</artifactId> + <classifier>linux-x86_64</classifier> + <scope>runtime</scope> + </dependency> + <!--dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-native-cpu</artifactId> + <classifier>win-x86_64</classifier> + <scope>runtime</scope> + </dependency--> + <!--dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-native-cpu</artifactId> + <classifier>osx-x86_64</classifier> + <scope>runtime</scope> + </dependency--> + <!--dependency> + <groupId>ai.djl.pytorch</groupId> + <artifactId>pytorch-native-cpu</artifactId> + <classifier>osx-aarch64</classifier> + <scope>runtime</scope> + </dependency--> + + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <version>20240303</version> + </dependency> + + </dependencies> +</project>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/Detection.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,56 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import java.io.Serializable; + +class Detection implements Comparable<Detection>, Serializable { + private Rectangle rectangle; + private int label; + private double score; + + public Detection(Rectangle rectangle, + int label, + double score) { + this.rectangle = rectangle; + this.label = label; + this.score = score; + } + + public Rectangle getRectangle() { + return rectangle; + } + + public int getLabel() { + return label; + } + + public double getScore() { + return score; + } + + @Override + public int compareTo(Detection peak) { + return Double.compare(peak.score, score); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/DicomCode.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,62 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import org.json.JSONObject; + +class DicomCode { + private static final String CODE_VALUE = "0008,0100"; + private static final String CODING_SCHEME_DESIGNATOR = "0008,0102"; + private static final String CODE_MEANING = "0008,0104"; + + private String codeValue; + private String codingSchemeDesignator; + private String codeMeaning; + + public DicomCode(String codeValue, + String codingSchemeDesignator, + String codeMeaning) { + this.codeValue = codeValue; + this.codingSchemeDesignator = codingSchemeDesignator; + this.codeMeaning = codeMeaning; + } + + public String getCodeValue() { + return codeValue; + } + + public String getCodingSchemeDesignator() { + return codingSchemeDesignator; + } + + public String getCodeMeaning() { + return codeMeaning; + } + + public JSONObject toJson() { + return new JSONObject(). + put(CODE_VALUE, codeValue). + put(CODING_SCHEME_DESIGNATOR, codingSchemeDesignator). + put(CODE_MEANING, codeMeaning); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/DicomToolbox.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,287 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.IOException; +import java.util.List; + +public class DicomToolbox { + private static final String MAPPING_RESOURCE = "0008,0105"; + private static final String REFERENCED_SOP_CLASS_UID = "0008,1150"; + private static final String REFERENCED_SOP_INSTANCE_UID = "0008,1155"; + private static final String REFERENCED_SOP_SEQUENCE = "0008,1199"; + + private static final String CONTENT_TEMPLATE_SEQUENCE = "0040,a504"; + private static final String TEMPLATE_IDENTIFIER = "0040,db00"; + private static final String RELATIONSHIP_TYPE = "0040,a010"; + private static final String CONCEPT_NAME_CODE_SEQUENCE = "0040,a043"; + private static final String VALUE_TYPE = "0040,a040"; + private static final String CONCEPT_CODE_SEQUENCE = "0040,a168"; + private static final String CONTINUITY_OF_CONTENT = "0040,a050"; + private static final String CONTENT_SEQUENCE = "0040,a730"; + private static final String TEXT_VALUE_ATTRIBUTE = "0040,a160"; + private static final String UID_ATTRIBUTE = "0040,a124"; + private static final String MEASURED_VALUE_SEQUENCE = "0040,a300"; + private static final String MEASUREMENT_UNITS_CODE_SEQUENCE = "0040,08ea"; + private static final String FLOATING_POINT_VALUE = "0040,a161"; + private static final String NUMERIC_VALUE = "0040,a30a"; + private static final String GRAPHIC_DATA = "0070,0022"; + private static final String GRAPHIC_TYPE = "0070,0023"; + + static JSONObject createDicomSR(OrthancConnection client, + String instance, + List<Detection> detections, + double aspectRatio) throws IOException, InterruptedException { + final JSONObject instanceTags = client.doGetAsJsonObject("/instances/" + instance + "/tags?short"); + final String trackingUid = client.doGetAsString("/tools/generate-uid?level=instance"); + + String instanceStudy = client.doGetAsJsonObject("/instances/" + instance + "/study").getString("ID"); + + // https://dicom.nema.org/medical/dicom/2024a/output/chtml/part16/chapter_A.html#sect_TID_1500 + // Sample TID 1500: https://dicom.nema.org/medical/dicom/current/output/chtml/part21/sect_a.7.2.html + + + JSONObject json = new JSONObject(); + json.put("Parent", instanceStudy); + + JSONObject tags = new JSONObject(); + tags.put("0008,0016", "1.2.840.10008.5.1.4.1.1.88.33"); // SOP Class UID + tags.put("0008,0060", "SR"); // Modality + tags.put("0008,0070", ""); // Manufacturer + tags.put("0008,1111", new JSONArray()); // Referenced Performed Procedure Step Sequence is type 2 (required but can be empty) + tags.put("0020,0011", "1"); // Series number + tags.put("0020,0013", "1"); // Instance number + tags.put("0040,a040", "CONTAINER"); // Value type for SR: https://dicom.nema.org/medical/dicom/2024a/output/chtml/part03/sect_C.17.3.html + + // CID 7021 Measurement Report Document Title: https://dicom.nema.org/medical/dicom/2024a/output/chtml/part16/sect_CID_7021.html + tags.append("0040,a043", new DicomCode("126000", "DCM", "Imaging Measurement Report").toJson()); + + tags.put("0040,a050", "CONTINUOUS"); // Continuity of content + tags.put("0040,a372", new JSONArray()); // Performed procedure code sequence + + // Current Requested Procedure Evidence: References to the StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID, and SOPClassUID + tags.append("0040,a375", new JSONObject(). + put("0020,000d", instanceTags.getString("0020,000d")). + append("0008,1115", new JSONObject(). + put("0020,000e", instanceTags.getString("0020,000e")). + append("0008,1199", new JSONObject(). + put("0008,1150", instanceTags.getString("0008,0016")). + put("0008,1155", instanceTags.getString("0008,0018"))))); + + tags.put("0040,a491", "PARTIAL"); // Completion flag + tags.put("0040,a493", "UNVERIFIED"); // Verification flag + tags.put("0040,a496", "PRELIMINARY"); // Preliminary flag + + // Content template sequence, which indicates TID 1500 + setTemplateId(tags, "1500"); + + JSONArray contentSequence = new JSONArray(); + contentSequence.put(createCodeRelationship( + new DicomCode("121049", "DCM", "Language of Content Item and Descendants"), + new DicomCode("en-US", "RFC5646", "English (United States)"))); + + // This is "procedure_reported" in "highdicom.sr.MeasurementReport()" + contentSequence.put(createCodeRelationship( + new DicomCode("121058", "DCM", "Procedure reported"), + new DicomCode("43468-8", "LN", "XR unspecified body region"))); + + JSONArray measurementGroups = new JSONArray(); + + for (Detection detection : detections) { + JSONArray measurements = new JSONArray(); + measurements.put(createTextContext( + new DicomCode("112039", "DCM", "Tracking Identifier"), + "Orthanc Deep Learning for Mammography")); + + measurements.put(createUidReference( + new DicomCode("112040", "DCM", "Tracking Unique Identifier"), + trackingUid)); + + measurements.put(createNumericValue( + new DicomCode("111047", "DCM", "Probability of cancer"), + new DicomCode("%", "UCUM", "Percent"), + detection.getScore() * 100.0)); + + measurements.put(createRectangle( + new DicomCode("111030", "DCM", "Image Region"), + instanceTags.getString("0008,0016"), instanceTags.getString("0008,0018"), + detection.getRectangle(), aspectRatio)); + + JSONObject measurementGroup = createContainer(new DicomCode("125007", "DCM", "Measurement Group"), measurements); + setTemplateId(measurementGroup, "1410"); + + measurementGroups.put(measurementGroup); + } + + JSONObject imagingMeasurements = createContainer(new DicomCode("126010", "DCM", "Imaging Measurements"), measurementGroups); + contentSequence.put(imagingMeasurements); + + tags.put("0040,a730", contentSequence); + + json.put("Tags", tags); + + return client.doPostAsJsonObject("/tools/create-dicom", json.toString()); + } + + static public class Point2D { + private double x; + private double y; + + public Point2D(double x, + double y) { + this.x = x; + this.y = y; + } + + public double getX() { + return x; + } + + public double getY() { + return y; + } + } + + public static JSONObject setTemplateId(JSONObject target, + String templateId) { + if (target.has(CONTENT_TEMPLATE_SEQUENCE)) { + throw new IllegalStateException(); + } else { + target.append(CONTENT_TEMPLATE_SEQUENCE, new JSONObject(). + put(MAPPING_RESOURCE, "DCMR"). + put(TEMPLATE_IDENTIFIER, templateId)); + return target; + } + } + + public static JSONObject createCodeRelationship(DicomCode concept, + DicomCode referencedCode) { + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "HAS CONCEPT MOD"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.put(VALUE_TYPE, "CODE"); + result.append(CONCEPT_CODE_SEQUENCE, referencedCode.toJson()); + return result; + } + + public static JSONObject createContainer(DicomCode concept, + JSONArray content) { + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "CONTAINS"); + result.put(VALUE_TYPE, "CONTAINER"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.put(CONTINUITY_OF_CONTENT, "CONTINUOUS"); + result.put(CONTENT_SEQUENCE, content); + return result; + } + + public static JSONObject createTextContext(DicomCode concept, + String value) { + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "HAS OBS CONTEXT"); + result.put(VALUE_TYPE, "TEXT"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.put(TEXT_VALUE_ATTRIBUTE, value); + return result; + } + + public static JSONObject createUidReference(DicomCode concept, + String uid) { + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "HAS OBS CONTEXT"); + result.put(VALUE_TYPE, "UIDREF"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.put(UID_ATTRIBUTE, uid); + return result; + } + + public static JSONObject createNumericValue(DicomCode concept, + DicomCode unit, + double value) { + String ds = String.valueOf(value); + if (ds.length() > 16) { + ds = ds.substring(0, 16); // The DS VR must have less than 16 characters + } + + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "CONTAINS"); + result.put(VALUE_TYPE, "NUM"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.append(MEASURED_VALUE_SEQUENCE, new JSONObject(). + append(MEASUREMENT_UNITS_CODE_SEQUENCE, unit.toJson()). + put(FLOATING_POINT_VALUE, String.valueOf(value)). + put(NUMERIC_VALUE, ds)); + return result; + } + + public static JSONObject createSelectedFromImage(String sopClassUid, + String sopInstanceUid) { + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "SELECTED FROM"); + result.put(VALUE_TYPE, "IMAGE"); + result.append(REFERENCED_SOP_SEQUENCE, new JSONObject(). + put(REFERENCED_SOP_CLASS_UID, sopClassUid). + put(REFERENCED_SOP_INSTANCE_UID, sopInstanceUid)); + result.append(CONCEPT_NAME_CODE_SEQUENCE, new DicomCode("111040", "DCM", "Original Source").toJson()); + return result; + } + + public static JSONObject createPolyline2D(DicomCode concept, + String sopClassUid, + String sopInstanceUid, + Point2D vertices[]) { + String polyline = new String(); + for (int i = 0; i < vertices.length; i++) { + if (!polyline.isEmpty()) { + polyline = polyline + "\\"; + } + polyline += String.valueOf(vertices[i].getX()) + "\\" + String.valueOf(vertices[i].getY()); + } + + JSONObject result = new JSONObject(); + result.put(RELATIONSHIP_TYPE, "CONTAINS"); + result.put(VALUE_TYPE, "SCOORD"); + result.append(CONCEPT_NAME_CODE_SEQUENCE, concept.toJson()); + result.append(CONTENT_SEQUENCE, createSelectedFromImage(sopClassUid, sopInstanceUid)); + result.put(GRAPHIC_DATA, polyline); // WARNING: This necessitates Orthanc > 1.12.4: https://orthanc.uclouvain.be/hg/orthanc/rev/dedbf019a707 + result.put(GRAPHIC_TYPE, "POLYLINE"); + return result; + } + + public static JSONObject createRectangle(DicomCode concept, + String sopClassUid, + String sopInstanceUid, + Rectangle rectangle, + double aspectRatio) { + Point2D vertices[] = new Point2D[5]; + vertices[0] = new Point2D(rectangle.getX1() * aspectRatio, rectangle.getY1() * aspectRatio); + vertices[1] = new Point2D(rectangle.getX2() * aspectRatio, rectangle.getY1() * aspectRatio); + vertices[2] = new Point2D(rectangle.getX2() * aspectRatio, rectangle.getY2() * aspectRatio); + vertices[3] = new Point2D(rectangle.getX1() * aspectRatio, rectangle.getY2() * aspectRatio); + vertices[4] = new Point2D(rectangle.getX1() * aspectRatio, rectangle.getY1() * aspectRatio); + return createPolyline2D(concept, sopClassUid, sopInstanceUid, vertices); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/ImageProcessing.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,86 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.DataType; +import ai.djl.ndarray.types.Shape; + +import java.awt.*; +import java.awt.image.BufferedImage; +import java.awt.image.DataBuffer; + +public class ImageProcessing { + static BufferedImage resizeImage(BufferedImage originalImage, int targetWidth, int targetHeight) { + BufferedImage resizedImage = new BufferedImage(targetWidth, targetHeight, originalImage.getType()); + Graphics2D graphics2D = resizedImage.createGraphics(); + graphics2D.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + graphics2D.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + graphics2D.drawImage(originalImage, 0, 0, targetWidth, targetHeight, null); + graphics2D.dispose(); + return resizedImage; + } + + static NDArray imageToTensor(NDManager manager, + BufferedImage image) { + if (image.getType() != image.TYPE_USHORT_GRAY /* 16 bpp */ && + image.getType() != image.TYPE_BYTE_GRAY /* 8 bpp */) { + throw new IllegalArgumentException(); + } + + float pixels[] = new float[image.getHeight() * image.getWidth()]; + + DataBuffer db = image.getData().getDataBuffer(); + + int pos = 0; + for (int y = 0; y < image.getHeight(); y++) { + for (int x = 0; x < image.getWidth(); x++, pos++) { + pixels[pos] = db.getElemFloat(pos); + } + } + + return manager.create(pixels, new Shape(1, image.getHeight(), image.getWidth())); + } + + static NDArray standardize(NDArray image) { + if (image.getDataType() != DataType.FLOAT32 || + image.getShape().dimension() != 3 || + (image.getShape().get(0) != 1 && + image.getShape().get(0) != 3)) { + throw new IllegalArgumentException(); + } + + // Standardize the image to zero mean and 1 standard deviation + NDArray doubleImage = image.toType(DataType.FLOAT64, false); + NDArray squared = doubleImage.mul(doubleImage); + + double asum = doubleImage.sum().getDouble(); + double asumOfSquares = squared.sum().getDouble(); + double n = doubleImage.getShape().size(); + double amean = asum / n; + double astd = Math.sqrt((asumOfSquares - asum * asum / n) / n); + + return image.add(-amean).div(astd); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/Main.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,230 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDManager; +import be.uclouvain.orthanc.Callbacks; +import be.uclouvain.orthanc.ChangeType; +import be.uclouvain.orthanc.Functions; +import be.uclouvain.orthanc.HttpMethod; +import be.uclouvain.orthanc.ResourceType; +import be.uclouvain.orthanc.RestOutput; +import org.apache.commons.compress.utils.IOUtils; +import org.json.JSONObject; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class Main { + private static final String MODEL_PATH = "2024-03-08-retina_res50_trained_08_03.torchscript"; + private static final String STONE_VERSION = "2024-03-15-StoneWebViewer-DICOM-SR"; + private static final String STONE_PATH = "2024-03-15-StoneWebViewer-DICOM-SR.zip"; + + private static ZipFile stone; + private static NDManager manager; + private static RetinaNet retinaNet; + private static Map<String, String> mimeTypes = new HashMap<>(); + + static { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + OrthancConnection.download("2024-03-08-retina_res50_trained_08_03.torchscript", executor, + "https://orthanc.uclouvain.be/downloads/cross-platform/orthanc-mammography/models/2024-03-08-retina_res50_trained_08_03.torchscript", + 146029397L, "b3de8f562de683bc3515fe93ae102fd4"); + OrthancConnection.download("2024-03-15-StoneWebViewer-DICOM-SR.zip", executor, + "https://github.com/jodogne/orthanc-mammography/raw/master/viewer/2024-03-15-StoneWebViewer-DICOM-SR.zip", + 4742571L, "de952da6fc74a9d4b78ca5064a6a7318"); + } catch (IOException | NoSuchAlgorithmException | InterruptedException e) { + throw new RuntimeException(e); + } finally { + executor.shutdown(); + } + + try { + stone = new ZipFile(STONE_PATH); + } catch (IOException e) { + throw new RuntimeException(e); + } + + Functions.logWarning("Initializing Deep Java Library"); + manager = NDManager.newBaseManager(); + + Functions.logWarning("Loading RetinaNet model"); + try { + retinaNet = new RetinaNet(MODEL_PATH); + } catch (Exception e) { + throw new RuntimeException(e); + } + + Functions.logWarning("RetinaNet model is ready"); + + mimeTypes.put("css", "text/css"); + mimeTypes.put("gif", "image/gif"); + mimeTypes.put("html", "text/html"); + mimeTypes.put("jpeg", "image/jpeg"); + mimeTypes.put("js", "text/javascript"); + mimeTypes.put("png", "image/png"); + + try (InputStream stream = Main.class.getResourceAsStream("OrthancExplorer.js")) { + byte[] content = IOUtils.toByteArray(stream); + Functions.extendOrthancExplorer(new String(content, StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException(e); + } + + Callbacks.register(new Callbacks.OnChange() { + @Override + public void call(ChangeType changeType, ResourceType resourceType, String resourceId) { + switch (changeType) { + case ORTHANC_STARTED: + OrthancConnection connection = OrthancConnection.createForPlugin(); + try { + if (!connection.isOrthancVersionAbove(1, 12, 5)) { + throw new RuntimeException("Your version of Orthanc must be >= 1.12.5 to run this plugin"); + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + break; + + case ORTHANC_STOPPED: + Functions.logWarning("Finalizing Deep Java Library"); + if (retinaNet != null) { + retinaNet.close(); + } + + if (manager != null) { + manager.close(); + } + + System.gc(); + System.runFinalization(); + break; + + default: + break; + } + } + }); + + Callbacks.register("/java-mammography-apply", new Callbacks.OnRestRequest() { + @Override + public void call(RestOutput output, + HttpMethod method, + String uri, + String[] regularExpressionGroups, + Map<String, String> headers, + Map<String, String> getParameters, + byte[] body) { + if (method != HttpMethod.POST) { + output.sendMethodNotAllowed("POST"); + return; + } + + JSONObject request = new JSONObject(new String(body, StandardCharsets.UTF_8)); + + String instanceId = request.getString("instance"); + if (instanceId == null) { + throw new RuntimeException("Missing instance identifier"); + } + + Functions.logWarning("Applying RetinaNet to instance: " + instanceId); + OrthancConnection connection = OrthancConnection.createForPlugin(); + + try { + BufferedImage image = connection.getGrayscaleFrame(instanceId, 0); + + double largestSide = Math.max(image.getWidth(), image.getHeight()); + BufferedImage resized = ImageProcessing.resizeImage(image, + (int) Math.round((double) image.getWidth() * 2048.0 / largestSide), + (int) Math.round((double) image.getHeight() * 2048.0 / largestSide)); + + double aspectRatio = (double) image.getWidth() / (double) resized.getWidth(); + + NDArray grayscale = ImageProcessing.imageToTensor(manager, resized); + grayscale = ImageProcessing.standardize(grayscale); + + NDArray rgb = grayscale.concat(grayscale).concat(grayscale); // Create a "RGB" image from grayscale pixel values + List<Detection> detections = retinaNet.apply(rgb); + + JSONObject created = DicomToolbox.createDicomSR(connection, instanceId, detections, aspectRatio); + + String id = created.getString("ID"); + Functions.logWarning("Detection results stored in DICOM-SR instance: " + id); + + output.answerBuffer(created.toString().getBytes(StandardCharsets.UTF_8), "application/json"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + Callbacks.register("/java-mammography-viewer/(.*)", new Callbacks.OnRestRequest() { + @Override + public void call(RestOutput output, + HttpMethod method, + String uri, + String[] regularExpressionGroups, + Map<String, String> headers, + Map<String, String> getParameters, + byte[] body) { + if (method != HttpMethod.GET) { + output.sendMethodNotAllowed("GET"); + return; + } + + String path = regularExpressionGroups[0]; + int dot = path.lastIndexOf("."); + if (dot < 0) { + output.sendHttpStatus((short) 404, new byte[0]); + } else { + String extension = path.substring(dot + 1); + String mime = mimeTypes.getOrDefault(extension, "application/octet-stream"); + + ZipEntry entry = stone.getEntry(STONE_VERSION + "/" + path); + if (entry == null) { + output.sendHttpStatus((short) 404, new byte[0]); + } else { + try (InputStream stream = stone.getInputStream(entry)) { + output.answerBuffer(IOUtils.toByteArray(stream), mime); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + }); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/OrthancConnection.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,277 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import org.json.JSONObject; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; + +public abstract class OrthancConnection { + private static final String PIXEL_REPRESENTATION = "0028,0103"; + private static final String BITS_STORED = "0028,0101"; + private static final String SAMPLES_PER_PIXEL = "0028,0002"; + + public static OrthancConnection createHttpClient(ExecutorService executor, + String baseUrl) { + return new OrthancConnection() { + private HttpClient client = HttpClient.newBuilder().executor(executor).build(); + + @Override + public byte[] doGetAsByteArray(String uri) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + uri)) + .build(); + HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200) { + throw new RuntimeException(); + } else { + return response.body(); + } + } + + @Override + public byte[] doPostAsByteArray(String uri, byte[] body) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + uri)) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .build(); + HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200) { + throw new RuntimeException(); + } else { + return response.body(); + } + } + }; + } + + public static OrthancConnection createForPlugin() { + return new OrthancConnection() { + @Override + public byte[] doGetAsByteArray(String uri) throws IOException, InterruptedException { + return be.uclouvain.orthanc.Functions.restApiGet(uri); + } + + @Override + public byte[] doPostAsByteArray(String uri, byte[] body) throws IOException, InterruptedException { + return be.uclouvain.orthanc.Functions.restApiPost(uri, body); + } + }; + } + + public abstract byte[] doGetAsByteArray(String uri) throws IOException, InterruptedException; + + public abstract byte[] doPostAsByteArray(String uri, + byte[] body) throws IOException, InterruptedException; + + public String doGetAsString(String uri) throws IOException, InterruptedException { + return new String(doGetAsByteArray(uri), StandardCharsets.UTF_8); + } + + public JSONObject doGetAsJsonObject(String uri) throws IOException, InterruptedException { + return new JSONObject(doGetAsString(uri)); + } + + public String doPostAsString(String uri, + String body) throws IOException, InterruptedException { + byte[] answer = doPostAsByteArray(uri, body.getBytes(StandardCharsets.UTF_8)); + return new String(answer, StandardCharsets.UTF_8); + } + + public JSONObject doPostAsJsonObject(String uri, + String body) throws IOException, InterruptedException { + return new JSONObject(doPostAsString(uri, body)); + } + + public BufferedImage getGrayscaleFrame(String instance, + int frame) throws IOException, InterruptedException { + if (frame < 0) { + throw new IllegalArgumentException(); + } + + JSONObject tags = doGetAsJsonObject("/instances/" + instance + "/tags?short"); + + if (!tags.has(PIXEL_REPRESENTATION) || + tags.getInt(PIXEL_REPRESENTATION) == 1) { + throw new IllegalArgumentException("Negative pixels not supported"); + } + + if (!tags.has(BITS_STORED) || + (tags.getInt(BITS_STORED) != 8 && + tags.getInt(BITS_STORED) != 16)) { + throw new IllegalArgumentException("Pixel depth not supported"); + } + + if (!tags.has(SAMPLES_PER_PIXEL) || + tags.getInt(SAMPLES_PER_PIXEL) != 1) { + throw new IllegalArgumentException("Color images not implemented"); + } + + byte[] png = doGetAsByteArray("/instances/" + instance + "/frames/" + frame + "/image-uint16"); + try (ByteArrayInputStream stream = new ByteArrayInputStream(png)) { + return ImageIO.read(stream); + } + } + + boolean isOrthancVersionAbove(int major, + int minor, + int revision) throws IOException, InterruptedException { + JSONObject system = doGetAsJsonObject("/system"); + String version = system.getString("Version"); + if (version == null) { + throw new RuntimeException("Not an Orthanc server"); + } + + if (version.equals("mainline")) { + return true; + } else { + String[] items = version.split("\\."); + if (items.length != 3) { + throw new RuntimeException("Cannot parse Orthanc version: " + version); + } + + int thisMajor = Integer.valueOf(items[0]); + int thisMinor = Integer.valueOf(items[1]); + int thisRevision = Integer.valueOf(items[2]); + + return (thisMajor > major || + (thisMajor == major && thisMinor > minor) || + (thisMajor == major && thisMinor == minor && thisRevision >= revision)); + } + } + + + public static void download(String targetPath, + ExecutorService executor, + String url, + long expectedSize, + String expectedMd5) throws IOException, NoSuchAlgorithmException, InterruptedException { + class FileDownloader implements HttpResponse.BodyHandler<Void>, Consumer<Optional<byte[]>> { + private FileOutputStream target; + private String url; + private boolean success; + private long contentLength; + private long size; + + public FileDownloader(final FileOutputStream target, + final String url) throws FileNotFoundException { + this.target = target; + this.success = false; + } + + @Override + public HttpResponse.BodySubscriber<Void> apply(final HttpResponse.ResponseInfo responseInfo) { + if (responseInfo.statusCode() != 200) { + throw new RuntimeException("URL does not exist: " + url); + } + + final OptionalLong contentLength = responseInfo.headers().firstValueAsLong("Content-Length"); + if (contentLength.isEmpty()) { + throw new RuntimeException("Server does not provide a content length: " + url); + } + this.contentLength = contentLength.getAsLong(); + + return HttpResponse.BodySubscribers.ofByteArrayConsumer(this); + } + + @Override + public void accept(final Optional<byte[]> bytes) { + if (bytes.isEmpty()) { + System.out.println(); + System.out.flush(); + if (this.success) { + throw new IllegalStateException("File already closed"); + } + if (this.size != this.contentLength) { + throw new RuntimeException("Server has not answered with the proper content length"); + } + this.success = true; + } else { + try { + this.target.write(bytes.get()); + } catch (IOException e) { + System.out.println(); + throw new RuntimeException("Cannot write to file"); + } + this.size += bytes.get().length; + final int BAR_WIDTH = 30; + final int a = Math.min(30, Math.round(this.size / (float) this.contentLength * 30.0f)); + System.out.print("\r Progress: [" + "=".repeat(a) + " ".repeat(30 - a) + "]"); + System.out.flush(); + } + } + + boolean isSuccess() { + return this.success; + } + } + + + System.out.println("Downloading: " + url); + + if (Files.exists(Paths.get(targetPath))) { + System.out.println(" File already downloaded"); + } else { + FileOutputStream target = new FileOutputStream(targetPath); + HttpClient client = HttpClient.newBuilder().executor(executor).followRedirects(HttpClient.Redirect.NORMAL).build(); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).build(); + + FileDownloader consumer = new FileDownloader(target, url); + HttpResponse<Void> response = client.send(request, (HttpResponse.BodyHandler<Void>) consumer); + target.close(); + + if (!consumer.isSuccess()) { + throw new IOException("Could not download: " + url); + } + } + + byte[] content = Files.readAllBytes(Paths.get(targetPath)); + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(content); + + final String actualMd5 = new BigInteger(1, md.digest()).toString(16); + if (content.length != expectedSize || + !actualMd5.equals(expectedMd5)) { + throw new IOException("Incorrect content in a download file, please remove it and retry: " + targetPath); + } + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/Rectangle.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,76 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import java.io.Serializable; + +class Rectangle implements Serializable { + private float x1; + private float y1; + private float x2; + private float y2; + + public Rectangle(float x1, + float y1, + float x2, + float y2) { + if (x1 > x2) { + throw new IllegalArgumentException(); + } + if (y1 > y2) { + throw new IllegalArgumentException(); + } + this.x1 = x1; + this.y1 = y1; + this.x2 = x2; + this.y2 = y2; + } + + public float getX1() { + return x1; + } + + public float getY1() { + return y1; + } + + public float getX2() { + return x2; + } + + public float getY2() { + return y2; + } + + public float getWidth() { + return x2 - x1; + } + + public float getHeight() { + return y2 - y1; + } + + public float getArea() { + return getWidth() * getHeight(); + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/java/RetinaNet.java Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,348 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +import ai.djl.MalformedModelException; +import ai.djl.inference.Predictor; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.TranslateException; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +public class RetinaNet { + private ZooModel model; + + private static double sigmoid(double x) { + // This corresponds to "torch.sigmoid()", aka. "torch.expit()" + return 1.0f / (1.0f + Math.exp(-x)); + } + + private static float clamp(float value, + float min, + float max) { + if (value < min) { + return min; + } else if (value > max) { + return max; + } else { + return value; + } + } + + /** + * This function corresponds to method "decode_single()" in class "BoxCoder" in: + * https://github.com/pytorch/vision/blob/main/torchvision/models/detection/_utils.py + */ + public static Rectangle decodeSingle(float[] rel_codes, + float[] boxes, + int image_shape_width, + int image_shape_height) { + /** + * The following constant come from the constructor of "BoxCoder" in: + * https://github.com/pytorch/vision/blob/main/torchvision/models/detection/_utils.py + * + * self, weights: Tuple[float, float, float, float], bbox_xform_clip: float = math.log(1000.0 / 16) + */ + final float BBOX_XFORM_CLIP = (float) Math.log(1000.0 / 16.0); + + /** + * The following constants come from the following line in + * https://github.com/pytorch/vision/blob/main/torchvision/models/detection/retinanet.py + * + * self.box_coder = det_utils.BoxCoder(weights=(1.0, 1.0, 1.0, 1.0)) + */ + final float WX = 1.0f; + final float WY = 1.0f; + final float WW = 1.0f; + final float WH = 1.0f; + + final float widths = boxes[2] - boxes[0]; + final float heights = boxes[3] - boxes[1]; + final float ctr_x = boxes[0] + 0.5f * widths; + final float ctr_y = boxes[1] + 0.5f * heights; + + final float dx = rel_codes[0] / WX; + final float dy = rel_codes[1] / WY; + final float dw = Math.min(rel_codes[2] / WW, BBOX_XFORM_CLIP); // This corresponds to "torch.clamp()" + final float dh = Math.min(rel_codes[3] / WH, BBOX_XFORM_CLIP); + + final float pred_ctr_x = dx * widths + ctr_x; + final float pred_ctr_y = dy * heights + ctr_y; + final float pred_w = (float) (Math.exp(dw) * widths); + final float pred_h = (float) (Math.exp(dh) * heights); + + final float c_to_c_h = 0.5f * pred_h; + final float c_to_c_w = 0.5f * pred_w; + + final float pred_boxes1 = pred_ctr_x - c_to_c_w; + final float pred_boxes2 = pred_ctr_y - c_to_c_h; + final float pred_boxes3 = pred_ctr_x + c_to_c_w; + final float pred_boxes4 = pred_ctr_y + c_to_c_h; + + // The calls to clamp() correspond to function "box_ops.clip_boxes_to_image()" + return new Rectangle( + clamp(pred_boxes1, 0, image_shape_width), + clamp(pred_boxes2, 0, image_shape_height), + clamp(pred_boxes3, 0, image_shape_width), + clamp(pred_boxes4, 0, image_shape_height)); + } + + public RetinaNet(String path) throws ModelNotFoundException, MalformedModelException, IOException { + Criteria criteria = Criteria.builder() + .setTypes(NDList.class, NDList.class) + .optModelPath(Paths.get(path)) + //.optOption("mapLocation", "true") // this model requires mapLocation for GPU + //.optTranslator(translator) + .optProgress(new ProgressBar()).build(); + + model = criteria.loadModel(); + } + + public ZooModel getModel() { + return model; + } + + public void close() { + model.close(); + } + + public List<Detection> apply(NDArray image) throws TranslateException { + if (image.getShape().dimension() != 3 || + image.getShape().get(0) != 3) { + throw new RuntimeException(); + } + + final int imageWidth = (int) image.getShape().get(2); + final int imageHeight = (int) image.getShape().get(1); + + Predictor<NDList, NDList> predictor = model.newPredictor(); + + NDList output = predictor.predict(new NDList(image)); + + if (output.size() <= 3) { + throw new IllegalArgumentException(); + } + + NDArray logits_per_image = output.get(0); + NDArray box_regression_per_image = output.get(1); + NDArray anchors_per_image = output.get(2); + + final int numberOfClasses = (int) logits_per_image.getShape().get(2); + + if (logits_per_image.getShape().dimension() != 3 || + logits_per_image.getShape().get(0) != 1 || + numberOfClasses != 2 /* "2" corresponds to a binary classification task */ || + box_regression_per_image.getShape().dimension() != 3 || + box_regression_per_image.getShape().get(0) != 1 || + box_regression_per_image.getShape().get(2) != 4 || + anchors_per_image.getShape().dimension() != 2 || + anchors_per_image.getShape().get(1) != 4) { + throw new RuntimeException(); + } + + + final int numberOfLevels = output.size() - 3; // This corresponds to "features" in Python export + + int[] num_anchors_per_level = new int[numberOfLevels]; + + { + int[] sizes = new int[numberOfLevels]; + int HW = 0; + + for (int i = 0; i < numberOfLevels; i++) { + NDArray feature = output.get(i + 3); + if (feature.getShape().dimension() != 4 || + feature.getShape().get(0) != 1) { + throw new RuntimeException(); + } + + sizes[i] = (int) (feature.getShape().get(2) * feature.getShape().get(3)); + HW += sizes[i]; + } + + final int HWA = (int) logits_per_image.getShape().get(1); + + if (HWA % HW != 0) { + throw new RuntimeException(); + } + + final int A = (int) (HWA / HW); + + for (int i = 0; i < numberOfLevels; i++) { + num_anchors_per_level[i] = sizes[i] * A; + } + } + + + int anchorsIndex[] = new int[numberOfLevels]; + int countAnchors = 0; + for (int i = 0; i < numberOfLevels; i++) { + anchorsIndex[i] = countAnchors; + countAnchors += num_anchors_per_level[i]; + } + + if (logits_per_image.getShape().get(1) != countAnchors || + box_regression_per_image.getShape().get(1) != countAnchors || + anchors_per_image.getShape().get(0) != countAnchors) { + throw new RuntimeException(); + } + + final float SCORE_THRESHOLD = 0.05f; + final int TOP_K_CANDIDATES = 1000; + + // Convert as float array, because direct access to NDArray is terribly slow + float logits_per_image_as_float[] = logits_per_image.toFloatArray(); + float box_regression_per_image_as_float[] = box_regression_per_image.toFloatArray(); + float anchors_per_image_as_float[] = anchors_per_image.toFloatArray(); + + List<Detection> detections = new LinkedList<>(); + + for (int level = 0; level < anchorsIndex.length; level++) { + float box_regression_per_level[][] = new float[num_anchors_per_level[level]][4]; + float logits_per_level[][] = new float[num_anchors_per_level[level]][2]; + float anchors_per_level[][] = new float[num_anchors_per_level[level]][4]; + + for (int i = 0; i < num_anchors_per_level[level]; i++) { + int index = anchorsIndex[level] + i; + for (int j = 0; j < 4; j++) { + box_regression_per_level[i][j] = box_regression_per_image_as_float[4 * index + j]; + } + for (int j = 0; j < 2; j++) { + logits_per_level[i][j] = logits_per_image_as_float[2 * index + j]; + } + for (int j = 0; j < 4; j++) { + anchors_per_level[i][j] = anchors_per_image_as_float[4 * index + j]; + } + } + + List<Detection> candidates = new LinkedList<>(); + for (int i = 0; i < num_anchors_per_level[level]; i++) { + for (int label = 0; label < numberOfClasses /* This is actually "2" */; label++) { + double score = sigmoid(logits_per_level[i][label]); + if (score > SCORE_THRESHOLD) { + Rectangle rectangle = decodeSingle( + new float[]{box_regression_per_level[i][0], box_regression_per_level[i][1], box_regression_per_level[i][2], box_regression_per_level[i][3]}, + new float[]{anchors_per_level[i][0], anchors_per_level[i][1], anchors_per_level[i][2], anchors_per_level[i][3]}, + imageWidth, + imageHeight); + + // This is an entry in "topk_idxs" in Python + candidates.add(new Detection(rectangle, label, score)); + } + } + } + + candidates.stream().sorted().limit(TOP_K_CANDIDATES).forEach(detection -> detections.add(detection)); + } + + + /** + * This is non-maximal suppression, which corresponds to + * function "batched_nms()", then "_batched_nms_vanilla()" in: + * https://github.com/pytorch/vision/blob/main/torchvision/ops/boxes.py + * + * Note that "iou_threshold" equals "self.nms_thresh" of the RetinaNet. + **/ + + final float NMS_THRESH = 0.3f; // Note that by default, "retinanet.py" uses 0.5 + + List<Detection> toKeep = new LinkedList<>(); + + for (int label = 0; label < numberOfClasses; label++) { + /** + * This corresponds to "torch.ops.torchvision.nms(boxes, + * scores, iou_threshold)", which is the native function + * "nms_kernel_impl()" implemented in C++: + * https://github.com/pytorch/vision/blob/main/torchvision/csrc/ops/cpu/nms_kernel.cpp + **/ + + List<Detection> tmp = new ArrayList<>(); + + for (Detection detection : detections) { + if (detection.getLabel() == label) { + tmp.add(detection); + } + } + + if (tmp.size() > 0) { + /** + * Performs non-maximum suppression (NMS) on the boxes according + * to their intersection-over-union (IoU). + * NMS iteratively removes lower scoring boxes which have an + * IoU greater than iou_threshold with another (higher scoring) + * box. + */ + Collections.sort(tmp); // Sort by decreasing scores + final Detection[] dets = tmp.toArray(new Detection[0]); + final int ndets = dets.length; + + boolean[] suppressed = new boolean[ndets]; + + for (int i = 0; i < ndets; i++) { + if (!suppressed[i]) { + toKeep.add(dets[i]); + + final float iarea = dets[i].getRectangle().getArea(); + + for (int j = i + 1; j < ndets; j++) { + if (!suppressed[j]) { + final float xx1 = Math.max(dets[i].getRectangle().getX1(), dets[j].getRectangle().getX1()); + final float yy1 = Math.max(dets[i].getRectangle().getY1(), dets[j].getRectangle().getY1()); + final float xx2 = Math.min(dets[i].getRectangle().getX2(), dets[j].getRectangle().getX2()); + final float yy2 = Math.min(dets[i].getRectangle().getY2(), dets[j].getRectangle().getY2()); + final float w = Math.max(0, xx2 - xx1); + final float h = Math.max(0, yy2 - yy1); + final float inter = w * h; + final float ovr = inter / (iarea + dets[j].getRectangle().getArea() - inter); + if (ovr > NMS_THRESH) { + suppressed[j] = true; + } + } + } + } + } + } + } + + List<Detection> toSerialize = new LinkedList<>(); + + for (Detection detection : toKeep) { + if (detection.getLabel() == 1 /* "1" is the "mass" label */ && + detection.getScore() >= 0.2f /* This is the "minimum_score=0.2" in "dicom_sr.py" */) { + toSerialize.add(detection); + } + } + + return toSerialize; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Samples/MammographyDeepLearning/src/main/resources/OrthancExplorer.js Wed Jun 12 13:58:29 2024 +0200 @@ -0,0 +1,133 @@ +/** + * SPDX-FileCopyrightText: 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * SPDX-License-Identifier: GPL-3.0-or-later + **/ + +/** + * Java plugin for Orthanc + * Copyright (C) 2023-2024 Sebastien Jodogne, UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +var mammographyButton = null; + +$('#instance').live('pagecreate', function() { + mammographyButton = $('<a>') + .attr('data-role', 'button') + .attr('href', '#') + .attr('data-icon', 'search') + .attr('data-theme', 'e') + .text('Deep learning for mammography'); + + mammographyButton.insertBefore($('#instance-delete').parent().parent()); + + mammographyButton.click(function() { + if ($.mobile.pageData) { + $.ajax({ + type: 'POST', + url: '../java-mammography-apply', + dataType: 'json', + contentType: 'application/json', + data: JSON.stringify({ + 'instance' : $.mobile.pageData.uuid, + }), + cache: false, + success: function(result) { + window.location.assign('explorer.html#series?uuid=' + result.ParentSeries); + }, + error: function() { + alert('Cannot apply deep learning model'); + } + }); + } + }); +}); + +$('#instance').live('pagebeforeshow', function() { + mammographyButton.hide(); +}); + +$('#instance').live('pageshow', function() { + $.ajax({ + url: '../instances/' + $.mobile.pageData.uuid + '/tags?simplify', + dataType: 'json', + cache: false, + success: function(tags) { + if (tags.Modality == 'MG') { + mammographyButton.show(); + } + } + }); +}); + + +$('#study').live('pagebeforecreate', function() { + var b = $('<a>') + .attr('data-role', 'button') + .attr('href', '#') + .attr('data-icon', 'search') + .attr('data-theme', 'e') + .text('Stone Web Viewer (for mammography)'); + + b.insertBefore($('#study-delete').parent().parent()); + b.click(function() { + if ($.mobile.pageData) { + $.ajax({ + url: '../studies/' + $.mobile.pageData.uuid, + dataType: 'json', + cache: false, + success: function(study) { + var studyInstanceUid = study.MainDicomTags.StudyInstanceUID; + window.open('../java-mammography-viewer/index.html?study=' + studyInstanceUid); + } + }); + } + }); +}); + + +$('#series').live('pagebeforecreate', function() { + var b = $('<a>') + .attr('data-role', 'button') + .attr('href', '#') + .attr('data-icon', 'search') + .attr('data-theme', 'e') + .text('Stone Web Viewer (for mammography)'); + + b.insertBefore($('#series-delete').parent().parent()); + b.click(function() { + if ($.mobile.pageData) { + $.ajax({ + url: '../series/' + $.mobile.pageData.uuid, + dataType: 'json', + cache: false, + success: function(series) { + $.ajax({ + url: '../studies/' + series.ParentStudy, + dataType: 'json', + cache: false, + success: function(study) { + var studyInstanceUid = study.MainDicomTags.StudyInstanceUID; + var seriesInstanceUid = series.MainDicomTags.SeriesInstanceUID; + window.open('../java-mammography-viewer/index.html?study=' + studyInstanceUid + + '&series=' + seriesInstanceUid); + } + }); + } + }); + } + }); +});