Optimize the JSON parsing in NpmPackageIndexBuilder.seeFile

hapifhir · Feb 5, 2025 · 544c208 · 544c208
1 parent e67fbb9
commit 544c208
Showing 1 changed file with 51 additions and 51 deletions.
diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackageIndexBuilder.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackageIndexBuilder.java
@@ -7,11 +7,10 @@
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.HashSet;
-import java.util.Set;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonToken;
 import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
 import org.hl7.fhir.utilities.FileUtilities;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
@@ -77,57 +76,58 @@ public void start(String filename) {
 
   public boolean seeFile(String name, byte[] content) {
     if (name.endsWith(".json")) {
-      try {
-        JsonObject json = JsonParser.parseObject(content);
-        if (json.has("resourceType")) {
-          // ok we treat it as a resource
-          JsonObject fi = new JsonObject();
-          files.add(fi);
-          fi.add("filename", name);
-          fi.add("resourceType", json.asString("resourceType")); 
-          if (json.hasPrimitive("id")) {
-            fi.add("id", json.asString("id"));
-          }
-          if (json.hasPrimitive("url")) {
-            fi.add("url", json.asString("url"));
-          }
-          if (json.hasPrimitive("version")) {
-            fi.add("version", json.asString("version"));
-          }
-          if (json.hasPrimitive("kind")) {
-            fi.add("kind", json.asString("kind"));
-          }
-          if (json.hasPrimitive("type")) {
-            fi.add("type", json.asString("type"));
-          }
-          if (json.hasPrimitive("supplements")) {
-            fi.add("supplements", json.asString("supplements"));
-          }
-          if (json.hasPrimitive("content")) {
-            fi.add("content", json.asString("content"));
-          }
-          if (json.hasPrimitive("valueSet")) {
-            fi.add("valueSet", json.asString("valueSet"));
-          }
-          if (json.hasPrimitive("derivation")) {
-            fi.add("derivation", json.asString("derivation"));
+      /*
+       * We are only interested in some String fields on the first level of the JSON file.
+       * We can then use a streaming parser to get the values of these fields instead of parsing the whole file and
+       * allocating memory for everything in it.
+       * This relies on the fact that the key 'resourceType' will happen before all other keys (which should be the
+       * case in all FHIR resources).
+       * Otherwise, the parser should go over the file content twice: one to detect that key, and, if the key is
+       * found, a second time to get the values of the other keys.
+       */
+      try (final var parser = new JsonFactory().createParser(content)) {
+        JsonObject fi = null;
+
+        while (parser.nextToken() != JsonToken.END_OBJECT) {
+          String fieldName = parser.currentName();
+
+          if ("resourceType".equals(fieldName)) {
+            parser.nextToken();
+            fi = new JsonObject();
+            files.add(fi);
+            fi.add("filename", name);
+            fi.add(fieldName, parser.getText());
           }
-
-          if (psql != null) {
-            psql.setString(1, name); // FileName); 
-            psql.setString(2, json.asString("resourceType")); // ResourceType"); 
-            psql.setString(3, json.asString("id")); // Id"); 
-            psql.setString(4, json.asString("url")); // Url"); 
-            psql.setString(5, json.asString("version")); // Version"); 
-            psql.setString(6, json.asString("kind")); // Kind");
-            psql.setString(7, json.asString("type")); // Type"); 
-            psql.setString(8, json.asString("supplements")); // Supplements"); 
-            psql.setString(9, json.asString("content")); // Content");
-            psql.setString(10, json.asString("valueSet")); // ValueSet");
-            psql.setString(10, json.asString("derivation")); // ValueSet");
-            psql.execute();
+
+          if (fi != null && ("id".equals(fieldName)
+              || "url".equals(fieldName)
+              || "version".equals(fieldName)
+              || "kind".equals(fieldName)
+              || "type".equals(fieldName)
+              || "supplements".equals(fieldName)
+              || "content".equals(fieldName)
+              || "valueSet".equals(fieldName)
+              || "derivation".equals(fieldName))
+          ) {
+            parser.nextToken();
+            fi.add(fieldName, parser.getText());
           }
         }
+
+        if (fi != null && psql != null) {
+          psql.setString(1, name); // FileName);
+          psql.setString(2, fi.asString("resourceType")); // ResourceType");
+          psql.setString(3, fi.asString("id")); // Id");
+          psql.setString(4, fi.asString("url")); // Url");
+          psql.setString(5, fi.asString("version")); // Version");
+          psql.setString(6, fi.asString("kind")); // Kind");
+          psql.setString(7, fi.asString("type")); // Type");
+          psql.setString(8, fi.asString("supplements")); // Supplements");
+          psql.setString(9, fi.asString("content")); // Content");
+          psql.setString(10, fi.asString("valueSet")); // ValueSet");
+          psql.setString(10, fi.asString("derivation")); // Derivation");
+          psql.execute();
+        }
       } catch (Exception e) {
 //        System.out.println("Error parsing "+name+": "+e.getMessage());
         if (name.contains("openapi")) {