Add partial parsing support (#6810)

* Some parser improvements

* Cleanup

* Work on tokenization

* Call presearch pointcut for conditional URL evaluation

* Clean up changelog

* Resolve fixmes

* Changelogs

* Add one more changelog

* Test fix

* Spotless

* Update hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_2_0/6810-call-presearch-interceptor-for-match-url-searches.yaml

Co-authored-by: tadgh <[email protected]>

* Address review comments

* Build fix

* Fix android issue

* Test fix

* Test fixes

* Bump core

* Test fixes

* Bump corelib

---------

Co-authored-by: tadgh <[email protected]>

Author: jamesagnew

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/BaseParser.java

@@ -50,6 +50,7 @@
 import ca.uhn.fhir.util.UrlUtil;
 import com.google.common.base.Charsets;
 import jakarta.annotation.Nullable;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.output.StringBuilderWriter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
@@ -295,6 +296,20 @@ public void encodeToWriter(IBase theElement, Writer theWriter) throws DataFormat
 		}
 	}
 
+	@Override
+	public void parseInto(Reader theSource, IBase theTarget) throws IOException {
+		if (theTarget instanceof IPrimitiveType) {
+			((IPrimitiveType<?>) theTarget).setValueAsString(IOUtils.toString(theSource));
+		} else {
+			doParseIntoComplexStructure(theSource, theTarget);
+		}
+	}
+
+	protected void doParseIntoComplexStructure(Reader theSource, IBase theTarget) {
+		throw new InternalErrorException(
+				Msg.code(2633) + "This " + getEncoding() + " parser does not support parsing non-resource values");
+	}
+
 	protected void encodeResourceToWriter(IBaseResource theResource, Writer theWriter, EncodeContext theEncodeContext)
 			throws IOException {
 		Validate.notNull(theResource, "theResource can not be null");

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/IParser.java

@@ -22,8 +22,10 @@
 import ca.uhn.fhir.context.ConfigurationException;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.ParserOptions;
+import ca.uhn.fhir.i18n.Msg;
 import ca.uhn.fhir.model.api.IResource;
 import ca.uhn.fhir.rest.api.EncodingEnum;
+import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import ca.uhn.fhir.util.CollectionUtil;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
@@ -35,6 +37,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Reader;
+import java.io.StringReader;
 import java.io.Writer;
 import java.util.Collection;
 import java.util.List;
@@ -512,4 +515,53 @@ default IParser setEncodeElements(@Nonnull String... theEncodeElements) {
 	 * @see ParserOptions
 	 */
 	IParser setDontStripVersionsFromReferencesAtPaths(Collection<String> thePaths);
+
+	/**
+	 * Parses an object fragment into the given structure.
+	 *
+	 * @param theSource The source value to parse and use to populate {@literal theTarget}.
+	 *                  If {@literal theTarget} is an instance of {@link org.hl7.fhir.instance.model.api.IPrimitiveType}
+	 *                  the value is treated as a simple string value. So for example, when populating
+	 *                  a {@literal DateTimeTime}, the value should resemble
+	 *                  {@literal 2020-01-01}, not {@literal <birthDate value="2020-01-01"/>}.
+	 *                  If {@literal theTarget} is a complex structure, the value should be
+	 *                  a container element suitable for the parser's encoding. So for example,
+	 *                  if the target is an {@literal Identifier}, the value would be expected
+	 *                  to resemble {@literal <identifier><system value="..."/><value value="..."/></identifier>}
+	 *                  or <code>{"system":"...", "value":"..."}</code>.
+	 * @param theTarget The target structure to populate. Note that this structure is not
+	 *                  cleared automatically by the parser, so existing values will be
+	 *                  overwritten only if {@literal theSource} has a value for the
+	 *                  given element.
+	 * @since 8.2.0
+	 */
+	default void parseInto(String theSource, IBase theTarget) {
+		try {
+			parseInto(new StringReader(theSource), theTarget);
+		} catch (IOException e) {
+			throw new InternalErrorException(
+					Msg.code(2634) + "Encountered IOException during read from - This should not happen!", e);
+		}
+	}
+
+	/**
+	 * Parses an object fragment into the given structure.
+	 *
+	 * @param theSource The source value to parse and use to populate {@literal theTarget}.
+	 *                  If {@literal theTarget} is an instance of {@link org.hl7.fhir.instance.model.api.IPrimitiveType}
+	 *                  the value is treated as a simple string value. So for example, when populating
+	 *                  a {@literal DateTimeTime}, the value should resemble
+	 *                  {@literal 2020-01-01}, not {@literal <birthDate value="2020-01-01"/>}.
+	 *                  If {@literal theTarget} is a complex structure, the value should be
+	 *                  a container element suitable for the parser's encoding. So for example,
+	 *                  if the target is an {@literal Identifier}, the value would be expected
+	 *                  to resemble {@literal <identifier><system value="..."/><value value="..."/></identifier>}
+	 *                  or <code>{"system":"...", "value":"..."}</code>.
+	 * @param theTarget The target structure to populate. Note that this structure is not
+	 *                  cleared automatically by the parser, so existing values will be
+	 *                  overwritten only if {@literal theSource} has a value for the
+	 *                  given element.
+	 * @since 8.2.0
+	 */
+	void parseInto(Reader theSource, IBase theTarget) throws IOException;
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/JsonParser.java

@@ -236,6 +236,20 @@ protected void doEncodeToWriter(IBase theElement, Writer theWriter, EncodeContex
 		eventWriter.close();
 	}
 
+	@Override
+	protected void doParseIntoComplexStructure(Reader theSource, IBase theTarget) {
+		JsonLikeStructure jsonStructure = new JacksonStructure();
+		jsonStructure.load(theSource);
+
+		ParserState<IBase> state =
+				ParserState.getComplexObjectState(this, getContext(), getContext(), true, theTarget, getErrorHandler());
+		state.enteringNewElement(null, null);
+
+		parseChildren(jsonStructure.getRootObject(), state);
+
+		state.endingElement();
+	}
+
 	@Override
 	public <T extends IBaseResource> T doParseResource(Class<T> theResourceType, Reader theReader) {
 		JsonLikeStructure jsonStructure = new JacksonStructure();

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/ParserState.java

@@ -52,6 +52,7 @@
 import ca.uhn.fhir.util.FhirTerser;
 import ca.uhn.fhir.util.ReflectionUtil;
 import ca.uhn.fhir.util.XmlUtil;
+import jakarta.annotation.Nullable;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.math.NumberUtils;
@@ -72,6 +73,7 @@
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -203,15 +205,17 @@ public IBase newInstance(RuntimeChildDeclaredExtensionDefinition theDefinition)
 	}
 
 	public ICompositeType newCompositeInstance(
-			BasePreResourceState thePreResourceState,
+			@Nullable BasePreResourceState thePreResourceState,
 			BaseRuntimeChildDefinition theChild,
 			BaseRuntimeElementCompositeDefinition<?> theCompositeTarget) {
 		ICompositeType retVal =
 				(ICompositeType) theCompositeTarget.newInstance(theChild.getInstanceConstructorArguments());
 		if (retVal instanceof IBaseReference) {
 			IBaseReference ref = (IBaseReference) retVal;
 			myGlobalReferences.add(ref);
-			thePreResourceState.getLocalReferences().add(ref);
+			if (thePreResourceState != null) {
+				thePreResourceState.getLocalReferences().add(ref);
+			}
 		}
 		return retVal;
 	}
@@ -552,6 +556,36 @@ protected IBase getCurrentElement() {
 		}
 	}
 
+	/**
+	 * This state represents the start of parsing an arbitrary fragment
+	 * of a FHIR resource. In other words, it is the initial state when
+	 * {@link IParser#parseInto(Reader, IBase)} is called.
+	 */
+	private class PreElementCompositeState extends BaseState {
+
+		private final String myElementName;
+		private final BaseRuntimeElementCompositeDefinition<?> myDef;
+		private final IBase myInstance;
+
+		PreElementCompositeState(
+				String theElementName, BaseRuntimeElementCompositeDefinition<?> theDef, IBase theInstance) {
+			super(null);
+			myElementName = theElementName;
+			myDef = theDef;
+			myInstance = theInstance;
+		}
+
+		@Override
+		public void enteringNewElement(String theNamespaceUri, String theLocalPart) throws DataFormatException {
+			push(new ElementCompositeState(null, myElementName, myDef, myInstance));
+		}
+
+		@Override
+		public void endingElement() {
+			pop();
+		}
+	}
+
 	private class ElementCompositeState extends BaseState {
 
 		private final BaseRuntimeElementCompositeDefinition<?> myDefinition;
@@ -1769,6 +1803,21 @@ public void doPop() {
 		}
 	}
 
+	static ParserState<IBase> getComplexObjectState(
+			IParser theParser,
+			FhirContext theContext,
+			FhirContext theFhirContext,
+			boolean theJsonMode,
+			IBase theTarget,
+			IParserErrorHandler theErrorHandler) {
+		ParserState<IBase> retVal = new ParserState<>(theParser, theContext, theJsonMode, theErrorHandler);
+
+		BaseRuntimeElementCompositeDefinition<?> def =
+				(BaseRuntimeElementCompositeDefinition<?>) theFhirContext.getElementDefinition(theTarget.getClass());
+		retVal.push(retVal.new PreElementCompositeState(def.getName(), def, theTarget));
+		return retVal;
+	}
+
 	/**
 	 * @param theResourceType May be null
 	 */
@@ -1779,7 +1828,7 @@ static <T extends IBaseResource> ParserState<T> getPreResourceInstance(
 			boolean theJsonMode,
 			IParserErrorHandler theErrorHandler)
 			throws DataFormatException {
-		ParserState<T> retVal = new ParserState<T>(theParser, theContext, theJsonMode, theErrorHandler);
+		ParserState<T> retVal = new ParserState<>(theParser, theContext, theJsonMode, theErrorHandler);
 		if (theResourceType == null) {
 			if (theContext.getVersion().getVersion().isRi()) {
 				retVal.push(retVal.new PreResourceStateHl7Org(theResourceType));

hapi-fhir-base/src/main/java/ca/uhn/fhir/parser/XmlParser.java

@@ -58,7 +58,6 @@
 import org.hl7.fhir.instance.model.api.IIdType;
 import org.hl7.fhir.instance.model.api.IPrimitiveType;
 
-import java.io.IOException;
 import java.io.Reader;
 import java.io.Writer;
 import java.util.ArrayList;
@@ -144,7 +143,7 @@ public void doEncodeResourceToWriter(IBaseResource theResource, Writer theWriter
 
 	@Override
 	protected void doEncodeToWriter(IBase theElement, Writer theWriter, EncodeContext theEncodeContext)
-			throws IOException, DataFormatException {
+			throws DataFormatException {
 		XMLStreamWriter eventWriter;
 		try {
 			eventWriter = createXmlWriter(theWriter);
@@ -165,6 +164,16 @@ protected void doEncodeToWriter(IBase theElement, Writer theWriter, EncodeContex
 		}
 	}
 
+	@Override
+	protected void doParseIntoComplexStructure(Reader theSource, IBase theTarget) {
+		XMLEventReader streamReader = createStreamReader(theSource);
+
+		ParserState<IBase> state = ParserState.getComplexObjectState(
+				this, getContext(), getContext(), false, theTarget, getErrorHandler());
+
+		doXmlLoop(streamReader, state);
+	}
+
 	@Override
 	public <T extends IBaseResource> T doParseResource(Class<T> theResourceType, Reader theReader) {
 		XMLEventReader streamReader = createStreamReader(theReader);
@@ -265,7 +274,7 @@ private <T> T doXmlLoop(XMLEventReader streamReader, ParserState<T> parserState)
 			}
 			return parserState.getObject();
 		} catch (XMLStreamException e) {
-			throw new DataFormatException(Msg.code(1852) + e);
+			throw new DataFormatException(Msg.code(1852) + "Failed to parse XML content: " + e.getMessage());
 		}
 	}
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/LogicUtil.java renamed to hapi-fhir-base/src/main/java/ca/uhn/fhir/util/LogicUtil.java

@@ -1,6 +1,6 @@
 /*
  * #%L
- * HAPI FHIR JPA Server
+ * HAPI FHIR - Core Library
  * %%
  * Copyright (C) 2014 - 2025 Smile CDR, Inc.
  * %%
@@ -17,9 +17,7 @@
  * limitations under the License.
  * #L%
  */
-package ca.uhn.fhir.jpa.util;
-
-import ca.uhn.fhir.util.CoverageIgnore;
+package ca.uhn.fhir.util;
 
 public class LogicUtil {
 

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ResourceUtil.java

@@ -32,7 +32,7 @@
 public class ResourceUtil {
 
 	private static final String ENCODING = "ENCODING_TYPE";
-	private static final String RAW_ = "RAW_%s";
+	private static final String RAW_ = "RAW_";
 
 	private ResourceUtil() {}
 
@@ -77,6 +77,6 @@ public static String getRawStringFromResourceOrNull(@Nonnull IBaseResource theRe
 	}
 
 	private static String getRawUserDataKey(EncodingEnum theEncodingEnum) {
-		return String.format(RAW_, theEncodingEnum.name());
+		return RAW_ + theEncodingEnum.name();
 	}
 }

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_0_0/6469-reduce-memory-overhead-for-searches.yaml

@@ -3,6 +3,5 @@ type: perf
 issue: 6469
 title: "Searching for a large number of resources can use a lot of
         memory, due to the nature of deduplication of results in memory.
-        We will instead push this responsibility to the db to save
-        reduce this overhead.
+        We will instead push this responsibility to the db to reduce this overhead.
 "

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_0_0/6589_optimize_transactions_in_mass_ingestion_mode renamed to hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/8_0_0/6589-optimize_transactions_in_mass_ingestion_mode.yaml

@@ -0,0 +1,7 @@
+---
+type: add
+issue: 6810
+title: "The Pointcut `STORAGE_PRESEARCH_REGISTERED` will now also be called before the 
+   internal FHIR searches being performed in order to resolve conditional create/update/etc
+   URLs when processing FHIR transactions. Previously this pointcut was not called for
+   these specific searches."