BCDA-9021: Add _typeFilter Param to Demo (#1143)

## 🎫 Ticket

https://jira.cms.gov/browse/BCDA-9021

## 🛠 Changes

This PR:
* updates the BCDA `/demo` API to point to BFD V2. (temporary until v3
is released)
* adds the _typeFilter parameter for the `/demo` API. (while we are
pointing at BFD v2, only the service-date subquery on the EOB resource
will work)

## ℹ️ Context

For Connectathon, we want to get a demo api up and running for people to
test against. This way we can preview BFD V3 and the _typeFilter
parameter

## 🧪 Validation

### Test 1: /demo with _typefilter
Make an $export request to the `/demo` API and include the _typeFilter
parameter for service-date (example:
`/api/demo/Patient/$export?_typeFilter=ExplanationOfBenefit%3Fservice-date%3Dgt2001-07-01`)
- When you pull the EOB file, make sure only EOBs from the specified
date range are included (ex above: after July 1, 2001)

### Test 2: demo without _typefilter
Make an $export request to the `/demo` API and do not include any
_typeFilter param:
 - When you pull the EOB file, there should not be any extra filtering

### Test 3: demo with invalid _typefilter
Make an $export request to the `/demo` API and include an invalid
_typeFilter parameter
- ([examples of bad query
parameters](https://github.com/CMSgov/bcda-app/blob/2b8a9aace01f9f563ab0ef3c3a7a3a521c60fdc9/bcda/web/middleware/validation_test.go#L62-L67)):
- `_typeFilter=MedicationRequest%%3Fstatus%%3Dactive` - invalid resource
type
- `_typeFilter=ExplanationOfBenefit%%3Fservice-dateactive` - invalid
parameter/value
- ` _typeFilter=ExplanationOfBenefit%%3Fstatus%%3Dactive` - invalid
subquery parameter
- When you make the $export request, you should see an OperationOutcome
response with an appropriate error

### Test 4: v2 with (ignored) _typefilter
[*regression*] Make an $export request to the `/v2` API and include the
_typeFilter parameter for service-date (example:
`/api/demo/Patient/$export?_typeFilter=ExplanationOfBenefit%3Fservice-date%3Dgt2001-07-01`)
 - When you pull the EOB file, there should not be any extra filtering

Author: colby-seyferth-nava

bcda/api/requests.go

@@ -582,6 +582,7 @@ func (h *Handler) bulkRequest(w http.ResponseWriter, r *http.Request, reqType co
 		ComplexDataRequestType: complexDataRequestType,
 		ResourceTypes:          resourceTypes,
 		Since:                  rp.Since,
+		TypeFilter:             rp.TypeFilter,
 		CreationTime:           time.Now(),
 		ClaimsDate:             timeConstraints.ClaimsDate,
 		OptOutDate:             timeConstraints.OptOutDate,

bcda/api/v3/api.go

@@ -291,7 +291,7 @@ func Metadata(w http.ResponseWriter, r *http.Request) {
 		Publisher: &fhirdatatypes.String{Value: "Centers for Medicare & Medicaid Services"},
 		Kind:      &fhircapabilitystatement.CapabilityStatement_KindCode{Value: fhircodes.CapabilityStatementKindCode_INSTANCE},
 		Instantiates: []*fhirdatatypes.Canonical{
-			{Value: bbServer + "/v3/fhir/metadata"},
+			{Value: bbServer + constants.BFDV3Path + "/metadata"},
 			{Value: "http://hl7.org/fhir/uv/bulkdata/CapabilityStatement/bulk-data"},
 		},
 		Software: &fhircapabilitystatement.CapabilityStatement_Software{

bcda/client/bluebutton.go

@@ -226,6 +226,7 @@ func (bbc *BlueButtonClient) GetExplanationOfBenefit(jobData worker_types.JobEnq
 
 	updateParamWithServiceDate(&params, claimsWindow)
 	updateParamWithLastUpdated(&params, jobData.Since, jobData.TransactionTime)
+	updateParamWithTypeFilter(&params, jobData.TypeFilter)
 
 	u, err := bbc.getURL("ExplanationOfBenefit", params)
 	if err != nil {
@@ -423,6 +424,12 @@ func updateParamWithLastUpdated(params *url.Values, since string, transactionTim
 	}
 }
 
+func updateParamWithTypeFilter(params *url.Values, typeFilter [][]string) {
+	for _, paramPair := range typeFilter {
+		params.Add(paramPair[0], paramPair[1])
+	}
+}
+
 func updateParamsWithClaimsDefaults(params *url.Values, mbi string) {
 	params.Set("excludeSAMHSA", "true")
 	params.Set("includeTaxNumbers", "true")

bcda/client/bluebutton_test.go

@@ -365,6 +365,7 @@ func (s *BBRequestTestSuite) TearDownAllSuite() {
 func (s *BBRequestTestSuite) TestValidateRequest() {
 	old := conf.GetEnv("BB_CLIENT_PAGE_SIZE")
 	jobDataNoSince := worker_types.JobEnqueueArgs{ID: 1, CMSID: "A0000", Since: "", TransactionTime: now}
+	jobDataWithTypeFilter := worker_types.JobEnqueueArgs{ID: 1, CMSID: "A0000", Since: "gt2020-02-14", TypeFilter: [][]string{{"service-date", "gt2022-06-26"}}, TransactionTime: now}
 	defer conf.SetEnv(s.T(), "BB_CLIENT_PAGE_SIZE", old)
 	conf.SetEnv(s.T(), "BB_CLIENT_PAGE_SIZE", "0") // Need to ensure that requests do not have the _count parameter
 
@@ -743,6 +744,25 @@ func (s *BBRequestTestSuite) TestValidateRequest() {
 				hasClaimRequiredURLEncodedBody,
 			},
 		},
+		{
+			"GetExplanationOfBenefitWithTypeFilterServiceDate",
+			func(bbClient *client.BlueButtonClient) (interface{}, error) {
+				return bbClient.GetExplanationOfBenefit(jobDataWithTypeFilter, "patient1", client.ClaimsWindow{})
+			},
+			func(t *testing.T, payload interface{}) {
+				result, ok := payload.(*fhirModels.Bundle)
+				assert.True(t, ok)
+				assert.NotEmpty(t, result.Entries)
+			},
+			[]func(*testing.T, *http.Request){
+				sinceChecker,
+				nowChecker,
+				excludeSAMHSAChecker,
+				ServiceDateChecker,
+				hasDefaultRequestHeaders,
+				hasBulkRequestHeaders,
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -860,6 +880,9 @@ func nowChecker(t *testing.T, req *http.Request) {
 func noServiceDateChecker(t *testing.T, req *http.Request) {
 	assert.Empty(t, req.URL.Query()[constants.TestSvcDate])
 }
+func ServiceDateChecker(t *testing.T, req *http.Request) {
+	assert.Contains(t, req.URL.String(), "service-date=gt2022-06-26")
+}
 func serviceDateUpperBoundChecker(t *testing.T, req *http.Request) {
 	// We expect that service date only contains YYYY-MM-DD
 	assert.Contains(t, req.URL.Query()[constants.TestSvcDate], fmt.Sprintf("le%s", claimsDate.UpperBound.Format(constants.TestSvcDateResult)))

bcda/constants/constants.go

@@ -49,7 +49,7 @@ const CCLF8FileNum = int(8)
 
 const BFDV1Path = "/v1/fhir"
 const BFDV2Path = "/v2/fhir"
-const BFDV3Path = "/v3/fhir"
+const BFDV3Path = "/v2/fhir" // TODO: V3
 const V3Version = "demo"
 
 const GetExistingBenes = "GetExistingBenes"

bcda/service/service.go

@@ -320,6 +320,7 @@ func (s *service) createQueueJobs(ctx context.Context, args worker_types.Prepare
 								BeneficiaryIDs:  jobIDs,
 								ResourceType:    rt,
 								Since:           sinceArg,
+								TypeFilter:      args.TypeFilter,
 								TransactionID:   ctx.Value(middleware.CtxTransactionKey).(string),
 								TransactionTime: transactionTime,
 								BBBasePath:      args.BFDPath,

bcda/web/middleware/validation.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"net/url"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -24,7 +26,7 @@ type RequestParameters struct {
 	ResourceTypes []string
 	Version       string // e.g. v1, v2
 	RequestURL    string
-	// TypeFilter    string
+	TypeFilter    [][]string
 }
 
 // const BBSystemURL = "https://bluebutton.cms.gov/fhir/CodeSystem/Adjudication-Status"
@@ -126,26 +128,60 @@ func ValidateRequestURL(next http.Handler) http.Handler {
 			rp.ResourceTypes = resourceTypes
 		}
 
-		// TODO: V3 _typeFilter
-		// params, ok = r.URL.Query()["_typeFilter"]
-		// if ok {
-		// 	allowedQueryParams := []string{
-		// 		"ExplanationOfBenefit?",
-		// 		"ExplanationOfBenefit?tag=PartiallyAdjudicated",
-		// 		"ExplanationOfBenefit?_tag=Adjudicated",
-		// 		"ExplanationOfBenefit?_tag=PartiallyAdjudicated,Adjudicated",
-		// 		"ExplanationOfBenefit?_source=FISS",
-		// 		"ExplanationOfBenefit?_source=MCS",
-		// 	}
-		// 	if utils.ContainsString(allowedQueryParams, params[0]) {
-		// 		rp.TypeFilter = params[0]
-		// 	} else {
-		// 		errMsg := fmt.Sprintf("invalid _typeFilter (or currently unimplemented query) %s", params[0])
-		// 		log.API.Error(errMsg)
-		// 		rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
-		// 		return
-		// 	}
-		// }
+		// validate _typeFilter params
+		params, ok = r.URL.Query()["_typeFilter"]
+		if (version == "demo") && ok {
+			var typeFilterParams [][]string
+			for _, subQuery := range params {
+
+				// The subquery is url-encoded. So we will first decode so we can parse it
+				decodedQuery, err := url.QueryUnescape(subQuery)
+				if err != nil {
+					errMsg := fmt.Sprintf("failed to unescape %s", subQuery)
+					log.API.Error(errMsg)
+					rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
+					return
+				}
+
+				// Expected format is: <resourceType>?<paramList>
+				resourceType, queryParams, ok := strings.Cut(decodedQuery, "?")
+				if !ok {
+					errMsg := fmt.Sprintf("missing question mark %s", decodedQuery)
+					log.API.Error(errMsg)
+					rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
+				}
+
+				// Right now, we are only accepting ExplanationOfBenefit subqueries
+				if resourceType != "ExplanationOfBenefit" {
+					errMsg := fmt.Sprintf("Invalid _typeFilter Resource Type (Only EOBs valid): %s", resourceType)
+					log.API.Error(errMsg)
+					rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
+				}
+
+				// Loop through the param list from the subquery
+				paramAry := strings.Split(queryParams, "&")
+				for _, paramPair := range paramAry {
+
+					paramName, paramValue, ok := strings.Cut(paramPair, "=")
+					if !ok {
+						errMsg := fmt.Sprintf("Invalid _typeFilter parameter/value: %s", paramPair)
+						log.API.Error(errMsg)
+						rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
+					}
+
+					if slices.Contains([]string{"service-date", "_tag", "_profile"}, paramName) {
+						typeFilterParams = append(typeFilterParams, []string{paramName, paramValue})
+					} else {
+						errMsg := fmt.Sprintf("Invalid _typeFilter subquery parameter: %s", paramName)
+						log.API.Error(errMsg)
+						rw.Exception(r.Context(), w, http.StatusBadRequest, responseutils.RequestErr, errMsg)
+						return
+					}
+				}
+
+				rp.TypeFilter = typeFilterParams
+			}
+		}
 
 		ctx := SetRequestParamsCtx(r.Context(), rp)
 

bcda/web/middleware/validation_test.go

@@ -27,7 +27,7 @@ func TestValidRequestURL(t *testing.T) {
 
 	now := time.Now().Add(-24 * time.Hour).Round(time.Millisecond)
 	req, err := http.NewRequest("GET",
-		fmt.Sprintf("/api/v1/Patient/$export?_type=Patient&_since=%s&_outputFormat=ndjson",
+		fmt.Sprintf("/api/v1/Patient/$export?_type=Patient&_since=%s&_outputFormat=ndjson&_typeFilter=ExplanationOfBenefit%%3Fservice-date%%3Dgt2001-04-01",
 			now.Format(time.RFC3339Nano)),
 		nil)
 	assert.NoError(t, err)
@@ -46,6 +46,7 @@ func TestValidRequestURL(t *testing.T) {
 func TestInvalidRequestURL(t *testing.T) {
 
 	base := "/api/v1/Patient/$export?"
+	baseV3 := constants.V3Path + "Patient/$export?"
 	tests := []struct {
 		name   string
 		url    string
@@ -59,6 +60,12 @@ func TestInvalidRequestURL(t *testing.T) {
 			"Date must be a date that has already passed"},
 		{"repeatedType", fmt.Sprintf("%s_type=Patient,Patient", base), "Repeated resource type Patient"},
 		{"noVersion", "/api/Patient$export", "cannot retrieve version"},
+		{"invalidTypeFilterResourceType", fmt.Sprintf("%s_typeFilter=MedicationRequest%%3Fstatus%%3Dactive", baseV3),
+			"Invalid _typeFilter Resource Type (Only EOBs valid): MedicationRequest"},
+		{"invalidTypeFilterSubquery", fmt.Sprintf("%s_typeFilter=ExplanationOfBenefit%%3Fservice-dateactive", baseV3),
+			"Invalid _typeFilter parameter/value: service-dateactive"},
+		{"invalidTypeFilterSubqueryParam", fmt.Sprintf("%s_typeFilter=ExplanationOfBenefit%%3Fstatus%%3Dactive", baseV3),
+			"Invalid _typeFilter subquery parameter: status"},
 	}
 
 	for _, tt := range tests {

bcdaworker/queueing/worker_prepare.go

@@ -122,6 +122,7 @@ func (p *PrepareJobWorker) prepareExportJobs(ctx context.Context, args worker_ty
 		ID:              id,
 		ACOID:           args.Job.ACOID.String(),
 		Since:           args.Since.String(),
+		TypeFilter:      args.TypeFilter,
 		TransactionTime: time.Now(),
 		CMSID:           args.CMSID,
 	}

bcdaworker/queueing/worker_types/prepare_types.go

@@ -21,6 +21,7 @@ type PrepareJobArgs struct {
 	ComplexDataRequestType string
 	ResourceTypes          []string
 	Since                  time.Time
+	TypeFilter             [][]string
 	CreationTime           time.Time
 	ClaimsDate             time.Time
 	OptOutDate             time.Time