Merge remote-tracking branch 'origin/master' into tk/update-guardrails

Author: toddkazakov

Makefile

@@ -18,8 +18,8 @@ else
 	VERSION_BASE:=$(shell git describe --abbrev=0 --tags 2> /dev/null || echo 'v0.0.0')
 endif
 VERSION_BASE:=$(VERSION_BASE:v%=%)
-VERSION_SHORT_COMMIT:=$(shell git rev-parse --short HEAD)
-VERSION_FULL_COMMIT:=$(shell git rev-parse HEAD)
+VERSION_SHORT_COMMIT:=$(shell git rev-parse --short HEAD || echo "dev")
+VERSION_FULL_COMMIT:=$(shell git rev-parse HEAD || echo "dev")
 VERSION_PACKAGE:=$(REPOSITORY_PACKAGE)/application
 
 GO_BUILD_FLAGS:=-buildvcs=false
@@ -30,6 +30,13 @@ TRANSFORM_GO_BUILD_CMD:=sed 's|\.\(.*\)\(/[^/]*\)/[^/]*|_bin\1\2\2 .\1\2/.|'
 
 GO_BUILD_CMD:=go build $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -o
 
+GINKGO_FLAGS += --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r
+GINKGO_CI_WATCH_FLAGS += --randomize-all --succinct --fail-on-pending --cover --trace --race
+GINKGO_CI_FLAGS += $(GINKGO_CI_WATCH_FLAGS) --randomize-suites --keep-going
+
+GOTEST_PKGS ?= ./...
+GOTEST_FLAGS ?=
+
 ifdef TRAVIS_BRANCH
 ifdef TRAVIS_COMMIT
     DOCKER:=true
@@ -115,7 +122,7 @@ format-write-changed:
 imports: goimports
 	@echo "goimports -d -e -local 'github.com/tidepool-org/platform'"
 	@cd $(ROOT_DIRECTORY) && \
-		O=`find . -not -path './.gvm_local/*' -not -path './vendor/*' -not -path '**/test/mock.go' -not -name '**_gen.go' -name '*.go' -type f -exec goimports -d -e -local 'github.com/tidepool-org/platform' {} \; 2>&1` && \
+		O=`find . -not -path './.gvm_local/*' -not -path './vendor/*' -not -path '**/test/mock.go' -not -name '*mock.go' -not -name '**_gen.go' -name '*.go' -type f -exec goimports -d -e -local 'github.com/tidepool-org/platform' {} \; 2>&1` && \
 		[ -z "$${O}" ] || (echo "$${O}" && exit 1)
 
 imports-write: goimports
@@ -192,28 +199,35 @@ service-restart-all:
 	@cd $(ROOT_DIRECTORY) && for SERVICE in migrations tools; do $(MAKE) service-restart SERVICE="$${SERVICE}"; done
 
 test: ginkgo
-	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)
+	@echo "ginkgo $(GINKGO_FLAGS) $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo $(GINKGO_FLAGS) $(TEST)
 
 test-until-failure: ginkgo
-	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r -untilItFails $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r -untilItFails $(TEST)
+	@echo "ginkgo $(GINKGO_FLAGS) -untilItFails $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo $(GINKGO_FLAGS) -untilItFails $(TEST)
 
 test-watch: ginkgo
-	@echo "ginkgo watch --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo watch --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r $(TEST)
+	@echo "ginkgo watch $(GINKGO_FLAGS) $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo watch $(GINKGO_FLAGS) $(TEST)
 
 ci-test: ginkgo
-	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-suites --randomize-all --succinct --fail-on-pending --cover --trace --race --keep-going $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-suites --randomize-all --succinct --fail-on-pending --cover --trace --race --keep-going $(TEST)
+	@echo "ginkgo $(GINKGO_FLAGS) $(GINKGO_CI_FLAGS) $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo $(GINKGO_FLAGS) $(GINKGO_CI_FLAGS) $(TEST)
 
 ci-test-until-failure: ginkgo
-	@echo "ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-suites --randomize-all --succinct --fail-on-pending --cover --trace --race --keep-going -untilItFails $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-suites --randomize-all --succinct --fail-on-pending --cover --trace --race --keep-going -untilItFails $(TEST)
+	@echo "ginkgo $(GINKGO_FLAGS) $(GINKGO_CI_FLAGS) -untilItFails $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo $(GINKGO_FLAGS) $(GINKGO_CI_FLAGS) -untilItFails $(TEST)
 
 ci-test-watch: ginkgo
-	@echo "ginkgo watch --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-all --succinct --fail-on-pending --cover --trace --race $(TEST)"
-	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo watch --require-suite --poll-progress-after=10s --poll-progress-interval=20s -r --randomize-all --succinct --fail-on-pending --cover --trace --race $(TEST)
+	@echo "ginkgo watch $(GINKGO_FLAGS) $(GINKGO_CI_WATCH_FLAGS) $(TEST)"
+	@cd $(ROOT_DIRECTORY) && . ./env.test.sh && ginkgo watch $(GINKGO_FLAGS) $(GINKGO_CI_WATCH_FLAGS) $(TEST)
+
+go-test:
+	. ./env.test.sh && go test $(GOTEST_FLAGS) $(GOTEST_PKGS)
+
+go-ci-test: GOTEST_FLAGS += -count=1 -race -shuffle=on -cover
+go-ci-test: GOTEST_PKGS = ./...
+go-ci-test: go-test
 
 deploy: clean-deploy deploy-services deploy-migrations deploy-tools
 
@@ -340,4 +354,4 @@ gopath-implode:
 	deploy deploy-services deploy-migrations deploy-tools ci-deploy bundle-deploy \
 	docker docker-build docker-push ci-docker \
 	clean clean-bin clean-cover clean-debug clean-deploy clean-all pre-commit \
-	gopath-implode
+	gopath-implode go-test

appvalidate/appvalidate.go

@@ -0,0 +1,60 @@
+// Package appvalidate handles the logic for validating whether an app is a
+// valid instance of your app via Apple's App Attest service.
+package appvalidate
+
+import (
+	"regexp"
+	"time"
+
+	"github.com/tidepool-org/platform/structure"
+
+	structValidator "github.com/tidepool-org/platform/structure/validator"
+)
+
+//go:generate mockgen -build_flags=--mod=mod -destination=./mock.go -package=appvalidate github.com/tidepool-org/platform/appvalidate Repository,ChallengeGenerator
+
+var (
+	// base64 regex that supports base64.URLEncoding ("+/" replaced by "-_") or base64.StdEncoding. Used for base64 payloads like the attestation and assertion object.
+	base64Chars = regexp.MustCompile("^(?:[A-Za-z0-9+/\\-_]{4})*(?:[A-Za-z0-9+/\\-_]{2}==|[A-Za-z0-9+/\\-_]{3}=)?$")
+)
+
+// AppValidation represents the entire state of a person's attestation /
+// assertion status that determines if they are using a legitimate instance
+// of an iOS app.
+type AppValidation struct {
+	UserID                  string     `json:"userId" bson:"userId,omitempty"`
+	KeyID                   string     `json:"keyId" bson:"keyId,omitempty"`
+	PublicKey               string     `json:"-" bson:"publicKey,omitempty"`
+	Verified                bool       `json:"verified" bson:"verified"`
+	FraudAssessmentReceipt  string     `json:"-" bson:"fraudAssessmentReceipt,omitempty"`
+	AttestationChallenge    string     `json:"-" bson:"attestationChallenge,omitempty"`
+	AssertionVerifiedTime   *time.Time `json:"-" bson:"assertionVerifiedTime,omitempty"`
+	AssertionChallenge      string     `json:"-" bson:"assertionChallenge,omitempty"`
+	AttestationVerifiedTime *time.Time `json:"-" bson:"attestationVerifiedTime"`
+	AssertionCounter        uint32     `json:"assertionCounter" bson:"assertionCounter"`
+}
+
+// NewAppValidation creates a new AppValidation from a ChallengeCreate. Once a
+// person starts the attestation process by requesting an attestation
+// challenge, a new AppValidation needs to be persisted to keep track of the
+// progress and state of the attestation and future assertions.
+func NewAppValidation(attestChallenge string, create *ChallengeCreate) (*AppValidation, error) {
+	if err := structValidator.New().Validate(create); err != nil {
+		return nil, err
+	}
+	validation := AppValidation{
+		UserID:               create.UserID,
+		KeyID:                create.KeyID,
+		AttestationChallenge: attestChallenge,
+	}
+	if err := structValidator.New().Validate(&validation); err != nil {
+		return nil, err
+	}
+	return &validation, nil
+}
+
+func (av *AppValidation) Validate(v structure.Validator) {
+	v.String("attestationChallenge", &av.AttestationChallenge).NotEmpty()
+	v.String("userId", &av.UserID).NotEmpty()
+	v.String("keyId", &av.KeyID).NotEmpty()
+}

appvalidate/assertion.go

@@ -0,0 +1,73 @@
+package appvalidate
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/tidepool-org/platform/structure"
+
+	appAssert "github.com/bas-d/appattest/assertion"
+	appUtils "github.com/bas-d/appattest/utils"
+)
+
+// AssertionVerify is the expected request body used by clients to complete
+// the assertion process. Assertion can only be done after attestation is
+// completed. The Assertion should be the base64 encoding of the binary CBOR
+// data returned from the iOS APIs.
+type AssertionVerify struct {
+	UserID     string              `json:"-"`
+	KeyID      string              `json:"keyId"`
+	ClientData AssertionClientData `json:"clientData"`
+	Assertion  string              `json:"assertion"`
+}
+
+// AssertionUpdate contains the assertion fields to update in an AppValidation
+// to pass to a repository.
+type AssertionUpdate struct {
+	Challenge        string    `bson:"assertionChallenge,omitempty"`
+	VerifiedTime     time.Time `bson:"assertionVerifiedTime,omitempty"`
+	AssertionCounter uint32    `bson:"assertionCounter,omitempty"`
+}
+
+type AssertionResponse struct {
+	Data any `json:"data"`
+}
+
+type AssertionClientData struct {
+	Challenge   string          `json:"challenge"`
+	Partner     string          `json:"partner"`     // Which partner are we requesting a secret from
+	PartnerData json.RawMessage `json:"partnerData"` // Data to send to partner - This is a RawMessage because it is partner specific. The validation of this is delayed until later.
+}
+
+func NewAssertionVerify(userID string) *AssertionVerify {
+	return &AssertionVerify{
+		UserID: userID,
+	}
+}
+
+func (av *AssertionVerify) Validate(v structure.Validator) {
+	v.String("assertion", &av.Assertion).NotEmpty().Matches(base64Chars)
+	v.String("clientData.challenge", &av.ClientData.Challenge).NotEmpty()
+	v.String("clientData.partner", &av.ClientData.Partner).OneOf(partners...)
+
+	v.String("userId", &av.UserID).NotEmpty()
+	v.String("keyId", &av.KeyID).NotEmpty()
+}
+
+func transformAssertion(av *AssertionVerify) (*appAssert.AuthenticatorAssertionResponse, error) {
+	clientDataRaw, err := json.Marshal(av.ClientData)
+	if err != nil {
+		return nil, err
+	}
+
+	var assertion appUtils.URLEncodedBase64
+	assertionRaw := b64StdEncodingToURLEncoding(av.Assertion)
+	if err := assertion.UnmarshalJSON([]byte(assertionRaw)); err != nil {
+		return nil, err
+	}
+
+	return &appAssert.AuthenticatorAssertionResponse{
+		RawClientData: appUtils.URLEncodedBase64(clientDataRaw),
+		Assertion:     assertion,
+	}, nil
+}

appvalidate/attestation.go

@@ -0,0 +1,73 @@
+package appvalidate
+
+import (
+	"encoding/base64"
+	"time"
+
+	"github.com/tidepool-org/platform/structure"
+
+	appAttest "github.com/bas-d/appattest/attestation"
+	appUtils "github.com/bas-d/appattest/utils"
+)
+
+// AttestationVerify is the request body used to validate an app's
+// attestation. It is decoded from a JSON object.
+// https://developer.apple.com/documentation/devicecheck/establishing_your_app_s_integrity#3561588
+// KeyID and AttestationObject is data returned by the iOS APIs.
+// Attestation will be returned in CBOR format and should be base64
+// encoded before sending.
+type AttestationVerify struct {
+	Attestation string `json:"attestation"`
+	Challenge   string `json:"challenge"`
+	KeyID       string `json:"keyId"`
+	UserID      string `json:"-"`
+}
+
+func NewAttestationVerify(userID string) *AttestationVerify {
+	return &AttestationVerify{
+		UserID: userID,
+	}
+}
+
+func (av *AttestationVerify) Validate(v structure.Validator) {
+	v.String("challenge", &av.Challenge).NotEmpty()
+	v.String("attestation", &av.Attestation).Matches(base64Chars)
+	v.String("userId", &av.UserID).NotEmpty()
+	v.String("keyId", &av.KeyID).NotEmpty()
+}
+
+type AttestationUpdate struct {
+	PublicKey              string    `bson:"publicKey,omitempty"`
+	Verified               bool      `bson:"verified"`
+	FraudAssessmentReceipt string    `bson:"fraudAssessmentReceipt,omitempty"`
+	VerifiedTime           time.Time `bson:"attestationVerifiedTime"`
+}
+
+func (au *AttestationUpdate) Validate(v structure.Validator) {
+	v.String("publicKey", &au.PublicKey).NotEmpty()
+	v.String("fraudAssessmentReceipt", &au.FraudAssessmentReceipt).NotEmpty()
+	v.Time("assertionVerifiedTime", &au.VerifiedTime).NotZero()
+}
+
+func transformAttestation(av *AttestationVerify) (*appAttest.AuthenticatorAttestationResponse, error) {
+	// The appattest library expects all the data to use base64 URLEncoding when the data from IOS is base64 StdEncoding so convert first.
+
+	clientDataRaw := make([]byte, base64.RawURLEncoding.EncodedLen(len([]byte(av.Challenge))))
+	base64.RawURLEncoding.Encode(clientDataRaw, []byte(av.Challenge))
+	var clientData appUtils.URLEncodedBase64
+	if err := clientData.UnmarshalJSON(clientDataRaw); err != nil {
+		return nil, err
+	}
+
+	attestationRaw := b64StdEncodingToURLEncoding(av.Attestation)
+	var attestation appUtils.URLEncodedBase64
+	if err := attestation.UnmarshalJSON([]byte(attestationRaw)); err != nil {
+		return nil, err
+	}
+
+	return &appAttest.AuthenticatorAttestationResponse{
+		ClientData:        clientData,
+		KeyID:             av.KeyID,
+		AttestationObject: attestation,
+	}, nil
+}

appvalidate/base64.go

@@ -0,0 +1,9 @@
+package appvalidate
+
+import (
+	"strings"
+)
+
+func b64StdEncodingToURLEncoding(s string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(s, "+", "-"), "/", "_"), "=", "\"")
+}

appvalidate/challenge.go

@@ -0,0 +1,44 @@
+package appvalidate
+
+import (
+	"github.com/tidepool-org/platform/id"
+	"github.com/tidepool-org/platform/structure"
+)
+
+// ChallengeCreate is the expected request body used to create an attestation
+// or assertion challenge.
+type ChallengeCreate struct {
+	UserID string `json:"-"` // json ignored because taken from request.Details and not from user supplied input.
+	KeyID  string `json:"keyId"`
+}
+
+// ChallengeResult is the response to a successful request with
+// ChallengeCreate
+type ChallengeResult struct {
+	Challenge string `json:"challenge"`
+}
+
+func NewChallengeCreate(userID string) *ChallengeCreate {
+	return &ChallengeCreate{
+		UserID: userID,
+	}
+}
+
+func (c *ChallengeCreate) Validate(v structure.Validator) {
+	v.String("userId", &c.UserID).NotEmpty()
+	v.String("keyId", &c.KeyID).NotEmpty()
+}
+
+type ChallengeGenerator interface {
+	GenerateChallenge(size int) (string, error)
+}
+
+type challengeGenerator struct{}
+
+func NewChallengeGenerator() ChallengeGenerator {
+	return challengeGenerator{}
+}
+
+func (c challengeGenerator) GenerateChallenge(size int) (string, error) {
+	return id.New(size)
+}