Merge pull request #528 from quickfixgo/fix-persist-seqnum

Resolves bug related to persisting and incrementing sequence numbers, upgrades mongodb to require replica sets, improves project docs and dev process

Author: ackleymi

.devcontainer/docker-compose.yml

@@ -21,17 +21,15 @@ services:
     # (Adding the "ports" property to this file will not forward from a Codespace.)
 
   db:
-    image: mongo:latest
+    image: bitnami/mongodb:latest
     restart: unless-stopped
     volumes:
       - mongodb-data:/data/db
     ports:
        - 27017:27017
-    # Uncomment to change startup options
-    # environment:
-    #  MONGO_INITDB_ROOT_USERNAME: root
-    #  MONGO_INITDB_ROOT_PASSWORD: example
-    #  MONGO_INITDB_DATABASE: your-database-here
+    environment:
+      MONGODB_REPLICA_SET_MODE: primary
+      ALLOW_EMPTY_PASSWORD: 'yes'
 
     # Add "forwardPorts": ["27017"] to **devcontainer.json** to forward MongoDB locally.
     # (Adding the "ports" property to this file will not forward from a Codespace.)

.github/workflows/ci.yaml

@@ -37,8 +37,8 @@ jobs:
         env:
           GOLANGCI_LINT_VERSION: '1.46.2'
           GOLANGCI_LINT_SHA256: '242cd4f2d6ac0556e315192e8555784d13da5d1874e51304711570769c4f2b9b'
-      - name: Test style
-        run: make test-style
+      - name: Run Lint
+        run: make lint
 
   build:
     name: build
@@ -65,6 +65,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Start MongoDB
         uses: supercharge/[email protected]
+        with:
+          mongodb-replica-set: replicaset
       - name: Install ruby
         uses: ruby/setup-ruby@v1
         with:

.golangci.yml

@@ -7,7 +7,6 @@ run:
 linters:
   disable-all: true
   enable:
-    - deadcode
     - dupl
     - gofmt
     - goimports
@@ -17,7 +16,6 @@ linters:
     - misspell
     - revive
     - unused
-    - varcheck
     - staticcheck
 
 linters-settings:

Makefile

@@ -1,25 +1,3 @@
-GOBIN         = $(shell go env GOBIN)
-ifeq ($(GOBIN),)
-GOBIN         = $(shell go env GOPATH)/bin
-endif
-GOX           = $(GOBIN)/gox
-GOIMPORTS     = $(GOBIN)/goimports
-ARCH          = $(shell uname -p)
-
-# ------------------------------------------------------------------------------
-#  dependencies
-
-# If go install is run from inside the project directory it will add the
-# dependencies to the go.mod file. To avoid that we change to a directory
-# without a go.mod file when downloading the following dependencies
-
-$(GOX):
-	(cd /; GO111MODULE=on go install github.com/mitchellh/gox@latest)
-
-$(GOIMPORTS):
-	(cd /; GO111MODULE=on go install golang.org/x/tools/cmd/goimports@latest)
-
-# ------------------------------------------------------------------------------
 
 all: vet test
 
@@ -29,39 +7,29 @@ clean:
 generate: clean
 	mkdir -p gen; cd gen; go run ../cmd/generate-fix/generate-fix.go -pkg-root=github.com/quickfixgo/quickfix/gen ../spec/*.xml
 
-generate-dist:
-	cd ..; go run quickfix/cmd/generate-fix/generate-fix.go quickfix/spec/*.xml
-
-test-style:
-	GO111MODULE=on golangci-lint run
-
-.PHONY: format
-format: $(GOIMPORTS)
-	GO111MODULE=on go list -f '{{.Dir}}' ./... | xargs $(GOIMPORTS) -w -local github.com/quickfixgo/quickfix
-
 fmt:
-	go fmt `go list ./... | grep -v quickfix/gen`
+	gofmt -l -w -s $(shell find . -type f -name '*.go')
 
 vet:
 	go vet `go list ./... | grep -v quickfix/gen`
 
 test: 
 	MONGODB_TEST_CXN=mongodb://db:27017 go test -v -cover . ./datadictionary ./internal
 
-build-src: 
-	go build -v `go list ./...`
-
-build-test-srv: 
-	cd _test; go build -o echo_server ./test-server/
+linters-install:
+	@golangci-lint --version >/dev/null 2>&1 || { \
+		echo "installing linting tools..."; \
+		curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.46.2; \
+	}
 
-build: build-src build-test-srv
-
-test-ci: 
-	go test -v -cover . ./datadictionary ./internal
+lint: linters-install
+	golangci-lint run
 
-generate-ci: clean
-	mkdir -p gen; cd gen; go run ../cmd/generate-fix/generate-fix.go -pkg-root=github.com/quickfixgo/quickfix/gen ../spec/$(shell echo $(FIX_TEST) | tr  '[:lower:]' '[:upper:]').xml; 
+# ---------------------------------------------------------------
+# Targets related to running acceptance tests -
 
+build-test-srv:
+	cd _test; go build -o echo_server ./test-server/
 fix40:
 	cd _test; ./runat.sh $@.cfg 5001 "definitions/server/$@/*.def"
 fix41:
@@ -83,3 +51,20 @@ ACCEPT_SUITE=fix40 fix41 fix42 fix43 fix44 fix50 fix50sp1 fix50sp2
 accept: $(ACCEPT_SUITE)
 
 .PHONY: test $(ACCEPT_SUITE)
+# ---------------------------------------------------------------
+
+# ---------------------------------------------------------------
+# These targets are specific to the Github CI Runner -
+
+build-src:
+	go build -v `go list ./...`
+
+build: build-src build-test-srv
+
+test-ci:
+	go test -v -cover . ./datadictionary ./internal
+
+generate-ci: clean
+	mkdir -p gen; cd gen; go run ../cmd/generate-fix/generate-fix.go -pkg-root=github.com/quickfixgo/quickfix/gen ../spec/$(shell echo $(FIX_TEST) | tr  '[:lower:]' '[:upper:]').xml; 
+
+# ---------------------------------------------------------------

README.md

@@ -1,5 +1,4 @@
-QuickFIX/Go
-===========
+# QuickFIX/Go
 
 [![Build Status](https://github.com/quickfixgo/quickfix/workflows/CI/badge.svg)](https://github.com/quickfixgo/quickfix/actions) [![GoDoc](https://godoc.org/github.com/quickfixgo/quickfix?status.png)](https://godoc.org/github.com/quickfixgo/quickfix) [![Go Report Card](https://goreportcard.com/badge/github.com/quickfixgo/quickfix)](https://goreportcard.com/report/github.com/quickfixgo/quickfix)
 
@@ -8,29 +7,28 @@ QuickFIX/Go
 
 Open Source [FIX Protocol](http://www.fixprotocol.org/) library implemented in Go
 
-Getting Started and Documentation
----------------------------------
+## Getting Started and Documentation
 
 * [User Manual](http://quickfixgo.org/docs)
 * [API Documentation](https://godoc.org/github.com/quickfixgo/quickfix)
 
-### Installation
+## Installation
 
 To install QuickFIX/Go, use `go get`:
 
 ```sh
-$ go get github.com/quickfixgo/quickfix
+go get github.com/quickfixgo/quickfix
 ```
 
-### Staying up to date
+## Staying up to date
 
 To update QuickFIX/Go to the latest version, use `go get -u github.com/quickfixgo/quickfix`.
 
-### Example Apps
+## Example Apps
 
 See [examples](https://github.com/quickfixgo/examples) for some simple examples of using QuickFIX/Go.
 
-### FIX Message Generation
+## FIX Message Generation
 
 QuickFIX/Go includes separate packages for tags, fields, enums, messages, and message components generated from the FIX 4.0 - FIX5.0SP2 specs. See:
 
@@ -51,14 +49,17 @@ For most FIX applications, these generated resources are sufficient. Custom FIX
 
 Following installation, `generate-fix` is installed to `$GOPATH/bin/generate-fix`. Run `$GOPATH/bin/generate-fix --help` for usage instructions.
 
-Developing QuickFIX/Go
-----------------------
+## Developing QuickFIX/Go
 
-If you wish to work on QuickFIX/Go itself, you will first need [Go](http://www.golang.org) installed and configured on your machine (version 1.13+ is preferred, but the minimum required version is 1.6). 
+If you wish to work on QuickFIX/Go itself, you will need [Docker](https://docs.docker.com/get-docker/) and [VSCode](https://code.visualstudio.com/download) on your machine.
 
-Next, using [Git](https://git-scm.com/), clone the repository via `git clone [email protected]:quickfixgo/quickfix.git`
+* Clone the repo and open it with VSCode with Docker running
+* This repo comes with vscode devcontainer configs in `./.devcontainer/`
+* Click the pop-up to re-open the project in the Dev Container
+* This opens the project in a docker container pre-configured with everything you need
 
-### Installing Dependencies
+
+## Installing Dependencies
 
 As of Go version 1.13, QuickFIX/Go uses [modules](https://github.com/golang/go/wiki/Modules) to manage dependencies. You may require `GO111MODULE=on`. To install dependencies, run 
 
@@ -68,48 +69,50 @@ go mod download
 
 **Note:** No vendored dependencies are included in the QuickFIX/Go source.
 
-### Build and Test
+## Build and Test
 
 The default make target runs [go vet](https://godoc.org/golang.org/x/tools/cmd/vet) and unit tests.
 
 ```sh
-$ make
+make
 ```
 
 If this exits with exit status 0, then everything is working!
 
-### Generated Code
+## Generated Code
 
 Generated code from the FIX40-FIX50SP2 specs are available as separate repos under the [QuickFIX/Go organization](https://github.com/quickfixgo).  The source specifications for this generated code is located in `spec/`.  Generated code can be identified by the `.generated.go` suffix.  Any changes to generated code must be captured by changes to source in `cmd/generate-fix`.  After making changes to the code generator source, run the following to re-generate the source
 
 ```sh
-$ make generate-dist
+make generate
 ```
 
 If you are making changes to the generated code, please create Pull Requests for these changes for the affected repos.
 
-### Acceptance Tests
+## Acceptance Tests
 
 QuickFIX/Go has a comprehensive acceptance test suite covering the FIX protocol.  These are the same tests used across all QuickFIX implementations.
 
-QuickFIX/Go acceptance tests depend on ruby in path.
+QuickFIX/Go acceptance tests depend on ruby in path, if you are using the dev container, it is already installed
 
 To run acceptance tests,
 
-        # generate code locally
-        make generate
+```sh
+# generate code locally
+make generate
 
-		# build acceptance test rig
-		make build-test-srv
+# build acceptance test rig
+make build-test-srv
 
-		# run acceptance tests
-		make accept
+# run acceptance tests
+make accept
+```
 
-### Dependencies
+## Dependencies
 
 If you are developing QuickFIX/Go, there are a few tasks you might need to perform related to dependencies.
 
-#### Adding/updating a dependency
+### Adding/updating a dependency
 
 If you are adding or updating a dependency, you will need to update the `go.mod` and `go.sum` in the same Pull Request as the code that depends on it. You should do this in a separate commit from your code, as this makes PR review easier and Git history simpler to read in the future. 
 

cmd/generate-fix/generate-fix.go

@@ -122,7 +122,6 @@ func main() {
 		usage()
 	}
 
-
 	args := flag.Args()
 	if len(args) == 1 {
 		dictpath := args[0]

cmd/generate-fix/internal/template_helpers.go

@@ -34,7 +34,7 @@ func checkIfTimeImportRequiredForFields(fTypes []*datadictionary.FieldType) (ok
 		if vt, err = quickfixValueType(t); err != nil {
 			return
 		}
-	
+
 		if vt == "time.Time" {
 			return true, nil
 		}

config/configuration.go

@@ -55,6 +55,7 @@ const (
 	SQLStoreConnMaxLifetime      string = "SQLStoreConnMaxLifetime"
 	MongoStoreConnection         string = "MongoStoreConnection"
 	MongoStoreDatabase           string = "MongoStoreDatabase"
+	MongoStoreReplicaSet         string = "MongoStoreReplicaSet"
 	ValidateFieldsOutOfOrder     string = "ValidateFieldsOutOfOrder"
 	ResendRequestChunkSize       string = "ResendRequestChunkSize"
 	EnableLastMsgSeqNumProcessed string = "EnableLastMsgSeqNumProcessed"

config/doc.go

@@ -363,6 +363,10 @@ MongoStoreDatabase
 
 The MongoDB-specific name of the database to use.  Only used with MongoStoreFactory.
 
+MongoStoreReplicaSet
+
+The MongoDB-specific name of the replica set to use. Optional, only used with MongoStoreFactory.
+
 SQLStoreDriver
 
 The name of the database driver to use (see https://github.com/golang/go/wiki/SQLDrivers for the list of available drivers).  Only used with SqlStoreFactory.

filestore.go

@@ -287,8 +287,6 @@ func (store *fileStore) SaveMessage(seqNum int, msg []byte) error {
 		return fmt.Errorf("unable to write to file: %s: %s", store.headerFname, err.Error())
 	}
 
-	store.offsets[seqNum] = msgDef{offset: offset, size: len(msg)}
-
 	if _, err := store.bodyFile.Write(msg); err != nil {
 		return fmt.Errorf("unable to write to file: %s: %s", store.bodyFname, err.Error())
 	}
@@ -298,9 +296,19 @@ func (store *fileStore) SaveMessage(seqNum int, msg []byte) error {
 	if err := store.headerFile.Sync(); err != nil {
 		return fmt.Errorf("unable to flush file: %s: %s", store.headerFname, err.Error())
 	}
+
+	store.offsets[seqNum] = msgDef{offset: offset, size: len(msg)}
 	return nil
 }
 
+func (store *fileStore) SaveMessageAndIncrNextSenderMsgSeqNum(seqNum int, msg []byte) error {
+	err := store.SaveMessage(seqNum, msg)
+	if err != nil {
+		return err
+	}
+	return store.IncrNextSenderMsgSeqNum()
+}
+
 func (store *fileStore) getMessage(seqNum int) (msg []byte, found bool, err error) {
 	msgInfo, found := store.offsets[seqNum]
 	if !found {