allow decrypting net-messages via ParserConfig.NetMessageDecryptionKey (#323)

Author: markus-wa

go.mod

@@ -8,11 +8,14 @@ require (
 	github.com/markus-wa/go-unassert v0.1.2
 	github.com/markus-wa/gobitread v0.2.3
 	github.com/markus-wa/godispatch v1.4.1
+	github.com/markus-wa/ice-cipher-go v0.0.0-20220126215401-a6adadccc817
 	github.com/markus-wa/quickhull-go/v2 v2.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 )
 
 replace github.com/dustin/go-heatmap => github.com/markus-wa/go-heatmap v1.0.0
 
+replace github.com/stretchr/testify => github.com/stretchr/testify v1.6.1
+
 go 1.11

go.sum

@@ -26,6 +26,8 @@ github.com/markus-wa/gobitread v0.2.3 h1:COx7dtYQ7Q+77hgUmD+O4MvOcqG7y17RP3Z7Bbj
 github.com/markus-wa/gobitread v0.2.3/go.mod h1:PcWXMH4gx7o2CKslbkFkLyJB/aHW7JVRG3MRZe3PINg=
 github.com/markus-wa/godispatch v1.4.1 h1:Cdff5x33ShuX3sDmUbYWejk7tOuoHErFYMhUc2h7sLc=
 github.com/markus-wa/godispatch v1.4.1/go.mod h1:tk8L0yzLO4oAcFwM2sABMge0HRDJMdE8E7xm4gK/+xM=
+github.com/markus-wa/ice-cipher-go v0.0.0-20220126215401-a6adadccc817 h1:VB4ANo078mbGQNBgauLOzJkoHpIqrMygbPHVW0jjql0=
+github.com/markus-wa/ice-cipher-go v0.0.0-20220126215401-a6adadccc817/go.mod h1:JIsht5Oa9P50VnGJTvH2a6nkOqDFJbUeU1YRZYvdplw=
 github.com/markus-wa/quickhull-go/v2 v2.1.0 h1:DA2pzEzH0k5CEnlUsouRqNdD+jzNFb4DBhrX4Hpa5So=
 github.com/markus-wa/quickhull-go/v2 v2.1.0/go.mod h1:bOlBUpIzGSMMhHX0f9N8CQs0VZD4nnPeta0OocH7m4o=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -34,7 +36,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -68,6 +69,5 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/demoinfocs/demoinfocs_test.go

@@ -194,6 +194,43 @@ func TestDemoInfoCs(t *testing.T) {
 	assertGolden(t, assertions, "default", actual.Bytes())
 }
 
+func TestEncryptedNetMessages(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		t.Skip("skipping test due to -short flag")
+	}
+
+	infoF, err := os.Open(csDemosPath + "/match730_003528806449641685104_1453182610_271.dem.info")
+
+	b, err := ioutil.ReadAll(infoF)
+	assert.NoError(t, err)
+
+	k, err := demoinfocs.MatchInfoDecryptionKey(b)
+	assert.NoError(t, err)
+
+	f, err := os.Open(csDemosPath + "/match730_003528806449641685104_1453182610_271.dem")
+	assert.NoError(t, err)
+	defer mustClose(t, f)
+
+	cfg := demoinfocs.DefaultParserConfig
+	cfg.NetMessageDecryptionKey = k
+
+	p := demoinfocs.NewParserWithConfig(f, cfg)
+
+	p.RegisterEventHandler(func(message events.ChatMessage) {
+		t.Log(message)
+	})
+
+	err = p.ParseToEnd()
+	assert.NoError(t, err)
+}
+
+func TestMatchInfoDecryptionKey_Error(t *testing.T) {
+	_, err := demoinfocs.MatchInfoDecryptionKey([]byte{0})
+	assert.Error(t, err)
+}
+
 func TestRetake_BadBombsiteIndex(t *testing.T) {
 	t.Parallel()
 

pkg/demoinfocs/events/events.go

@@ -522,6 +522,10 @@ const (
 	WarnTypeBombsiteUnknown            // may occur on de_grind for bombsite B as the bounding box of the bombsite is wrong
 	WarnTypeTeamSwapPlayerNil          // TODO: figure out why this happens
 	WarnTypeGameEventBeforeDescriptors // may occur in POV demos
+
+	// WarnTypeMissingNetMessageDecryptionKey occurs when encrypted net-messages are encountered and the decryption key is missing
+	// See ParserConfig.NetMessageDecryptionKey
+	WarnTypeMissingNetMessageDecryptionKey
 )
 
 // ParserWarn signals that a non-fatal problem occurred during parsing.

pkg/demoinfocs/matchinfo.go

@@ -0,0 +1,26 @@
+package demoinfocs
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/pkg/errors"
+
+	"github.com/markus-wa/demoinfocs-golang/v2/pkg/demoinfocs/msg"
+)
+
+// MatchInfoDecryptionKey extracts the net-message decryption key stored in `match730_*.dem.info`.
+// Pass the whole contents of `match730_*.dem.info` to this function to get the key.
+func MatchInfoDecryptionKey(b []byte) ([]byte, error) {
+	m := new(msg.CDataGCCStrike15V2_MatchInfo)
+
+	err := proto.Unmarshal(b, m)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal MatchInfo message")
+	}
+
+	k := []byte(strings.ToUpper(strconv.FormatUint(m.Watchablematchinfo.ClDecryptdataKeyPub, 16)))
+
+	return k, nil
+}

pkg/demoinfocs/net_messages.go

@@ -2,6 +2,11 @@ package demoinfocs
 
 import (
 	"bytes"
+	"encoding/binary"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/markus-wa/go-unassert"
+	"github.com/markus-wa/ice-cipher-go/pkg/ice"
 
 	bit "github.com/markus-wa/demoinfocs-golang/v2/internal/bitread"
 	events "github.com/markus-wa/demoinfocs-golang/v2/pkg/demoinfocs/events"
@@ -71,3 +76,56 @@ func (p *parser) handleServerInfo(srvInfo *msg.CSVCMsg_ServerInfo) {
 		TickTime: p.TickTime(),
 	})
 }
+
+func (p *parser) handleEncryptedData(msg *msg.CSVCMsg_EncryptedData) {
+	if msg.KeyType != 2 {
+		return
+	}
+
+	if p.decryptionKey == nil {
+		p.msgDispatcher.Dispatch(events.ParserWarn{
+			Type:    events.WarnTypeMissingNetMessageDecryptionKey,
+			Message: "received encrypted net-message but no decryption key is set",
+		})
+
+		return
+	}
+
+	k := ice.NewKey(2, p.decryptionKey)
+	b := k.DecryptAll(msg.Encrypted)
+
+	r := bytes.NewReader(b)
+	br := bit.NewSmallBitReader(r)
+
+	paddingBytes := br.ReadSingleByte()
+	br.Skip(int(paddingBytes) << 3)
+
+	bBytesWritten := br.ReadBytes(4)
+	nBytesWritten := int(binary.BigEndian.Uint32(bBytesWritten))
+
+	unassert.Same(len(b), 5+int(paddingBytes)+nBytesWritten)
+
+	cmd := br.ReadVarInt32()
+	size := br.ReadVarInt32()
+
+	m := p.netMessageForCmd(int(cmd))
+
+	if m == nil {
+		err := br.Pool()
+		if err != nil {
+			p.setError(err)
+		}
+
+		return
+	}
+
+	msgB := br.ReadBytes(int(size))
+	err := proto.Unmarshal(msgB, m)
+	if err != nil {
+		p.setError(err)
+
+		return
+	}
+
+	p.msgDispatcher.Dispatch(m)
+}

pkg/demoinfocs/parser.go

@@ -63,6 +63,7 @@ type parser struct {
 	demoInfoProvider             demoInfoProvider // Provides demo infos to other packages that the core package depends on
 	err                          error            // Contains a error that occurred during parsing if any
 	errLock                      sync.Mutex       // Used to sync up error mutations between parsing & handling go-routines
+	decryptionKey                []byte           // Stored in `match730_*.dem.info` see MatchInfoDecryptionKey().
 
 	// Additional fields, mainly caching & tracking things
 
@@ -316,6 +317,10 @@ type ParserConfig struct {
 	// IgnoreErrBombsiteIndexNotFound tells the parser to not return an error when a bombsite-index from a game-event is not found in the demo.
 	// See https://github.com/markus-wa/demoinfocs-golang/issues/314
 	IgnoreErrBombsiteIndexNotFound bool
+
+	// NetMessageDecryptionKey tells the parser how to decrypt certain encrypted net-messages.
+	// See MatchInfoDecryptionKey() on how to retrieve the key from
+	NetMessageDecryptionKey []byte
 }
 
 // DefaultParserConfig is the default Parser configuration used by NewParser().
@@ -342,6 +347,7 @@ func NewParserWithConfig(demostream io.Reader, config ParserConfig) Parser {
 	p.userMessageHandler = newUserMessageHandler(&p)
 	p.bombsiteA.index = -1
 	p.bombsiteB.index = -1
+	p.decryptionKey = config.NetMessageDecryptionKey
 
 	dispatcherCfg := dp.Config{
 		PanicHandler: func(v interface{}) {
@@ -361,6 +367,7 @@ func NewParserWithConfig(demostream io.Reader, config ParserConfig) Parser {
 	p.msgDispatcher.RegisterHandler(p.handleSetConVar)
 	p.msgDispatcher.RegisterHandler(p.handleFrameParsed)
 	p.msgDispatcher.RegisterHandler(p.handleServerInfo)
+	p.msgDispatcher.RegisterHandler(p.handleEncryptedData)
 	p.msgDispatcher.RegisterHandler(p.gameState.handleIngameTickNumber)
 
 	if config.MsgQueueBufferSize >= 0 {

pkg/demoinfocs/parsing.go

@@ -292,6 +292,39 @@ var defaultNetMessageCreators = map[int]NetMessageCreator{
 	int(msg.SVC_Messages_svc_UserMessage):       func() proto.Message { return new(msg.CSVCMsg_UserMessage) },
 	int(msg.SVC_Messages_svc_ServerInfo):        func() proto.Message { return new(msg.CSVCMsg_ServerInfo) },
 	int(msg.NET_Messages_net_SetConVar):         func() proto.Message { return new(msg.CNETMsg_SetConVar) },
+	int(msg.SVC_Messages_svc_EncryptedData):     func() proto.Message { return new(msg.CSVCMsg_EncryptedData) },
+}
+
+func (p *parser) netMessageForCmd(cmd int) proto.Message {
+	msgCreator := defaultNetMessageCreators[cmd]
+
+	if msgCreator != nil {
+		return msgCreator()
+	}
+
+	var msgName string
+	if cmd < 8 || cmd >= 100 {
+		msgName = msg.NET_Messages_name[int32(cmd)]
+	} else {
+		msgName = msg.SVC_Messages_name[int32(cmd)]
+	}
+
+	if msgName == "" {
+		// Send a warning if the command is unknown
+		// This might mean our proto files are out of date
+		p.eventDispatcher.Dispatch(events.ParserWarn{Message: fmt.Sprintf("unknown message command %q", cmd)})
+		unassert.Error("unknown message command %q", cmd)
+	}
+
+	// Handle additional net-messages as defined by the user
+	msgCreator = p.additionalNetMessageCreators[cmd]
+	if msgCreator != nil {
+		return msgCreator()
+	}
+
+	debugUnhandledMessage(cmd, msgName)
+
+	return nil
 }
 
 //nolint:funlen
@@ -311,41 +344,19 @@ func (p *parser) parsePacket() {
 
 		p.bitReader.BeginChunk(size << 3)
 
-		msgCreator := defaultNetMessageCreators[cmd]
-
-		if msgCreator == nil {
-			var msgName string
-			if cmd < 8 || cmd >= 100 {
-				msgName = msg.NET_Messages_name[int32(cmd)]
-			} else {
-				msgName = msg.SVC_Messages_name[int32(cmd)]
-			}
-
-			if msgName == "" {
-				// Send a warning if the command is unknown
-				// This might mean our proto files are out of date
-				p.eventDispatcher.Dispatch(events.ParserWarn{Message: fmt.Sprintf("unknown message command %q", cmd)})
-				unassert.Error("unknown message command %q", cmd)
-			}
-
-			// Handle additional net-messages as defined by the user
-			msgCreator = p.additionalNetMessageCreators[cmd]
-			if msgCreator == nil {
-				debugUnhandledMessage(cmd, msgName)
-
-				// On to the next one
-				p.bitReader.EndChunk()
-
-				continue
-			}
+		m := p.netMessageForCmd(cmd)
+
+		if m == nil {
+			// On to the next one
+			p.bitReader.EndChunk()
+
+			continue
 		}
 
 		b := byteSlicePool.Get().(*[]byte)
 
 		p.bitReader.ReadBytesInto(b, size)
 
-		m := msgCreator()
-
 		err := proto.Unmarshal(*b, m)
 		if err != nil {
 			// TODO: Don't crash here, happens with demos that work in gotv

scripts/coverage.sh

@@ -3,7 +3,7 @@
 set -e
 
 scripts_dir=$(dirname "$0")
-$scripts_dir/download-test-data.sh default.7z unexpected_end_of_demo.7z regression-set.7z retake_unknwon_bombsite_index.7z
+$scripts_dir/download-test-data.sh default.7z unexpected_end_of_demo.7z regression-set.7z retake_unknwon_bombsite_index.7z valve_matchmaking.7z
 
 # don't cover mocks and generated protobuf code
 coverpkg_ignore='/(fake|msg)'