Prefix bdapi fix and Override testing (#188)

completed and tested MVP for prefix transport

Author: jmwample

application/lib/registration_ingest.go

@@ -413,7 +413,9 @@ func (rm *RegistrationManager) NewRegistrationC2SWrapper(c2sw *pb.C2SWrapper, in
 			dstPort = int(rr.GetDstPort())
 		}
 
-		if rr.TransportParams != nil {
+		// if Transport param overrides are defined and the client indicated that overrides are
+		// allowed, apply the overrides while creating the registration
+		if rr.GetTransportParams() != nil && !c2s.GetDisableRegistrarOverrides() {
 			c2s.TransportParams = rr.GetTransportParams()
 		}
 

application/transports/wrapping/min/client.go

@@ -9,6 +9,7 @@ import (
 	"github.com/refraction-networking/conjure/pkg/core"
 	pb "github.com/refraction-networking/gotapdance/protobuf"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
 // ClientTransport implements the client side transport interface for the Min transport. The
@@ -43,9 +44,21 @@ func (t *ClientTransport) GetParams() (proto.Message, error) {
 	return t.Parameters, nil
 }
 
+// ParseParams gives the specific transport an option to parse a generic object into parameters
+// provided by the station in the registration response during registration.
+func (t ClientTransport) ParseParams(data *anypb.Any) (any, error) {
+	if data == nil {
+		return nil, nil
+	}
+
+	var m = &pb.GenericTransportParams{}
+	err := transports.UnmarshalAnypbTo(data, m)
+	return m, err
+}
+
 // SetParams allows the caller to set parameters associated with the transport, returning an
 // error if the provided generic message is not compatible.
-func (t *ClientTransport) SetParams(p any) error {
+func (t *ClientTransport) SetParams(p any, unchecked ...bool) error {
 	params, ok := p.(*pb.GenericTransportParams)
 	if !ok {
 		return fmt.Errorf("unable to parse params")

application/transports/wrapping/obfs4/client.go

@@ -8,6 +8,7 @@ import (
 	pt "git.torproject.org/pluggable-transports/goptlib.git"
 	"gitlab.com/yawning/obfs4.git/transports/obfs4"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/refraction-networking/conjure/application/transports"
 	pb "github.com/refraction-networking/gotapdance/protobuf"
@@ -45,7 +46,7 @@ func (t *ClientTransport) GetParams() (proto.Message, error) {
 
 // SetParams allows the caller to set parameters associated with the transport, returning an
 // error if the provided generic message is not compatible.
-func (t *ClientTransport) SetParams(p any) error {
+func (t *ClientTransport) SetParams(p any, unchecked ...bool) error {
 	params, ok := p.(*pb.GenericTransportParams)
 	if !ok {
 		return fmt.Errorf("unable to parse params")
@@ -55,6 +56,18 @@ func (t *ClientTransport) SetParams(p any) error {
 	return nil
 }
 
+// ParseParams gives the specific transport an option to parse a generic object into parameters
+// provided by the station in the registration response during registration.
+func (t ClientTransport) ParseParams(data *anypb.Any) (any, error) {
+	if data == nil {
+		return nil, nil
+	}
+
+	var m = &pb.GenericTransportParams{}
+	err := transports.UnmarshalAnypbTo(data, m)
+	return m, err
+}
+
 // GetDstPort returns the destination port that the client should open the phantom connection to
 func (t *ClientTransport) GetDstPort(seed []byte) (uint16, error) {
 	if t.Parameters == nil || !t.Parameters.GetRandomizeDstPort() {

application/transports/wrapping/prefix/README.md

@@ -0,0 +1,62 @@
+
+# Prefix Transport
+
+**TLDR** - This transport allows up to prepend conjure connections with bytes that look like the
+initialization of other protocols. This can help to circumvent blocking in some areas and better
+understand censorship regimes, but is generally a short term solution.
+
+The `Prefix_Min` transport is a strictly improved version of the existing `Min` transport and we
+suggest migration.
+
+## Description
+
+This package implements the prefix transport for the conjure refraction-networking system. The
+prefix transport operates in much the same way as the min transport, sending a tag in the fist
+packet signalling to the station that the flow has knowledge of a secret shared with the station by
+a previous registration.
+
+TODO: Comparison to min transport
+
+### Prefixes Supported by Default
+
+TODO: The prefixes supported by default are as follows.
+
+### Ports
+
+TODO: Prefixes have default ports associated with them, but also allow port randomization.
+
+### :warning: Sharp Edges :warning:
+
+In general this transport will not properly mimic the protocols that are sent as a prefix and should
+not be expected to do so.
+
+## Integrating the Prefix Transport
+
+Though the client dialer allows the use of TrasnportType  for compatibility reasons, the prefix
+transport requires use of the newer Client Transport interface (`TransportConfig` in the dialer)
+which is implemented by the `prefix.ClientTransport` object.
+
+TODO: code change example.
+
+## Adding a Prefix / Bidirectional Registration Prefix Overrides
+
+In order to add a prefix ...
+
+## :construction: Road-Map
+
+These features are not necessarily planned or landing imminently, they are simply things that would
+be nice to have.
+
+- [ ] **Server Side Prefix Override From File** - file format shared between station and Reg server
+  describing available prefixes outside of defaults.
+
+- [ ] **TagEncodings** - Allow the tag to (by prefix configuration) be encoded using an encoder
+  expected by the station, Base64 for example.
+
+- [ ] **StreamEncodings** - Allow the Stream of client bytes to (by configuration) encoded /
+  encrypted using a scheme expected by the station, AES or Base64 for example.
+
+- [ ] **Randomization** - indicate segments of the prefix to be filled from a random source.
+
+- [ ] **Prefix Revocation** - If there is a prefix that is known to be blocked and we don't want
+  clients to use it, but we still want them to roll a random prefix, how do we do this?

application/transports/wrapping/prefix/client.go

@@ -1,6 +1,7 @@
 package prefix
 
 import (
+	"bufio"
 	"crypto/rand"
 	"fmt"
 	"io"
@@ -11,6 +12,7 @@ import (
 	"github.com/refraction-networking/conjure/pkg/core"
 	pb "github.com/refraction-networking/gotapdance/protobuf"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
 // ClientTransport implements the client side transport interface for the Min transport. The
@@ -42,6 +44,7 @@ type ClientParams struct {
 // behavior like a rand prefix for example.
 type Prefix interface {
 	Bytes() []byte
+	FlushAfterPrefix() bool
 	ID() PrefixID
 	DstPort([]byte) uint16
 }
@@ -91,16 +94,42 @@ func (t *ClientTransport) GetParams() (proto.Message, error) {
 	return t.parameters, nil
 }
 
+// ParseParams gives the specific transport an option to parse a generic object into parameters
+// provided by the station in the registration response during registration.
+func (t ClientTransport) ParseParams(data *anypb.Any) (any, error) {
+	if data == nil {
+		return nil, nil
+	}
+
+	var m = &pb.PrefixTransportParams{}
+	err := transports.UnmarshalAnypbTo(data, m)
+	return m, err
+}
+
 // SetParams allows the caller to set parameters associated with the transport, returning an
 // error if the provided generic message is not compatible or the parameters are otherwise invalid
-func (t *ClientTransport) SetParams(p any) error {
+func (t *ClientTransport) SetParams(p any, unchecked ...bool) error {
 	prefixParams, ok := p.(*pb.PrefixTransportParams)
 	if !ok {
-		return ErrBadParams
+		return fmt.Errorf("%w, incorrect param type", ErrBadParams)
 	}
 
 	if prefixParams == nil {
-		return ErrBadParams
+		return fmt.Errorf("%w, nil params", ErrBadParams)
+	}
+
+	if len(unchecked) != 0 && unchecked[0] {
+		// Overwrite the prefix bytes and type without checking the default set. This is used for
+		// RegResponse where the registrar may override the chosen prefix with a prefix outside of
+		// the prefixes that the client known about.
+		t.parameters = prefixParams
+		t.Prefix = &clientPrefix{
+			bytes:            prefixParams.GetPrefix(),
+			id:               PrefixID(prefixParams.GetPrefixId()),
+			flushAfterPrefix: prefixParams.GetFlushAfterPrefix(),
+		}
+
+		return nil
 	}
 
 	if prefix, ok := DefaultPrefixes[PrefixID(prefixParams.GetPrefixId())]; ok {
@@ -163,26 +192,6 @@ func (t *ClientTransport) GetDstPort(seed []byte) (uint16, error) {
 	return t.Prefix.DstPort(seed), nil
 }
 
-// Build is specific to the Prefix transport, providing a utility function for building the
-// prefix that the client should write to the wire before sending any client bytes.
-func (t *ClientTransport) Build() ([]byte, error) {
-	if t.Prefix == nil {
-		return nil, ErrBadParams
-	}
-
-	// Send hmac(seed, str) bytes to indicate to station (min transport)
-	prefix := t.Prefix.Bytes()
-
-	if t.TagObfuscator == nil {
-		t.TagObfuscator = transports.CTRObfuscator{}
-	}
-	obfuscatedID, err := t.TagObfuscator.Obfuscate(t.connectTag, t.stationPublicKey[:])
-	if err != nil {
-		return nil, err
-	}
-	return append(prefix, obfuscatedID...), nil
-}
-
 // PrepareKeys provides an opportunity for the transport to integrate the station public key
 // as well as bytes from the deterministic random generator associated with the registration
 // that this ClientTransport is attached to.
@@ -195,29 +204,52 @@ func (t *ClientTransport) PrepareKeys(pubkey [32]byte, sharedSecret []byte, hkdf
 // WrapConn gives the transport the opportunity to perform a handshake and wrap / transform the
 // incoming and outgoing bytes send by the implementing client.
 func (t *ClientTransport) WrapConn(conn net.Conn) (net.Conn, error) {
-	// Send hmac(seed, str) bytes to indicate to station (min transport) generated during Prepare(...)
-
-	// // Send hmac(seed, str) bytes to indicate to station (min transport)
-	// connectTag := core.ConjureHMAC(reg.keys.SharedSecret, "PrefixTransportHMACString")
+	if t.Prefix == nil {
+		return nil, ErrBadParams
+	}
 
-	prefix, err := t.Build()
-	if err != nil {
-		return nil, fmt.Errorf("failed to build prefix: %w", err)
+	if t.TagObfuscator == nil {
+		t.TagObfuscator = transports.CTRObfuscator{}
 	}
 
-	_, err = conn.Write(prefix)
+	obfuscatedID, err := t.TagObfuscator.Obfuscate(t.connectTag, t.stationPublicKey[:])
 	if err != nil {
 		return nil, err
 	}
+
+	w := bufio.NewWriter(conn)
+
+	var msg []byte = t.Prefix.Bytes()
+	if t.Prefix.FlushAfterPrefix() {
+		if _, err := w.Write(msg); err != nil {
+			return nil, err
+		}
+
+		w.Flush()
+		if _, err := w.Write(obfuscatedID); err != nil {
+			return nil, err
+		}
+
+		w.Flush()
+	} else {
+		msg = append(msg, obfuscatedID...)
+		if _, err := w.Write(msg); err != nil {
+			return nil, err
+		}
+
+		w.Flush()
+	}
+
 	return conn, nil
 }
 
 // ---
 
 type clientPrefix struct {
-	bytes []byte
-	id    PrefixID
-	port  uint16
+	bytes            []byte
+	id               PrefixID
+	port             uint16
+	flushAfterPrefix bool
 
 	// // Function allowing encoding / transformation of obfuscated ID bytes after they have been
 	// // obfuscated. Examples - base64 encode, padding
@@ -242,6 +274,10 @@ func (c *clientPrefix) DstPort([]byte) uint16 {
 	return c.port
 }
 
+func (c *clientPrefix) FlushAfterPrefix() bool {
+	return c.flushAfterPrefix
+}
+
 // ---
 
 // TryFromID returns a Prefix based on the Prefix ID. This is useful for non-static prefixes like the

application/transports/wrapping/prefix/long_prefixes.go

@@ -0,0 +1,17 @@
+package prefix
+
+import "encoding/hex"
+
+var httpGetComplete []byte = d("474554202f20485454502f312e310d0a4163636570743a202a2f2a0d0a436f6e6e656374696f6e3a20636c6f73650d0a0d0a")
+
+var tlsCompleteCHSNI []byte = d("1603010200010001fc0303a08b89dd2ce2e5bc764a91bbd5cae46fdc7062e2dd6e7eb891fcec639e228c0e2062bcb02f5174081bbcd30f87015ebaca6d74a11b23de273ff5b85a3ef89f704600208a8a130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010001932a2a000000230000002b000706eaea030403030010000e000c02683208687474702f312e31446900050003026832001b0003020002000000160014000011746c7366696e6765727072696e742e696f000b00020100000a000a00088a8a001d001700180033002b00298a8a000100001d0020a02f03fcf4a86e3df6c3f79aa659be0a5209946f9fd0e8fe2b3cc1b664f0985f00120000000d0012001004030804040105030805050108060601002d00020101000500050100000000ff01000100001700004a4a000100001500c6000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+
+var tlsCompleteCHNoSNI []byte = d("1603010200010001fc0303a937d48cce916eb20df06d75f71a5954ea7b6876217ce17956c140ebd891716e20a079e04d2379c8dcd78d3f1f8075f7deb4dab5f06c014c73bf45478a486154090020eaea130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010001935a5a0000ff01000100001b00030200020012000000170000000d0012001004030804040105030805050108060601000500050100000000446900050003026832002d00020101000b000201000010000e000c02683208687474702f312e3100230000002b000706eaea03040303000a000a00085a5a001d001700180033002b00295a5a000100001d00207df3a2aa3c8403fc92d8307833f51eb2576c43057afffecab53ea0bad45840642a2a000100001500e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+
+func d(in string) []byte {
+	out, err := hex.DecodeString(in)
+	if err != nil {
+		panic(err)
+	}
+	return out
+}