Merge branch 'main' into shay/no_redact

Author: H-Shay

.github/workflows/ci.yaml

@@ -38,7 +38,7 @@ jobs:
       matrix:
         include:
           - homeserver: Synapse
-            tags: synapse_blacklist msc3083 msc3787 faster_joins
+            tags: synapse_blacklist msc3083 msc3787 msc3874 faster_joins
             env: "COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
             timeout: 20m
 

internal/client/client.go

@@ -115,7 +115,7 @@ func (c *CSAPI) UploadContent(t *testing.T, fileBody []byte, fileName string, co
 func (c *CSAPI) DownloadContent(t *testing.T, mxcUri string) ([]byte, string) {
 	t.Helper()
 	origin, mediaId := SplitMxc(mxcUri)
-	res := c.MustDo(t, "GET", []string{"_matrix", "media", "v3", "download", origin, mediaId}, struct{}{})
+	res := c.MustDoFunc(t, "GET", []string{"_matrix", "media", "v3", "download", origin, mediaId})
 	contentType := res.Header.Get("Content-Type")
 	b, err := ioutil.ReadAll(res.Body)
 	if err != nil {
@@ -127,7 +127,7 @@ func (c *CSAPI) DownloadContent(t *testing.T, mxcUri string) ([]byte, string) {
 // CreateRoom creates a room with an optional HTTP request body. Fails the test on error. Returns the room ID.
 func (c *CSAPI) CreateRoom(t *testing.T, creationContent interface{}) string {
 	t.Helper()
-	res := c.MustDo(t, "POST", []string{"_matrix", "client", "v3", "createRoom"}, creationContent)
+	res := c.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "createRoom"}, WithJSONBody(t, creationContent))
 	body := ParseJSON(t, res)
 	return GetJSONFieldStr(t, body, "room_id")
 }
@@ -166,7 +166,7 @@ func (c *CSAPI) InviteRoom(t *testing.T, roomID string, userID string) {
 	body := map[string]interface{}{
 		"user_id": userID,
 	}
-	c.MustDo(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "invite"}, body)
+	c.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "invite"}, WithJSONBody(t, body))
 }
 
 func (c *CSAPI) GetGlobalAccountData(t *testing.T, eventType string) *http.Response {
@@ -186,7 +186,7 @@ func (c *CSAPI) SendEventSynced(t *testing.T, roomID string, e b.Event) string {
 	if e.StateKey != nil {
 		paths = []string{"_matrix", "client", "v3", "rooms", roomID, "state", e.Type, *e.StateKey}
 	}
-	res := c.MustDo(t, "PUT", paths, e.Content)
+	res := c.MustDoFunc(t, "PUT", paths, WithJSONBody(t, e.Content))
 	body := ParseJSON(t, res)
 	eventID := GetJSONFieldStr(t, body, "event_id")
 	t.Logf("SendEventSynced waiting for event ID %s", eventID)
@@ -333,7 +333,7 @@ func (c *CSAPI) RegisterUser(t *testing.T, localpart, password string) (userID,
 		"username": localpart,
 		"password": password,
 	}
-	res := c.MustDo(t, "POST", []string{"_matrix", "client", "v3", "register"}, reqBody)
+	res := c.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "register"}, WithJSONBody(t, reqBody))
 
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
@@ -411,21 +411,6 @@ func (c *CSAPI) GetDefaultRoomVersion(t *testing.T) gomatrixserverlib.RoomVersio
 	return gomatrixserverlib.RoomVersion(defaultVersion.Str)
 }
 
-// MustDo will do the HTTP request and fail the test if the response is not 2xx
-//
-// Deprecated: Prefer MustDoFunc. MustDo is the older format which doesn't allow for vargs
-// and will be removed in the future. MustDoFunc also logs HTTP response bodies on error.
-func (c *CSAPI) MustDo(t *testing.T, method string, paths []string, jsonBody interface{}) *http.Response {
-	t.Helper()
-	res := c.DoFunc(t, method, paths, WithJSONBody(t, jsonBody))
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		defer res.Body.Close()
-		body, _ := ioutil.ReadAll(res.Body)
-		t.Fatalf("CSAPI.MustDo %s %s returned HTTP %d : %s", method, res.Request.URL.String(), res.StatusCode, string(body))
-	}
-	return res
-}
-
 // WithRawBody sets the HTTP request body to `body`
 func WithRawBody(body []byte) RequestOpt {
 	return func(req *http.Request) {
@@ -482,7 +467,7 @@ func (c *CSAPI) MustDoFunc(t *testing.T, method string, paths []string, opts ...
 	if res.StatusCode < 200 || res.StatusCode >= 300 {
 		defer res.Body.Close()
 		body, _ := ioutil.ReadAll(res.Body)
-		t.Fatalf("CSAPI.MustDoFunc response return non-2xx code: %s - body: %s", res.Status, string(body))
+		t.Fatalf("CSAPI.MustDoFunc %s %s returned non-2xx code: %s - body: %s", method, res.Request.URL.String(), res.Status, string(body))
 	}
 	return res
 }
@@ -860,3 +845,48 @@ func SplitMxc(mxcUri string) (string, string) {
 
 	return origin, mediaId
 }
+
+// SendToDeviceMessages sends to-device messages over /sendToDevice/.
+//
+// The messages parameter is nested as follows:
+// user_id -> device_id -> content (map[string]interface{})
+func (c *CSAPI) SendToDeviceMessages(t *testing.T, evType string, messages map[string]map[string]map[string]interface{}) {
+	t.Helper()
+	c.txnID++
+	c.MustDoFunc(
+		t,
+		"PUT",
+		[]string{"_matrix", "client", "v3", "sendToDevice", evType, strconv.Itoa(c.txnID)},
+		WithJSONBody(
+			t,
+			map[string]map[string]map[string]map[string]interface{}{
+				"messages": messages,
+			},
+		),
+	)
+}
+
+// Check that sync has received a to-device message,
+// with optional user filtering.
+//
+// If fromUser == "", all messages will be passed through to the check function.
+// `check` will be called for all messages that have passed the filter.
+//
+// `check` gets passed the full event, including sender and type.
+func SyncToDeviceHas(fromUser string, check func(gjson.Result) bool) SyncCheckOpt {
+	return func(clientUserID string, topLevelSyncJSON gjson.Result) error {
+		err := loopArray(
+			topLevelSyncJSON, "to_device.events", func(result gjson.Result) bool {
+				if fromUser != "" && result.Get("sender").Str != fromUser {
+					return false
+				} else {
+					return check(result)
+				}
+			},
+		)
+		if err == nil {
+			return nil
+		}
+		return fmt.Errorf("SyncToDeviceHas(%v): %s", fromUser, err)
+	}
+}

internal/federation/server.go

@@ -248,6 +248,37 @@ func (s *Server) SendFederationRequest(
 	return err
 }
 
+// DoFederationRequest signs and sends an arbitrary federation request from this server, and returns the response.
+//
+// The requests will be routed according to the deployment map in `deployment`.
+func (s *Server) DoFederationRequest(
+	ctx context.Context,
+	t *testing.T,
+	deployment *docker.Deployment,
+	req gomatrixserverlib.FederationRequest) (*http.Response, error) {
+	if err := req.Sign(gomatrixserverlib.ServerName(s.serverName), s.KeyID, s.Priv); err != nil {
+		return nil, err
+	}
+
+	httpReq, err := req.HTTPRequest()
+	if err != nil {
+		return nil, err
+	}
+
+	httpClient := gomatrixserverlib.NewClient(gomatrixserverlib.WithTransport(&docker.RoundTripper{Deployment: deployment}))
+	start := time.Now()
+
+	var resp *http.Response
+	resp, err = httpClient.DoHTTPRequest(ctx, httpReq)
+
+	if httpError, ok := err.(gomatrix.HTTPError); ok {
+		t.Logf("[SSAPI] %s %s%s => error(%d): %s (%s)", req.Method(), req.Destination(), req.RequestURI(), httpError.Code, err, time.Since(start))
+	} else if err == nil {
+		t.Logf("[SSAPI] %s %s%s => %d (%s)", req.Method(), req.Destination(), req.RequestURI(), resp.StatusCode, time.Since(start))
+	}
+	return resp, err
+}
+
 // MustCreateEvent will create and sign a new latest event for the given room.
 // It does not insert this event into the room however. See ServerRoom.AddEvent for that.
 func (s *Server) MustCreateEvent(t *testing.T, room *ServerRoom, ev b.Event) *gomatrixserverlib.Event {
@@ -437,7 +468,7 @@ func (s *Server) Listen() (cancel func()) {
 	}()
 
 	return func() {
-		err := s.srv.Shutdown(context.Background())
+		err := s.srv.Close()
 		if err != nil {
 			s.t.Fatalf("ListenFederationServer: failed to shutdown server: %s", err)
 		}

internal/must/must.go

@@ -132,6 +132,35 @@ func MatchFederationRequest(t *testing.T, fedReq *gomatrixserverlib.FederationRe
 	}
 }
 
+// MatchGJSON performs JSON assertions on a gjson.Result object.
+func MatchGJSON(t *testing.T, jsonResult gjson.Result, matchers ...match.JSON) {
+	t.Helper()
+
+	MatchJSON(t, jsonResult.Str, matchers...)
+}
+
+// MatchJSON performs JSON assertions on a raw JSON string.
+func MatchJSON(t *testing.T, json string, matchers ...match.JSON) {
+	t.Helper()
+
+	MatchJSONBytes(t, []byte(json), matchers...)
+}
+
+// MatchJSONBytes performs JSON assertions on a raw json byte slice.
+func MatchJSONBytes(t *testing.T, rawJson []byte, matchers ...match.JSON) {
+	t.Helper()
+
+	if !gjson.ValidBytes(rawJson) {
+		t.Fatalf("MatchJSONBytes: rawJson is not valid JSON")
+	}
+
+	for _, jm := range matchers {
+		if err := jm(rawJson); err != nil {
+			t.Fatalf("MatchJSONBytes %s", err)
+		}
+	}
+}
+
 // EqualStr ensures that got==want else logs an error.
 func EqualStr(t *testing.T, got, want, msg string) {
 	t.Helper()

internal/web/server.go

@@ -0,0 +1,50 @@
+package web
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/matrix-org/complement/internal/config"
+)
+
+type Server struct {
+	URL      string
+	Port     int
+	server   *http.Server
+	listener net.Listener
+}
+
+func NewServer(t *testing.T, comp *config.Complement, configFunc func(router *mux.Router)) *Server {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		t.Fatalf("Could not create listener for web server: %s", err)
+	}
+
+	port := listener.Addr().(*net.TCPAddr).Port
+
+	r := mux.NewRouter()
+
+	configFunc(r)
+
+	server := &http.Server{Addr: ":0", Handler: r}
+
+	go server.Serve(listener)
+
+	return &Server{
+		URL:      fmt.Sprintf("http://%s:%d", comp.HostnameRunningComplement, port),
+		Port:     port,
+		server:   server,
+		listener: listener,
+	}
+}
+
+func (s *Server) Close() {
+	s.server.Close()
+	s.listener.Close()
+}

sytest.ignored.list

@@ -35,7 +35,8 @@ New room members see first user's profile information in per-room initialSync
 A departed room is still included in /initialSync (SPEC-216)
 Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
 initialSync sees my presence status
-Global initialSyncGlobal initialSync with limit=0 gives no messages
+Global initialSync
+Global initialSync with limit=0 gives no messages
 Room initialSync
 Room initialSync with limit=0 gives no messages
 Read receipts are visible to /initialSync

tests/csapi/apidoc_login_test.go

@@ -43,14 +43,14 @@ func TestLogin(t *testing.T) {
 		// sytest: POST /login can log in as a user
 		t.Run("POST /login can login as user", func(t *testing.T) {
 			t.Parallel()
-			res := unauthedClient.MustDo(t, "POST", []string{"_matrix", "client", "v3", "login"}, json.RawMessage(`{
+			res := unauthedClient.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "login"}, client.WithRawBody(json.RawMessage(`{
 				"type": "m.login.password",
 				"identifier": {
 					"type": "m.id.user",
 					"user": "@test_login_user:hs1"
 				},
 				"password": "superuser"
-			}`))
+			}`)))
 
 			must.MatchResponse(t, res, match.HTTPResponse{
 				JSON: []match.JSON{
@@ -63,15 +63,15 @@ func TestLogin(t *testing.T) {
 		t.Run("POST /login returns the same device_id as that in the request", func(t *testing.T) {
 			t.Parallel()
 			deviceID := "test_device_id"
-			res := unauthedClient.MustDo(t, "POST", []string{"_matrix", "client", "v3", "login"}, json.RawMessage(`{
+			res := unauthedClient.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "login"}, client.WithRawBody(json.RawMessage(`{
 				"type": "m.login.password",
 				"identifier": {
 					"type": "m.id.user",
 					"user": "@test_login_user:hs1"
 				},
 				"password": "superuser",
 				"device_id": "`+deviceID+`"
-			}`))
+			}`)))
 
 			must.MatchResponse(t, res, match.HTTPResponse{
 				JSON: []match.JSON{
@@ -84,14 +84,14 @@ func TestLogin(t *testing.T) {
 		t.Run("POST /login can log in as a user with just the local part of the id", func(t *testing.T) {
 			t.Parallel()
 
-			res := unauthedClient.MustDo(t, "POST", []string{"_matrix", "client", "v3", "login"}, json.RawMessage(`{
+			res := unauthedClient.MustDoFunc(t, "POST", []string{"_matrix", "client", "v3", "login"}, client.WithRawBody(json.RawMessage(`{
 				"type": "m.login.password",
 				"identifier": {
 					"type": "m.id.user",
 					"user": "test_login_user"
 				},
 				"password": "superuser"
-			}`))
+			}`)))
 
 			must.MatchResponse(t, res, match.HTTPResponse{
 				JSON: []match.JSON{

tests/csapi/apidoc_room_state_test.go

@@ -1,6 +1,7 @@
 package csapi_tests
 
 import (
+	"net/http"
 	"net/url"
 	"testing"
 	"time"
@@ -11,9 +12,6 @@ import (
 	"github.com/matrix-org/complement/internal/client"
 	"github.com/matrix-org/complement/internal/match"
 	"github.com/matrix-org/complement/internal/must"
-	"github.com/matrix-org/complement/runtime"
-
-	"net/http"
 )
 
 func TestRoomState(t *testing.T) {
@@ -330,7 +328,6 @@ func TestRoomState(t *testing.T) {
 			})
 		})
 		t.Run("GET /rooms/:room_id/joined_members is forbidden after leaving room", func(t *testing.T) {
-			runtime.SkipIf(t, runtime.Dendrite) // https://github.com/matrix-org/complement/pull/424
 			t.Parallel()
 			roomID := authedClient.CreateRoom(t, map[string]interface{}{})
 			authedClient.LeaveRoom(t, roomID)

tests/csapi/apidoc_server_capabilities_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/matrix-org/complement/internal/b"
 	"github.com/matrix-org/complement/internal/match"
 	"github.com/matrix-org/complement/internal/must"
-	"github.com/tidwall/gjson"
 )
 
 func TestServerCapabilities(t *testing.T) {
@@ -19,13 +18,13 @@ func TestServerCapabilities(t *testing.T) {
 
 	// sytest: GET /capabilities is present and well formed for registered user
 	data := authedClient.GetCapabilities(t)
-	j := gjson.ParseBytes(data)
-	if !j.Get(`capabilities.m\.room_versions`).Exists() {
-		t.Fatal("expected m.room_versions not found")
-	}
-	if !j.Get(`capabilities.m\.change_password`).Exists() {
-		t.Fatal("expected m.change_password not found")
-	}
+
+	must.MatchJSONBytes(
+		t,
+		data,
+		match.JSONKeyPresent(`capabilities.m\.room_versions`),
+		match.JSONKeyPresent(`capabilities.m\.change_password`),
+	)
 
 	// sytest: GET /v3/capabilities is not public
 	res := unauthedClient.DoFunc(t, "GET", []string{"_matrix", "client", "v3", "capabilities"})

tests/csapi/device_lists_test.go

@@ -328,7 +328,7 @@ func TestDeviceListUpdates(t *testing.T) {
 	t.Run("when remote user leaves a room", func(t *testing.T) { testOtherUserLeave(t, deployment, "hs1", "hs2") })
 	t.Run("when leaving a room with a local user", func(t *testing.T) { testLeave(t, deployment, "hs1", "hs1") })
 	t.Run("when leaving a room with a remote user", func(t *testing.T) {
-		runtime.SkipIf(t, runtime.Synapse) // https://github.com/matrix-org/synapse/issues/13650
+		runtime.SkipIf(t, runtime.Synapse) // FIXME: https://github.com/matrix-org/synapse/issues/13650
 		testLeave(t, deployment, "hs1", "hs2")
 	})
 	t.Run("when local user rejoins a room", func(t *testing.T) { testOtherUserRejoin(t, deployment, "hs1", "hs1") })