From c6a93ed0b7540db999a30b9f50ef5c3b7eaa5a37 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:12:16 +0100 Subject: [PATCH 001/124] remove http API; add basic mautrix VoIP hooks --- go.mod | 12 +++-- go.sum | 25 +++++++++- main.go | 141 +++++++++++++++++++++++++++++++++----------------------- 3 files changed, 115 insertions(+), 63 deletions(-) diff --git a/go.mod b/go.mod index 806ffa5..f35732a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,13 @@ go 1.18 require github.com/pion/webrtc/v3 v3.1.31 +require ( + github.com/pion/rtcp v1.2.9 + github.com/rs/zerolog v1.26.1 + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + maunium.net/go/mautrix v0.11.0 +) + require ( github.com/google/uuid v1.3.0 // indirect github.com/pion/datachannel v1.5.2 // indirect @@ -13,7 +20,6 @@ require ( github.com/pion/logging v0.2.2 // indirect github.com/pion/mdns v0.0.5 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.9 // indirect github.com/pion/rtp v1.7.13 // indirect github.com/pion/sctp v1.8.2 // indirect github.com/pion/sdp/v3 v3.0.4 // indirect @@ -22,8 +28,8 @@ require ( github.com/pion/transport v0.13.0 // indirect github.com/pion/turn/v2 v2.0.8 // indirect github.com/pion/udp v0.1.1 // indirect - golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 // indirect - golang.org/x/net v0.0.0-20220401154927-543a649e0bdd // indirect + golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect + golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index 680dca0..51f17be 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,11 @@ +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -20,8 +22,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -72,8 +76,12 @@ github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o= github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= github.com/pion/webrtc/v3 v3.1.31 h1:lR/EIm8z7RGJaVJ7LcGdGxDrjNRyTAjg8YbASt4EYRU= github.com/pion/webrtc/v3 v3.1.31/go.mod h1:bcD6vrgcflr6lkf3E8VEqnQT7Uf7y1AxcdUWYGKER1w= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -82,12 +90,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c= +golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -95,15 +107,19 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220401154927-543a649e0bdd h1:zYlwaUHTmxuf6H7hwO2dgwqozQmH7zf4x+/qql4oVWc= golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220513224357-95641704303c h1:nF9mHSvoKBLkQNQhJZNsc66z2UzAMUbLGjC95CF3pU0= +golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -116,6 +132,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -127,6 +144,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -141,6 +159,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -150,3 +169,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +maunium.net/go/mautrix v0.11.0 h1:B1FBHcvE4Mud+AC+zgNQQOw0JxSVrt40watCejhVA7w= +maunium.net/go/mautrix v0.11.0/go.mod h1:K29EcHwsNg6r7fMfwvi0GHQ9o5wSjqB9+Q8RjCIQEjA= diff --git a/main.go b/main.go index d3b36b8..555eb2f 100644 --- a/main.go +++ b/main.go @@ -2,72 +2,104 @@ package main import ( "flag" + "fmt" + "github.com/rs/zerolog/log" + yaml "gopkg.in/yaml.v3" "io/ioutil" - "log" - "net" - "net/http" - "text/template" -) + "reflect" -func main() { - fociCount := flag.Int("foci-count", 4, "How many FOCI should be started") - httpAddress := flag.String("http-address", ":8080", "Address for frontend for FOCI cluster") - flag.Parse() - - fociPorts := []int{} - for i := 0; i < *fociCount; i++ { - fociPort, err := createFoci() - if err != nil { - log.Fatal(err) - } - log.Printf("Starting FOCI on port %d ", fociPort) + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) - fociPorts = append(fociPorts, fociPort) +func loadConfig(configFilePath string) (*config, error) { + log.Info().Msgf("loaded %s", configFilePath) + file, err := ioutil.ReadFile(configFilePath) + if err != nil { + log.Fatal().Err(err).Msg("Failed to read config") } + var config config + if err := yaml.Unmarshal(file, &config); err != nil { + return nil, fmt.Errorf("Failed to unmarshal YAML: %s", err) + } + return &config, nil +} - log.Print("Serving HTTP on " + *httpAddress) - fileServer := &http.Server{ - Addr: *httpAddress, - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - indexHTML, err := ioutil.ReadFile("static/index.html") - if err != nil { - panic(err) - } - - if err := template.Must(template.New("").Parse(string(indexHTML))).Execute(w, fociPorts); err != nil { - log.Fatal(err) - } +func main() { + configFilePath := flag.String("config", "config.yaml", "Configuration file path") + flag.Parse() - }), + var config *config + var err error + if config, err = loadConfig(*configFilePath); err != nil { + log.Fatal().Err(err).Msg("Failed to load config file") } - log.Fatal(fileServer.ListenAndServe()) -} -func createFoci() (int, error) { - listener, err := net.Listen("tcp", ":0") + client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) if err != nil { - return 0, err + log.Fatal().Err(err).Msg("Failed to create client") } - fociServer := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - setCorsHeaders(w) - if r.URL.String() != "/createSession" || r.Method != "POST" { - return - } + syncer := client.Syncer.(*mautrix.DefaultSyncer) + syncer.ParseEventContent = true - if err := handleCreateSession(w, r); err != nil { - log.Fatal(err) - } + // add to-device flavours of the call events to mautrix for MSC3401 + var ( + CallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} + CallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} + CallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} + CallReject = event.Type{"m.call.reject", event.ToDeviceEventType} + CallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} + CallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} + CallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} + ) + event.TypeMap[CallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) + event.TypeMap[CallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) + event.TypeMap[CallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) + event.TypeMap[CallReject] = reflect.TypeOf(event.CallRejectEventContent{}) + event.TypeMap[CallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) + event.TypeMap[CallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) + event.TypeMap[CallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) - }), - } + syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallReject, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallSelectAnswer, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallNegotiate, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) + syncer.OnEventType(CallHangup, func(_ mautrix.EventSource, event *event.Event) { + log.Info().Interface("event", event) + }) - go func() { - log.Fatal(fociServer.Serve(listener)) - }() + // TODO: actually hook up events and hook up sessions in the SFU + // if err := handleCreateSession(w, r); err != nil { + // log.Fatal(err) + // } - return listener.Addr().(*net.TCPAddr).Port, nil + err = client.Sync() + if err != nil { + log.Panic().Err(err).Msg("Sync failed") + } +} + +type config struct { + UserID id.UserID + HomeserverURL string + AccessToken string + DeviceID id.DeviceID } type dataChannelMessage struct { @@ -78,10 +110,3 @@ type dataChannelMessage struct { Purpose string `json:"purpose"` SDP string `json:"sdp"` } - -func setCorsHeaders(w http.ResponseWriter) { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") - w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization,X-CSRF-Token") - w.Header().Set("Access-Control-Expose-Headers", "Authorization") -} From 1795332aba2edfb6e68a0125d5dcc2bac79619ec Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:12:54 +0100 Subject: [PATCH 002/124] add sample config --- config.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 config.yaml diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..86efdc0 --- /dev/null +++ b/config.yaml @@ -0,0 +1,4 @@ +homeserverurl: "http://localhost:8008" +userid: "@sfu:shadowfax" +accesstoken: "..." +deviceid: "JHLVIHAPQU" From 5fe938d389aa07449eee1dad1471b0833b1afb04 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:13:46 +0100 Subject: [PATCH 003/124] rename sample config --- config.yaml => config.yaml.sample | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config.yaml => config.yaml.sample (100%) diff --git a/config.yaml b/config.yaml.sample similarity index 100% rename from config.yaml rename to config.yaml.sample From a81ddeaa0d4e1a982853f705c6d32f7da74b3143 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:14:45 +0100 Subject: [PATCH 004/124] .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4adbf11 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +config.yaml +sfu-to-sfu From 1ac4be04f407831a288e752c25bc9c2221a4c701 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:15:28 +0100 Subject: [PATCH 005/124] typo --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 555eb2f..7e90a58 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( ) func loadConfig(configFilePath string) (*config, error) { - log.Info().Msgf("loaded %s", configFilePath) + log.Info().Msgf("Loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { log.Fatal().Err(err).Msg("Failed to read config") From 1368d31562c245f02e35cd91127e486091638e3f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 31 May 2022 01:16:38 +0100 Subject: [PATCH 006/124] go fmt --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 7e90a58..93925e7 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,7 @@ package main import ( "flag" "fmt" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v3" "io/ioutil" "reflect" From 0a0e3088401f61a40b6197b984114798ac591389 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 16:04:22 +0100 Subject: [PATCH 007/124] theoretically hook up matrix to the focus (WIP) --- foci.go | 254 --------------------------------------------- focus.go | 303 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 90 +++------------- matrix.go | 110 ++++++++++++++++++++ 4 files changed, 429 insertions(+), 328 deletions(-) delete mode 100644 foci.go create mode 100644 focus.go create mode 100644 matrix.go diff --git a/foci.go b/foci.go deleted file mode 100644 index d670c63..0000000 --- a/foci.go +++ /dev/null @@ -1,254 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "strings" - "sync" - "time" - - "github.com/pion/rtcp" - "github.com/pion/webrtc/v3" -) - -type streamDetail struct { - callID, deviceID, purpose string - track *webrtc.TrackLocalStaticRTP -} - -type setStreamDetails func(newCallID, newDeviceID, newPurpose string) - -type foci struct { - name string - streamDetailsMu sync.RWMutex - streamDetails []streamDetail -} - -func (f *foci) localStreamLookup(msg dataChannelMessage) (audioTrack, videoTrack webrtc.TrackLocal) { - f.streamDetailsMu.Lock() - defer f.streamDetailsMu.Unlock() - - for _, s := range f.streamDetails { - if s.callID == msg.CallID && s.deviceID == msg.DeviceID && s.purpose == msg.Purpose { - if s.track.Kind() == webrtc.RTPCodecTypeAudio { - audioTrack = s.track - } else { - videoTrack = s.track - } - } - } - return -} - -func (f *foci) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel, setPublishDetails setStreamDetails) { - sendError := func(errMsg string) { - marshaled, err := json.Marshal(&dataChannelMessage{ - Event: "error", - Message: errMsg, - }) - if err != nil { - panic(err) - } - - if err = d.SendText(string(marshaled)); err != nil { - panic(err) - } - } - - d.OnMessage(func(m webrtc.DataChannelMessage) { - if !m.IsString { - log.Fatal("Inbound message is not string") - } - - msg := &dataChannelMessage{} - if err := json.Unmarshal(m.Data, msg); err != nil { - log.Fatal(err) - } - - switch msg.Event { - case "publish": - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }); err != nil { - panic(err) - } - - answer, err := peerConnection.CreateAnswer(nil) - if err != nil { - panic(err) - } - - if err := peerConnection.SetLocalDescription(answer); err != nil { - panic(err) - } - - setPublishDetails(msg.CallID, msg.DeviceID, msg.Purpose) - - msg.SDP = answer.SDP - marshaled, err := json.Marshal(msg) - if err != nil { - panic(err) - } - - if err = d.SendText(string(marshaled)); err != nil { - panic(err) - } - case "subscribe": - var audioTrack, videoTrack webrtc.TrackLocal - - if msg.FOCI == f.name { - audioTrack, videoTrack = f.localStreamLookup(*msg) - } else { - audioTrack, videoTrack = remoteStreamLookup(*msg) - } - - if audioTrack == nil && videoTrack == nil { - sendError("No Such Stream") - return - } - - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }); err != nil { - panic(err) - } - - if audioTrack != nil { - if _, err := peerConnection.AddTrack(audioTrack); err != nil { - panic(err) - } - } - - if videoTrack != nil { - if _, err := peerConnection.AddTrack(videoTrack); err != nil { - panic(err) - } - } - - answer, err := peerConnection.CreateAnswer(nil) - if err != nil { - panic(err) - } - - if err := peerConnection.SetLocalDescription(answer); err != nil { - panic(err) - } - - msg.SDP = answer.SDP - marshaled, err := json.Marshal(msg) - if err != nil { - panic(err) - } - - if err = d.SendText(string(marshaled)); err != nil { - panic(err) - } - default: - log.Fatalf("Unknown msg Event type %s", msg.Event) - } - }) -} - -func (f *foci) handleCreateSession(w http.ResponseWriter, r *http.Request) error { - offer, err := io.ReadAll(r.Body) - if err != nil { - return err - } - - peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) - if err != nil { - return err - } - - var ( - publishDetailsMu sync.RWMutex - callID, deviceID, purpose string - ) - setPublishDetails := func(newCallID, newDeviceID, newPurpose string) { - publishDetailsMu.Lock() - defer publishDetailsMu.Unlock() - - callID = newCallID - deviceID = newDeviceID - purpose = newPurpose - } - - peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { - id := "audio" - if strings.Contains(trackRemote.Codec().MimeType, "video") { - id = "video" - - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval - go func() { - ticker := time.NewTicker(time.Millisecond * 200) - for range ticker.C { - if errSend := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); errSend != nil { - fmt.Println(errSend) - } - } - }() - - } - - publishDetailsMu.Lock() - f.streamDetailsMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, id, fmt.Sprintf("%s-%s-%s", callID, deviceID, purpose)) - if err != nil { - panic(err) - } - - f.streamDetails = append(f.streamDetails, streamDetail{ - callID: callID, - deviceID: deviceID, - purpose: purpose, - track: trackLocal, - }) - f.streamDetailsMu.Unlock() - publishDetailsMu.Unlock() - - copyRemoteToLocal(trackRemote, trackLocal) - }) - - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - f.dataChannelHandler(peerConnection, d, setPublishDetails) - }) - - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: string(offer), - }) - - answer, err := peerConnection.CreateAnswer(nil) - if err != nil { - return err - } - - gatherComplete := webrtc.GatheringCompletePromise(peerConnection) - if err = peerConnection.SetLocalDescription(answer); err != nil { - return err - } - <-gatherComplete - - _, err = fmt.Fprintf(w, peerConnection.LocalDescription().SDP) - return err -} - -func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { - buff := make([]byte, 1500) - for { - i, _, err := trackRemote.Read(buff) - if err != nil { - panic(err) - } - - if _, err = trackLocal.Write(buff[:i]); err != nil { - panic(err) - } - } - -} diff --git a/focus.go b/focus.go new file mode 100644 index 0000000..7def3f9 --- /dev/null +++ b/focus.go @@ -0,0 +1,303 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync" + "time" + + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" +) + +type trackDetail struct { + call *call + trackID string + streamID string + track *webrtc.TrackLocalStaticRTP +} + +type setTrackDetails func(call *call, track *webrtc.TrackLocal) + +// stolen from matrix-js-sdk +// TODO: actually use callState (will be needed for renegotiation) +const ( + Fledgling = "fledgling" + InviteSent = "invite_sent" + WaitLocalMedia = "wait_local_media" + CreateOffer = "create_offer" + CreateAnswer = "create_answer" + Connecting = "connecting" + Connected = "connected" + Ringing = "ringing" + Ended = "ended" +) + +type callState string + +type call struct { + callID string + userID string + deviceID string + client *Client + peerConnection *peerConnection + callState callState + conf *conf + // we track the call's tracks via the conf object. +} + +type calls struct { + callsMu sync.RWMutex + calls map[string]*call +} + +type conf struct { + confID string + calls calls + trackDetailsMu sync.RWMutex + // FIXME: we should really index tracks by {streamID, trackID} + trackDetails map[string]*trackDetail // by trackID. +} + +type confs struct { + confsMu sync.RWMutex + confs map[string]*conf +} + +type focus struct { + name string + confs confs +} + +func (f *focus) getConf(confID string, create bool) (*conf, error) { + f.confs.confsMu.Lock() + defer f.confs.confsMu.Unlock() + conf := f.confs.confs[confID] + if conf == nil { + if create { + conf := &conf{ + confID: confID, + } + f.confs.confs[confID] = conf + } else { + return _, error("No such conf") + } + } + return &conf +} + +func (c *conf) getCall(callID string, create bool) (*call, error) { + c.calls.callsMu.Lock() + defer c.calls.callsMu.Unlock() + call := c.calls.calls[callID] + if call == nil { + if create { + call := &call{ + callID: callID, + conf: c, + callState: WaitLocalMedia, + } + c.calls.calls[callID] = call + } else { + return _, error("No such call") + } + } + return &conf +} + +func (c *conf) localTrackLookup(trackID string) (track webrtc.TrackLocal) { + c.trackDetailsMu.Lock() + defer c.trackDetailsMu.Unlock() + return trackDetails[trackID] +} + +func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { + sendError := func(errMsg string) { + marshaled, err := json.Marshal(&dataChannelMessage{ + Event: "error", + Message: errMsg, + }) + if err != nil { + panic(err) + } + + if err = d.SendText(string(marshaled)); err != nil { + panic(err) + } + } + + d.OnMessage(func(m webrtc.DataChannelMessage) { + if !m.IsString { + log.Fatal("Inbound message is not string") + } + + msg := &dataChannelMessage{} + if err := json.Unmarshal(m.Data, msg); err != nil { + log.Fatal(err) + } + + switch msg.Op { + case "select": + // XXX: do we actually need to call setRemoteDescription + // at this point, given it shouldn't have changed since the + // caller connected? + + for _, trackDesc := range msg.Start { + // TODO: we should really be indexed by streamID too here + track := c.localTrackLookup(trackDesc.TrackID) + + // TODO: hook cascade back up. + // As we're not an AS, we'd rely on the client + // to send us a "connect" op to tell us how to + // connect to another focus in order to select + // its streams. + + if track == nil { + sendError("No Such Track") + return + } + + if track != nil { + if _, err := peerConnection.AddTrack(track); err != nil { + panic(err) + } + } + } + + // TODO: hook up msg.Stop to unsubscribe from tracks + + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + + if err := peerConnection.SetLocalDescription(answer); err != nil { + panic(err) + } + + // XXX: do we actually have to send the answer back to th caller? + // if so, do we have a problem that the updated SDP is sent over + // slow to-device messages rather than directly over DC, as per + // Sean's original POC? In terms of speed of selection... + + default: + log.Fatalf("Unknown msg Event type %s", msg.Event) + } + }) +} + +func (c *call) onInvite(content *CallInviteEventContent) error { + offer := content.Offer + + peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + return err + } + c.peerConnection = peerConnection + + peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + id := "audio" + if strings.Contains(trackRemote.Codec().MimeType, "video") { + id = "video" + + // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + go func() { + ticker := time.NewTicker(time.Millisecond * 200) + for range ticker.C { + if errSend := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); errSend != nil { + fmt.Println(errSend) + } + } + }() + + } + + c.conf.trackDetailsMu.Lock() + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, id, fmt.Sprintf("%s-%s-%s", c.callID, c.deviceID, trackRemote.id)) + if err != nil { + panic(err) + } + + c.conf.trackDetails[trackRemote.id] = trackDetail{ + call: c, + track: trackLocal, + } + c.conf.trackDetailsMu.Unlock() + + copyRemoteToLocal(trackRemote, trackLocal) + }) + + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { + log.Print("onDataChannel", d) + //f.dataChannelHandler(peerConnection, d, setPublishDetails) + }) + + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: string(offer), + }) + + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + return err + } + + gatherComplete := webrtc.GatheringCompletePromise(peerConnection) + if err = peerConnection.SetLocalDescription(answer); err != nil { + return err + } + <-gatherComplete + + // TODO: send any subsequent candidates we discover to the peer + + answerSdp := peerConnection.LocalDescription().SDP + + // TODO: sessions + answerEvtContent := &event.Content{ + Parsed: event.CallAnswerEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + PartyID: c.client.DeviceID, + Version: 1, + Answer: answerSdp, + }, + } + + toDeviceAnswer := &mautrix.ReqSendToDevice{ + Messages: map[c.UserID]map[c.DeviceID]*event.Content{ + toUser: { + toDevice: answerEvtContent, + }, + }, + } + + // TODO: E2EE + // TODO: to-device reliability + c.client.SendToDevice(event.CallAnswer, toDeviceAnswer) + + return err +} + +func (c *call) onCandidates(content *CallCandidatesEventContent) error { + // TODO: tell our peerConnection about the new candidates we just discovered + log.Print("ignoring candidates as not yet implemented", content) +} + +func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { + buff := make([]byte, 1500) + for { + i, _, err := trackRemote.Read(buff) + if err != nil { + panic(err) + } + + if _, err = trackLocal.Write(buff[:i]); err != nil { + panic(err) + } + } + +} diff --git a/main.go b/main.go index 098cf68..609be79 100644 --- a/main.go +++ b/main.go @@ -3,21 +3,16 @@ package main import ( "flag" "fmt" - "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v3" "io/ioutil" - "reflect" - - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/event" - "maunium.net/go/mautrix/id" + "log" ) func loadConfig(configFilePath string) (*config, error) { - log.Info().Msgf("Loading %s", configFilePath) + log.Printf("Loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { - log.Fatal().Err(err).Msg("Failed to read config") + log.Fatal("Failed to read config", err) } var config config if err := yaml.Unmarshal(file, &config); err != nil { @@ -33,65 +28,11 @@ func main() { var config *config var err error if config, err = loadConfig(*configFilePath); err != nil { - log.Fatal().Err(err).Msg("Failed to load config file") + log.Fatal("Failed to load config file", err) } - client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) - if err != nil { - log.Fatal().Err(err).Msg("Failed to create client") - } - - syncer := client.Syncer.(*mautrix.DefaultSyncer) - syncer.ParseEventContent = true - - // add to-device flavours of the call events to mautrix for MSC3401 - var ( - CallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} - CallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} - CallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} - CallReject = event.Type{"m.call.reject", event.ToDeviceEventType} - CallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} - CallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} - CallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} - ) - event.TypeMap[CallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) - event.TypeMap[CallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) - event.TypeMap[CallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) - event.TypeMap[CallReject] = reflect.TypeOf(event.CallRejectEventContent{}) - event.TypeMap[CallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) - event.TypeMap[CallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) - event.TypeMap[CallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) - - syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallReject, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallSelectAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallNegotiate, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - syncer.OnEventType(CallHangup, func(_ mautrix.EventSource, event *event.Event) { - log.Info().Interface("event", event) - }) - - // TODO: actually hook up events and hook up sessions in the SFU - // if err := handleCreateSession(w, r); err != nil { - // log.Fatal(err) - // } - - err = client.Sync() - if err != nil { - log.Panic().Err(err).Msg("Sync failed") + if err := initMatrix(config); err != nil { + log.Fatal("Failed to init Matrix", err) } } @@ -99,16 +40,17 @@ type config struct { UserID id.UserID HomeserverURL string AccessToken string - DeviceID id.DeviceID +} + +type trackDesc struct { + streamID string `json:"stream_id"` + trackID string `json:"track_id"` } type dataChannelMessage struct { - Event string `json:"event"` - Message string `json:"message,omitempty"` - ID string `json:"id"` - CallID string `json:"call_id"` - DeviceID string `json:"device_id"` - Purpose string `json:"purpose"` - SDP string `json:"sdp"` - FOCI string `json:"foci"` + Op string `json:"op"` + // XXX: is this even needed? we know which conf a given call is for... + ConfID string `json:"conf_id"` + Start []TrackDesc `json:"start"` + Stop []TrackDesc `json:"top"` } diff --git a/matrix.go b/matrix.go new file mode 100644 index 0000000..20d8dc3 --- /dev/null +++ b/matrix.go @@ -0,0 +1,110 @@ +package main + +import ( + "log" + "reflect" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func initMatrix(config config) error { + client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) + if err != nil { + log.Fatal("Failed to create client", err) + } + + whoami, err := client.Whoami() + if err != nil { + log.Fatal("Failed to identify SFU user", err) + } + if config.UserID != whoami.UserID { + log.Fatalf("Access token is for the wrong user: %s", config.UserID) + } + client.DeviceID = whoami.DeviceID + + focus := &focus{ + name: fmt.printf("%s (%s)", config.UserID, config.DeviceID), + } + + syncer := client.Syncer.(*mautrix.DefaultSyncer) + syncer.ParseEventContent = true + + // add to-device flavours of the call events to mautrix for MSC3401 + var ( + CallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} + CallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} + CallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} + CallReject = event.Type{"m.call.reject", event.ToDeviceEventType} + CallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} + CallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} + CallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} + ) + event.TypeMap[CallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) + event.TypeMap[CallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) + event.TypeMap[CallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) + event.TypeMap[CallReject] = reflect.TypeOf(event.CallRejectEventContent{}) + event.TypeMap[CallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) + event.TypeMap[CallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) + event.TypeMap[CallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) + + // TODO: E2EE + + syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + invite := event.Content.AsCallInvite() + conf := focus.getConf(invite.ConfID, true) + call := conf.getCall(invite.CallID, true) + call.userID = event.Sender + call.deviceID = invite.DeviceID + // TODO: check session IDs + call.onInvite(event) + }) + + syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + if conf := focus.getConf(event.ConfID); err != nil { + log.Printf("Got candidates for unknown conf %s", event.ConfID) + return + } + if call := conf.getCall(event.CallID); err != nil { + log.Printf("Got candidates for unknown call %s in conf %s", event.CallID, event.ConfID) + return + } + call.onCandidates(event.Content.AsCallCandidates()) + }) + + syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + // until we have cascading hooked up, we should never be receiving answer events + log.Printf("Ignoring unexpected answer event") + }) + + syncer.OnEventType(CallReject, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + // until we have cascading hooked up, we should never be receiving reject events + log.Printf("Ignoring unexpected reject event") + }) + + syncer.OnEventType(CallSelectAnswer, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + // until we have cascading hooked up, we should never be receiving answer events + log.Printf("Ignoring unexpected select answer event") + }) + + syncer.OnEventType(CallNegotiate, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + // TODO: process SDP renegotiation + }) + + syncer.OnEventType(CallHangup, func(_ mautrix.EventSource, event *event.Event) { + log.Print("event", event) + // TODO: process hangups + }) + + err = client.Sync() + if err != nil { + log.Panic("Sync failed", err) + } +} From 36b5b098a058286bd2f85c83279cb55dddb0d446 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 16:48:25 +0100 Subject: [PATCH 008/124] we can infer device_id from /whoami --- config.yaml.sample | 1 - 1 file changed, 1 deletion(-) diff --git a/config.yaml.sample b/config.yaml.sample index 86efdc0..fa19e40 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -1,4 +1,3 @@ homeserverurl: "http://localhost:8008" userid: "@sfu:shadowfax" accesstoken: "..." -deviceid: "JHLVIHAPQU" From 0586e3833949e04df3aed2d60171d52bdfd2e04d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 17:00:27 +0100 Subject: [PATCH 009/124] pin to right go-mautrix --- go.mod | 5 +++-- go.sum | 21 ++++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index f35732a..e0d371c 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,12 @@ require github.com/pion/webrtc/v3 v3.1.31 require ( github.com/pion/rtcp v1.2.9 - github.com/rs/zerolog v1.26.1 - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + gopkg.in/yaml.v3 v3.0.0 maunium.net/go/mautrix v0.11.0 ) +replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 + require ( github.com/google/uuid v1.3.0 // indirect github.com/pion/datachannel v1.5.2 // indirect diff --git a/go.sum b/go.sum index 51f17be..78ed01b 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,9 @@ -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -27,6 +25,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 h1:jbtDwQghcjLNuAoSiOotX8GBxATrIkmBnzT1hkYrlJo= +github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5/go.mod h1:CiKpMhAx5QZFHK03jpWb0iKI3sGU8x6+LfsOjDrcO8I= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -76,12 +76,8 @@ github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o= github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= github.com/pion/webrtc/v3 v3.1.31 h1:lR/EIm8z7RGJaVJ7LcGdGxDrjNRyTAjg8YbASt4EYRU= github.com/pion/webrtc/v3 v3.1.31/go.mod h1:bcD6vrgcflr6lkf3E8VEqnQT7Uf7y1AxcdUWYGKER1w= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -90,16 +86,13 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c= golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -107,9 +100,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -119,7 +110,6 @@ golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -132,7 +122,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -144,7 +133,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -167,7 +155,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -maunium.net/go/mautrix v0.11.0 h1:B1FBHcvE4Mud+AC+zgNQQOw0JxSVrt40watCejhVA7w= -maunium.net/go/mautrix v0.11.0/go.mod h1:K29EcHwsNg6r7fMfwvi0GHQ9o5wSjqB9+Q8RjCIQEjA= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From f604f239b69a27ab9a61638100fc4bcc442cc84b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 19:14:35 +0100 Subject: [PATCH 010/124] fix build --- cascade.go | 2 + focus.go | 110 +++++++++++++++++++++++++++++++---------------------- go.mod | 4 +- main.go | 5 ++- matrix.go | 28 ++++++++------ 5 files changed, 90 insertions(+), 59 deletions(-) diff --git a/cascade.go b/cascade.go index 461ad6e..a26c5dd 100644 --- a/cascade.go +++ b/cascade.go @@ -1,5 +1,6 @@ package main +/* import ( "bytes" "encoding/json" @@ -126,3 +127,4 @@ func remoteStreamLookup(msg dataChannelMessage) (webrtc.TrackLocal, webrtc.Track return <-audioTrack, <-videoTrack } +*/ \ No newline at end of file diff --git a/focus.go b/focus.go index 7def3f9..413176b 100644 --- a/focus.go +++ b/focus.go @@ -3,12 +3,15 @@ package main import ( "encoding/json" "fmt" - "io" "log" - "net/http" "strings" "sync" "time" + "errors" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" "github.com/pion/rtcp" "github.com/pion/webrtc/v3" @@ -41,10 +44,10 @@ type callState string type call struct { callID string - userID string - deviceID string - client *Client - peerConnection *peerConnection + userID id.UserID + deviceID id.DeviceID + client *mautrix.Client + peerConnection *webrtc.PeerConnection callState callState conf *conf // we track the call's tracks via the conf object. @@ -55,12 +58,17 @@ type calls struct { calls map[string]*call } +// FIXME: for uniqueness, should we index tracks by {callID, streamID, trackID}? +type trackKey struct { + streamID string + trackID string +} + type conf struct { confID string calls calls trackDetailsMu sync.RWMutex - // FIXME: we should really index tracks by {streamID, trackID} - trackDetails map[string]*trackDetail // by trackID. + trackDetails map[trackKey]*trackDetail // by trackID. } type confs struct { @@ -76,49 +84,52 @@ type focus struct { func (f *focus) getConf(confID string, create bool) (*conf, error) { f.confs.confsMu.Lock() defer f.confs.confsMu.Unlock() - conf := f.confs.confs[confID] - if conf == nil { + co := f.confs.confs[confID] + if co == nil { if create { - conf := &conf{ + co := conf{ confID: confID, } - f.confs.confs[confID] = conf + f.confs.confs[confID] = &co } else { - return _, error("No such conf") + return nil, errors.New("No such conf") } } - return &conf + return co, nil } func (c *conf) getCall(callID string, create bool) (*call, error) { c.calls.callsMu.Lock() defer c.calls.callsMu.Unlock() - call := c.calls.calls[callID] - if call == nil { + ca := c.calls.calls[callID] + if ca == nil { if create { - call := &call{ + ca := call{ callID: callID, conf: c, callState: WaitLocalMedia, } - c.calls.calls[callID] = call + c.calls.calls[callID] = &ca } else { - return _, error("No such call") + return nil, errors.New("No such call") } } - return &conf + return ca, nil } -func (c *conf) localTrackLookup(trackID string) (track webrtc.TrackLocal) { +func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLocal) { c.trackDetailsMu.Lock() defer c.trackDetailsMu.Unlock() - return trackDetails[trackID] + return c.trackDetails[trackKey{ + streamID: streamID, + trackID: trackID, + }].track } func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { sendError := func(errMsg string) { marshaled, err := json.Marshal(&dataChannelMessage{ - Event: "error", + Op: "error", Message: errMsg, }) if err != nil { @@ -142,13 +153,12 @@ func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webr switch msg.Op { case "select": - // XXX: do we actually need to call setRemoteDescription - // at this point, given it shouldn't have changed since the - // caller connected? + // TODO: call setRemoteDescription so we can negotiate the new track. + // where do we get the SDP from? should the DC include it, like + // in the original PoC? for _, trackDesc := range msg.Start { - // TODO: we should really be indexed by streamID too here - track := c.localTrackLookup(trackDesc.TrackID) + track := c.localTrackLookup(trackDesc.streamID, trackDesc.trackID) // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -179,18 +189,17 @@ func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webr panic(err) } - // XXX: do we actually have to send the answer back to th caller? - // if so, do we have a problem that the updated SDP is sent over - // slow to-device messages rather than directly over DC, as per - // Sean's original POC? In terms of speed of selection... + // TODO: send the answer back to the caller. + // XXX: ideally we would do this over DC rather than slow to-device messaging. + // or perhaps use a pool of tracks to avoid having to renegotiate default: - log.Fatalf("Unknown msg Event type %s", msg.Event) + log.Fatalf("Unknown operation %s", msg.Op) } }) } -func (c *call) onInvite(content *CallInviteEventContent) error { +func (c *call) onInvite(content *event.CallInviteEventContent) error { offer := content.Offer peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) @@ -217,12 +226,15 @@ func (c *call) onInvite(content *CallInviteEventContent) error { } c.conf.trackDetailsMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, id, fmt.Sprintf("%s-%s-%s", c.callID, c.deviceID, trackRemote.id)) + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, id, fmt.Sprintf("%s-%s-%s", c.callID, c.deviceID, trackRemote.ID())) if err != nil { panic(err) } - c.conf.trackDetails[trackRemote.id] = trackDetail{ + c.conf.trackDetails[trackKey{ + streamID: trackRemote.StreamID(), + trackID: trackRemote.ID(), + }] = &trackDetail{ call: c, track: trackLocal, } @@ -238,7 +250,7 @@ func (c *call) onInvite(content *CallInviteEventContent) error { peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, - SDP: string(offer), + SDP: offer.SDP, }) answer, err := peerConnection.CreateAnswer(nil) @@ -259,18 +271,23 @@ func (c *call) onInvite(content *CallInviteEventContent) error { // TODO: sessions answerEvtContent := &event.Content{ Parsed: event.CallAnswerEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - PartyID: c.client.DeviceID, - Version: 1, - Answer: answerSdp, + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Answer: event.CallData{ + Type: "answer", + SDP: answerSdp, + }, }, } toDeviceAnswer := &mautrix.ReqSendToDevice{ - Messages: map[c.UserID]map[c.DeviceID]*event.Content{ - toUser: { - toDevice: answerEvtContent, + Messages: map[id.UserID]map[id.DeviceID]*event.Content{ + c.userID: { + c.deviceID: answerEvtContent, }, }, } @@ -282,9 +299,10 @@ func (c *call) onInvite(content *CallInviteEventContent) error { return err } -func (c *call) onCandidates(content *CallCandidatesEventContent) error { +func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { // TODO: tell our peerConnection about the new candidates we just discovered log.Print("ignoring candidates as not yet implemented", content) + return nil } func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { diff --git a/go.mod b/go.mod index e0d371c..dacfd00 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,9 @@ require ( maunium.net/go/mautrix v0.11.0 ) -replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 +// replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 + +replace maunium.net/go/mautrix v0.11.0 => ../mautrix-go require ( github.com/google/uuid v1.3.0 // indirect diff --git a/main.go b/main.go index 609be79..2cdec86 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( yaml "gopkg.in/yaml.v3" "io/ioutil" "log" + + "maunium.net/go/mautrix/id" ) func loadConfig(configFilePath string) (*config, error) { @@ -42,7 +44,7 @@ type config struct { AccessToken string } -type trackDesc struct { +type TrackDesc struct { streamID string `json:"stream_id"` trackID string `json:"track_id"` } @@ -50,6 +52,7 @@ type trackDesc struct { type dataChannelMessage struct { Op string `json:"op"` // XXX: is this even needed? we know which conf a given call is for... + Message string `json:"message"` ConfID string `json:"conf_id"` Start []TrackDesc `json:"start"` Stop []TrackDesc `json:"top"` diff --git a/matrix.go b/matrix.go index 20d8dc3..fd5419e 100644 --- a/matrix.go +++ b/matrix.go @@ -1,15 +1,15 @@ package main import ( + "fmt" "log" "reflect" "maunium.net/go/mautrix" "maunium.net/go/mautrix/event" - "maunium.net/go/mautrix/id" ) -func initMatrix(config config) error { +func initMatrix(config *config) error { client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) if err != nil { log.Fatal("Failed to create client", err) @@ -25,7 +25,7 @@ func initMatrix(config config) error { client.DeviceID = whoami.DeviceID focus := &focus{ - name: fmt.printf("%s (%s)", config.UserID, config.DeviceID), + name: fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID), } syncer := client.Syncer.(*mautrix.DefaultSyncer) @@ -54,25 +54,29 @@ func initMatrix(config config) error { syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { log.Print("event", event) invite := event.Content.AsCallInvite() - conf := focus.getConf(invite.ConfID, true) - call := conf.getCall(invite.CallID, true) + conf, _ := focus.getConf(invite.ConfID, true) + call, _ := conf.getCall(invite.CallID, true) call.userID = event.Sender call.deviceID = invite.DeviceID // TODO: check session IDs - call.onInvite(event) + call.onInvite(invite) }) syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { log.Print("event", event) - if conf := focus.getConf(event.ConfID); err != nil { - log.Printf("Got candidates for unknown conf %s", event.ConfID) + candidates := event.Content.AsCallCandidates() + var conf *conf + var call *call + var err error + if conf, err = focus.getConf(candidates.ConfID, false); err != nil { + log.Printf("Got candidates for unknown conf %s", candidates.ConfID) return } - if call := conf.getCall(event.CallID); err != nil { - log.Printf("Got candidates for unknown call %s in conf %s", event.CallID, event.ConfID) + if call, err = conf.getCall(candidates.CallID, false); err != nil { + log.Printf("Got candidates for unknown call %s in conf %s", candidates.CallID, candidates.ConfID) return } - call.onCandidates(event.Content.AsCallCandidates()) + call.onCandidates(candidates) }) syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { @@ -107,4 +111,6 @@ func initMatrix(config config) error { if err != nil { log.Panic("Sync failed", err) } + + return nil } From 5af7532dbff6493cba3120a087ee245f95c07e2f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 19:17:14 +0100 Subject: [PATCH 011/124] fix dep --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index dacfd00..cb4fab1 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,7 @@ require ( maunium.net/go/mautrix v0.11.0 ) -// replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 - -replace maunium.net/go/mautrix v0.11.0 => ../mautrix-go +replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4 require ( github.com/google/uuid v1.3.0 // indirect diff --git a/go.sum b/go.sum index 78ed01b..e2d2ad4 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5 h1:jbtDwQghcjLNuAoSiOotX8GBxATrIkmBnzT1hkYrlJo= -github.com/matrix-org/mautrix-go v0.11.1-0.20220603155839-5467b9cac1a5/go.mod h1:CiKpMhAx5QZFHK03jpWb0iKI3sGU8x6+LfsOjDrcO8I= +github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4 h1:FzlLAmPztXGN0gufupwO7gI78J+6MDGiEzL1fnJXtXo= +github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4/go.mod h1:CiKpMhAx5QZFHK03jpWb0iKI3sGU8x6+LfsOjDrcO8I= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From 782bcd2b34833e9834aed85032a37192136fcee1 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 20:32:04 +0100 Subject: [PATCH 012/124] hook up SDP-over-DC signalling again --- cascade.go | 2 +- focus.go | 38 +++++++++++++++++++++++++------------- main.go | 12 +++++++----- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/cascade.go b/cascade.go index a26c5dd..51ea4fc 100644 --- a/cascade.go +++ b/cascade.go @@ -127,4 +127,4 @@ func remoteStreamLookup(msg dataChannelMessage) (webrtc.TrackLocal, webrtc.Track return <-audioTrack, <-videoTrack } -*/ \ No newline at end of file +*/ diff --git a/focus.go b/focus.go index 413176b..8236e7e 100644 --- a/focus.go +++ b/focus.go @@ -2,12 +2,12 @@ package main import ( "encoding/json" + "errors" "fmt" "log" "strings" "sync" "time" - "errors" "maunium.net/go/mautrix" "maunium.net/go/mautrix/event" @@ -61,14 +61,14 @@ type calls struct { // FIXME: for uniqueness, should we index tracks by {callID, streamID, trackID}? type trackKey struct { streamID string - trackID string + trackID string } type conf struct { confID string calls calls trackDetailsMu sync.RWMutex - trackDetails map[trackKey]*trackDetail // by trackID. + trackDetails map[trackKey]*trackDetail // by trackID. } type confs struct { @@ -122,14 +122,14 @@ func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLoc defer c.trackDetailsMu.Unlock() return c.trackDetails[trackKey{ streamID: streamID, - trackID: trackID, + trackID: trackID, }].track } func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { sendError := func(errMsg string) { marshaled, err := json.Marshal(&dataChannelMessage{ - Op: "error", + Op: "error", Message: errMsg, }) if err != nil { @@ -153,9 +153,12 @@ func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webr switch msg.Op { case "select": - // TODO: call setRemoteDescription so we can negotiate the new track. - // where do we get the SDP from? should the DC include it, like - // in the original PoC? + if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: msg.SDP, + }); err != nil { + panic(err) + } for _, trackDesc := range msg.Start { track := c.localTrackLookup(trackDesc.streamID, trackDesc.trackID) @@ -189,10 +192,19 @@ func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webr panic(err) } - // TODO: send the answer back to the caller. - // XXX: ideally we would do this over DC rather than slow to-device messaging. - // or perhaps use a pool of tracks to avoid having to renegotiate + response := dataChannelMessage{ + Op: "answer", + ID: msg.ID, + SDP: answer.SDP, + } + marshaled, err := json.Marshal(response) + if err != nil { + panic(err) + } + if err = d.SendText(string(marshaled)); err != nil { + panic(err) + } default: log.Fatalf("Unknown operation %s", msg.Op) } @@ -233,7 +245,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.conf.trackDetails[trackKey{ streamID: trackRemote.StreamID(), - trackID: trackRemote.ID(), + trackID: trackRemote.ID(), }] = &trackDetail{ call: c, track: trackLocal, @@ -279,7 +291,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }, Answer: event.CallData{ Type: "answer", - SDP: answerSdp, + SDP: answerSdp, }, }, } diff --git a/main.go b/main.go index 2cdec86..1f65b1a 100644 --- a/main.go +++ b/main.go @@ -50,10 +50,12 @@ type TrackDesc struct { } type dataChannelMessage struct { - Op string `json:"op"` + Op string `json:"op"` + ID string `json:"id"` + Message string `json:"message,omitempty` // XXX: is this even needed? we know which conf a given call is for... - Message string `json:"message"` - ConfID string `json:"conf_id"` - Start []TrackDesc `json:"start"` - Stop []TrackDesc `json:"top"` + ConfID string `json:"conf_id,omitempty"` + Start []TrackDesc `json:"start,omitempty"` + Stop []TrackDesc `json:"stop,omitempty"` + SDP string `json:"sdp,omitempty"` } From e9844a49d8936f431651bac2e13e174791fc3372 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 21:39:01 +0100 Subject: [PATCH 013/124] hook up sfu->client ice candidates --- focus.go | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/focus.go b/focus.go index 8236e7e..d5f2b64 100644 --- a/focus.go +++ b/focus.go @@ -126,7 +126,7 @@ func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLoc }].track } -func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { +func (c *call) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { sendError := func(errMsg string) { marshaled, err := json.Marshal(&dataChannelMessage{ Op: "error", @@ -161,7 +161,7 @@ func (c *conf) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webr } for _, trackDesc := range msg.Start { - track := c.localTrackLookup(trackDesc.streamID, trackDesc.trackID) + track := c.conf.localTrackLookup(trackDesc.streamID, trackDesc.trackID) // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -256,8 +256,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }) peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - log.Print("onDataChannel", d) - //f.dataChannelHandler(peerConnection, d, setPublishDetails) + c.dataChannelHandler(peerConnection, d) }) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -276,6 +275,42 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } <-gatherComplete + peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + ice := candidate.ToJSON() + + // TODO: batch these up a bit + candidateEvtContent := &event.Content{ + Parsed: event.CallCandidatesEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Candidates: []event.CallCandidate{ + event.CallCandidate{ + Candidate: ice.Candidate, + SDPMLineIndex: int(*ice.SDPMLineIndex), + SDPMID: *ice.SDPMid, + // XXX: what about ice.UsernameFragment? + }, + }, + }, + } + + toDevice := &mautrix.ReqSendToDevice{ + Messages: map[id.UserID]map[id.DeviceID]*event.Content{ + c.userID: { + c.deviceID: candidateEvtContent, + }, + }, + } + + // TODO: E2EE + // TODO: to-device reliability + c.client.SendToDevice(event.CallCandidates, toDevice) + }) + // TODO: send any subsequent candidates we discover to the peer answerSdp := peerConnection.LocalDescription().SDP @@ -296,7 +331,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }, } - toDeviceAnswer := &mautrix.ReqSendToDevice{ + toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ c.userID: { c.deviceID: answerEvtContent, @@ -306,7 +341,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { // TODO: E2EE // TODO: to-device reliability - c.client.SendToDevice(event.CallAnswer, toDeviceAnswer) + c.client.SendToDevice(event.CallAnswer, toDevice) return err } From 9129b4e23132f8434f20c23c2b282a56ae5cbae8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Jun 2022 21:57:07 +0100 Subject: [PATCH 014/124] hook up client->sfu candidates --- focus.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/focus.go b/focus.go index d5f2b64..d822730 100644 --- a/focus.go +++ b/focus.go @@ -126,7 +126,9 @@ func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLoc }].track } -func (c *call) dataChannelHandler(peerConnection *webrtc.PeerConnection, d *webrtc.DataChannel) { +func (c *call) dataChannelHandler(d *webrtc.DataChannel) { + peerConnection := c.peerConnection + sendError := func(errMsg string) { marshaled, err := json.Marshal(&dataChannelMessage{ Op: "error", @@ -256,7 +258,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }) peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - c.dataChannelHandler(peerConnection, d) + c.dataChannelHandler(d) }) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -347,8 +349,18 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { - // TODO: tell our peerConnection about the new candidates we just discovered - log.Print("ignoring candidates as not yet implemented", content) + for _, candidate := range content.Candidates { + sdpMLineIndex := uint16(candidate.SDPMLineIndex) + ice := webrtc.ICECandidateInit{ + Candidate: candidate.Candidate, + SDPMLineIndex: &sdpMLineIndex, + SDPMid: &candidate.SDPMID, + } + if err := c.peerConnection.AddICECandidate(ice); err != nil { + log.Printf("Failed to add ICE candidate", content) + return err + } + } return nil } From 79f887ac8855691d390018ed3e47dfac9f6828b4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 01:06:23 +0100 Subject: [PATCH 015/124] whack up the debug --- matrix.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/matrix.go b/matrix.go index fd5419e..0d4d8c3 100644 --- a/matrix.go +++ b/matrix.go @@ -52,7 +52,7 @@ func initMatrix(config *config) error { // TODO: E2EE syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) invite := event.Content.AsCallInvite() conf, _ := focus.getConf(invite.ConfID, true) call, _ := conf.getCall(invite.CallID, true) @@ -63,7 +63,7 @@ func initMatrix(config *config) error { }) syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) candidates := event.Content.AsCallCandidates() var conf *conf var call *call @@ -80,33 +80,42 @@ func initMatrix(config *config) error { }) syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) // until we have cascading hooked up, we should never be receiving answer events - log.Printf("Ignoring unexpected answer event") + log.Print("Ignoring unexpected answer event") }) syncer.OnEventType(CallReject, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) // until we have cascading hooked up, we should never be receiving reject events - log.Printf("Ignoring unexpected reject event") + log.Print("Ignoring unexpected reject event") }) syncer.OnEventType(CallSelectAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) // until we have cascading hooked up, we should never be receiving answer events - log.Printf("Ignoring unexpected select answer event") + log.Print("Ignoring unexpected select answer event") }) syncer.OnEventType(CallNegotiate, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) // TODO: process SDP renegotiation }) syncer.OnEventType(CallHangup, func(_ mautrix.EventSource, event *event.Event) { - log.Print("event", event) + log.Printf("event %+v", event) // TODO: process hangups }) + syncer.OnEvent(func(source mautrix.EventSource, evt *event.Event) { + log.Printf("onEvent %+v %+v", source, evt); + }) + + syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { + log.Printf("synced %+v %+v", resp, since); + return true + }) + err = client.Sync() if err != nil { log.Panic("Sync failed", err) From 5f1dc7242ba8062fe9445caa7417b5e95cd50a1c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 13:32:54 +0100 Subject: [PATCH 016/124] handle todevice msgs correctly --- matrix.go | 108 ++++++++++++++++++++++++------------------------------ 1 file changed, 48 insertions(+), 60 deletions(-) diff --git a/matrix.go b/matrix.go index 0d4d8c3..96e1c74 100644 --- a/matrix.go +++ b/matrix.go @@ -22,6 +22,7 @@ func initMatrix(config *config) error { if config.UserID != whoami.UserID { log.Fatalf("Access token is for the wrong user: %s", config.UserID) } + log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID focus := &focus{ @@ -51,68 +52,55 @@ func initMatrix(config *config) error { // TODO: E2EE - syncer.OnEventType(CallInvite, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - invite := event.Content.AsCallInvite() - conf, _ := focus.getConf(invite.ConfID, true) - call, _ := conf.getCall(invite.CallID, true) - call.userID = event.Sender - call.deviceID = invite.DeviceID - // TODO: check session IDs - call.onInvite(invite) - }) - - syncer.OnEventType(CallCandidates, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - candidates := event.Content.AsCallCandidates() - var conf *conf - var call *call - var err error - if conf, err = focus.getConf(candidates.ConfID, false); err != nil { - log.Printf("Got candidates for unknown conf %s", candidates.ConfID) - return - } - if call, err = conf.getCall(candidates.CallID, false); err != nil { - log.Printf("Got candidates for unknown call %s in conf %s", candidates.CallID, candidates.ConfID) - return + syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { + log.Printf("synced %+v %+v", resp, since) + + for _, evt := range resp.ToDevice.Events { + evt.Type.Class = event.ToDeviceEventType + err := evt.Content.ParseRaw(evt.Type) + if err != nil { + log.Printf("Failed to parse to-device event of type %s: %v", evt.Type.Type, err) + continue + } + log.Printf("event %+v", evt) + switch evt.Type.Type { + case CallInvite.Type: + invite := evt.Content.AsCallInvite() + conf, _ := focus.getConf(invite.ConfID, true) + call, _ := conf.getCall(invite.CallID, true) + call.userID = evt.Sender + call.deviceID = invite.DeviceID + // TODO: check session IDs + call.onInvite(invite) + case CallCandidates.Type: + candidates := evt.Content.AsCallCandidates() + var conf *conf + var call *call + var err error + if conf, err = focus.getConf(candidates.ConfID, false); err != nil { + log.Printf("Got candidates for unknown conf %s", candidates.ConfID) + return true + } + if call, err = conf.getCall(candidates.CallID, false); err != nil { + log.Printf("Got candidates for unknown call %s in conf %s", candidates.CallID, candidates.ConfID) + return true + } + call.onCandidates(candidates) + case CallAnswer.Type: + log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + case CallReject.Type: + log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + case CallSelectAnswer.Type: + log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + case CallNegotiate.Type: + log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + case CallHangup.Type: + log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + default: + log.Printf("Ignoring unrecognised to-device event of type %s", evt.Type.Type) + } } - call.onCandidates(candidates) - }) - - syncer.OnEventType(CallAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - // until we have cascading hooked up, we should never be receiving answer events - log.Print("Ignoring unexpected answer event") - }) - - syncer.OnEventType(CallReject, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - // until we have cascading hooked up, we should never be receiving reject events - log.Print("Ignoring unexpected reject event") - }) - syncer.OnEventType(CallSelectAnswer, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - // until we have cascading hooked up, we should never be receiving answer events - log.Print("Ignoring unexpected select answer event") - }) - - syncer.OnEventType(CallNegotiate, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - // TODO: process SDP renegotiation - }) - - syncer.OnEventType(CallHangup, func(_ mautrix.EventSource, event *event.Event) { - log.Printf("event %+v", event) - // TODO: process hangups - }) - - syncer.OnEvent(func(source mautrix.EventSource, evt *event.Event) { - log.Printf("onEvent %+v %+v", source, evt); - }) - - syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { - log.Printf("synced %+v %+v", resp, since); return true }) From 6e1eb291cf03194793611e0ddc58711d34c82bf7 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 14:07:50 +0100 Subject: [PATCH 017/124] fix stupid map lifecycle bugs --- focus.go | 14 ++++++++++---- matrix.go | 37 +++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/focus.go b/focus.go index d822730..fb848db 100644 --- a/focus.go +++ b/focus.go @@ -81,16 +81,22 @@ type focus struct { confs confs } +func (f *focus) Init(name string) { + f.name = name + f.confs.confs = make(map[string]*conf) +} + func (f *focus) getConf(confID string, create bool) (*conf, error) { f.confs.confsMu.Lock() defer f.confs.confsMu.Unlock() co := f.confs.confs[confID] if co == nil { if create { - co := conf{ + co = &conf{ confID: confID, } - f.confs.confs[confID] = &co + f.confs.confs[confID] = co + co.calls.calls = make(map[string]*call) } else { return nil, errors.New("No such conf") } @@ -104,12 +110,12 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { ca := c.calls.calls[callID] if ca == nil { if create { - ca := call{ + ca = &call{ callID: callID, conf: c, callState: WaitLocalMedia, } - c.calls.calls[callID] = &ca + c.calls.calls[callID] = ca } else { return nil, errors.New("No such call") } diff --git a/matrix.go b/matrix.go index 96e1c74..c0d1ac6 100644 --- a/matrix.go +++ b/matrix.go @@ -25,9 +25,8 @@ func initMatrix(config *config) error { log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID - focus := &focus{ - name: fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID), - } + focus := new(focus) + focus.Init(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true @@ -53,7 +52,7 @@ func initMatrix(config *config) error { // TODO: E2EE syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { - log.Printf("synced %+v %+v", resp, since) + //log.Printf("synced %+v %+v", resp, since) for _, evt := range resp.ToDevice.Events { evt.Type.Class = event.ToDeviceEventType @@ -62,27 +61,41 @@ func initMatrix(config *config) error { log.Printf("Failed to parse to-device event of type %s: %v", evt.Type.Type, err) continue } - log.Printf("event %+v", evt) + log.Printf("Received to-device event %s", evt.Type.Type) + + var conf *conf + var call *call + switch evt.Type.Type { case CallInvite.Type: invite := evt.Content.AsCallInvite() - conf, _ := focus.getConf(invite.ConfID, true) - call, _ := conf.getCall(invite.CallID, true) + if conf, err = focus.getConf(invite.ConfID, true); err != nil { + log.Printf("Failed to create conf %s %+v", invite.ConfID, err) + return true + } + if conf == nil { + log.Fatal("Failed to create conf") + } + if call, err = conf.getCall(invite.CallID, true); err != nil { + log.Printf("Failed to create call %s %+v", invite.CallID, err) + return true + } + if call == nil { + log.Fatal("Failed to create call") + } call.userID = evt.Sender call.deviceID = invite.DeviceID + call.client = client // TODO: check session IDs call.onInvite(invite) case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() - var conf *conf - var call *call - var err error if conf, err = focus.getConf(candidates.ConfID, false); err != nil { - log.Printf("Got candidates for unknown conf %s", candidates.ConfID) + log.Printf("Failed to find conf %s %+v", candidates.ConfID, err) return true } if call, err = conf.getCall(candidates.CallID, false); err != nil { - log.Printf("Got candidates for unknown call %s in conf %s", candidates.CallID, candidates.ConfID) + log.Printf("Failed to find call %s %+v", candidates.ConfID, err) return true } call.onCandidates(candidates) From 65993502359af73c112721e55e93cc9edcec647e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 17:40:59 +0100 Subject: [PATCH 018/124] fix up sessions --- focus.go | 65 +++++++++++++++++++++++++++++-------------------------- matrix.go | 7 +++++- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/focus.go b/focus.go index fb848db..aa780e3 100644 --- a/focus.go +++ b/focus.go @@ -43,13 +43,15 @@ const ( type callState string type call struct { - callID string - userID id.UserID - deviceID id.DeviceID - client *mautrix.Client - peerConnection *webrtc.PeerConnection - callState callState - conf *conf + callID string + userID id.UserID + deviceID id.DeviceID + localSessionID string + remoteSessionID string + client *mautrix.Client + peerConnection *webrtc.PeerConnection + callState callState + conf *conf // we track the call's tracks via the conf object. } @@ -97,6 +99,7 @@ func (f *focus) getConf(confID string, create bool) (*conf, error) { } f.confs.confs[confID] = co co.calls.calls = make(map[string]*call) + co.trackDetails = make(map[trackKey]*trackDetail) } else { return nil, errors.New("No such conf") } @@ -290,10 +293,13 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), }, Candidates: []event.CallCandidate{ event.CallCandidate{ @@ -305,32 +311,23 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }, }, } - - toDevice := &mautrix.ReqSendToDevice{ - Messages: map[id.UserID]map[id.DeviceID]*event.Content{ - c.userID: { - c.deviceID: candidateEvtContent, - }, - }, - } - - // TODO: E2EE - // TODO: to-device reliability - c.client.SendToDevice(event.CallCandidates, toDevice) + c.sendToDevice(event.CallCandidates, candidateEvtContent) }) // TODO: send any subsequent candidates we discover to the peer answerSdp := peerConnection.LocalDescription().SDP - // TODO: sessions answerEvtContent := &event.Content{ Parsed: event.CallAnswerEventContent{ BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), }, Answer: event.CallData{ Type: "answer", @@ -338,20 +335,26 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }, }, } + c.sendToDevice(event.CallAnswer, answerEvtContent) + return err +} + +func (c *call) sendToDevice(callType event.Type, content *event.Content) error { + log.Printf("%s | sending to device %s", c.callID, callType.Type) toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ c.userID: { - c.deviceID: answerEvtContent, + c.deviceID: content, }, }, } // TODO: E2EE // TODO: to-device reliability - c.client.SendToDevice(event.CallAnswer, toDevice) + c.client.SendToDevice(callType, toDevice) - return err + return nil } func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { diff --git a/matrix.go b/matrix.go index c0d1ac6..2cc5604 100644 --- a/matrix.go +++ b/matrix.go @@ -61,7 +61,6 @@ func initMatrix(config *config) error { log.Printf("Failed to parse to-device event of type %s: %v", evt.Type.Type, err) continue } - log.Printf("Received to-device event %s", evt.Type.Type) var conf *conf var call *call @@ -69,6 +68,7 @@ func initMatrix(config *config) error { switch evt.Type.Type { case CallInvite.Type: invite := evt.Content.AsCallInvite() + log.Printf("%s | Received to-device event %s", invite.CallID, evt.Type.Type) if conf, err = focus.getConf(invite.ConfID, true); err != nil { log.Printf("Failed to create conf %s %+v", invite.ConfID, err) return true @@ -85,11 +85,16 @@ func initMatrix(config *config) error { } call.userID = evt.Sender call.deviceID = invite.DeviceID + // XXX: hardcode the same sessionID for SFUs for now, as nobody should care + // much if they get restarted(?) + call.localSessionID = "sfu" + call.remoteSessionID = invite.SenderSessionID call.client = client // TODO: check session IDs call.onInvite(invite) case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() + log.Printf("%s | Received to-device event %s", candidates.CallID, evt.Type.Type) if conf, err = focus.getConf(candidates.ConfID, false); err != nil { log.Printf("Failed to find conf %s %+v", candidates.ConfID, err) return true From 6cb58429d410a217c86d81e0deb3864710660586 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 18:30:09 +0100 Subject: [PATCH 019/124] unmarshal DC correctly --- focus.go | 28 +++++++++++++++++++--------- main.go | 10 +++++----- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/focus.go b/focus.go index aa780e3..687be2c 100644 --- a/focus.go +++ b/focus.go @@ -126,13 +126,19 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { return ca, nil } -func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLocal) { +func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLocal, err error) { c.trackDetailsMu.Lock() defer c.trackDetailsMu.Unlock() - return c.trackDetails[trackKey{ + + trackDetail := c.trackDetails[trackKey{ streamID: streamID, trackID: trackID, - }].track + }] + if trackDetail == nil { + return nil, errors.New("No such track") + } else { + return trackDetail.track, nil + } } func (c *call) dataChannelHandler(d *webrtc.DataChannel) { @@ -162,6 +168,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatal(err) } + log.Printf("Received DC %s %s %+v %+v", msg.Op, msg.ConfID, msg.Start, msg.Stop) + switch msg.Op { case "select": if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -172,7 +180,11 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } for _, trackDesc := range msg.Start { - track := c.conf.localTrackLookup(trackDesc.streamID, trackDesc.trackID) + track, err := c.conf.localTrackLookup(trackDesc.StreamID, trackDesc.TrackID) + if err != nil { + sendError("No Such Track") + return + } // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -180,12 +192,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { // connect to another focus in order to select // its streams. - if track == nil { - sendError("No Such Track") - return - } - if track != nil { + log.Printf("adding track %s", track.ID()) if _, err := peerConnection.AddTrack(track); err != nil { panic(err) } @@ -213,6 +221,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { panic(err) } + log.Printf("Sending DC %s %s", response.Op, response.SDP) + if err = d.SendText(string(marshaled)); err != nil { panic(err) } diff --git a/main.go b/main.go index 1f65b1a..659bb7f 100644 --- a/main.go +++ b/main.go @@ -44,9 +44,9 @@ type config struct { AccessToken string } -type TrackDesc struct { - streamID string `json:"stream_id"` - trackID string `json:"track_id"` +type trackDesc struct { + StreamID string `json:"stream_id"` + TrackID string `json:"track_id"` } type dataChannelMessage struct { @@ -55,7 +55,7 @@ type dataChannelMessage struct { Message string `json:"message,omitempty` // XXX: is this even needed? we know which conf a given call is for... ConfID string `json:"conf_id,omitempty"` - Start []TrackDesc `json:"start,omitempty"` - Stop []TrackDesc `json:"stop,omitempty"` + Start []trackDesc `json:"start,omitempty"` + Stop []trackDesc `json:"stop,omitempty"` SDP string `json:"sdp,omitempty"` } From e2e8a822d9dfc229a893b37fcc30025edfd959af Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 19:09:59 +0100 Subject: [PATCH 020/124] debugging --- focus.go | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/focus.go b/focus.go index 687be2c..3f7919b 100644 --- a/focus.go +++ b/focus.go @@ -168,7 +168,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatal(err) } - log.Printf("Received DC %s %s %+v %+v", msg.Op, msg.ConfID, msg.Start, msg.Stop) + log.Printf("%s | Received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) switch msg.Op { case "select": @@ -221,7 +221,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { panic(err) } - log.Printf("Sending DC %s %s", response.Op, response.SDP) + log.Printf("%s | Sending DC %s %s", c.callID, response.Op, response.SDP) if err = d.SendText(string(marshaled)); err != nil { panic(err) @@ -242,6 +242,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.peerConnection = peerConnection peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + log.Printf("%s | discovered track on PC with id %s, streamID %s and codec %+v", c.callID, trackRemote.ID(), trackRemote.StreamID(), trackRemote.Codec()) id := "audio" if strings.Contains(trackRemote.Codec().MimeType, "video") { id = "video" @@ -271,6 +272,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { call: c, track: trackLocal, } + log.Printf("%s | published %s %s", c.callID, streamID, trackID) c.conf.trackDetailsMu.Unlock() copyRemoteToLocal(trackRemote, trackLocal) @@ -280,22 +282,6 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.dataChannelHandler(d) }) - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: offer.SDP, - }) - - answer, err := peerConnection.CreateAnswer(nil) - if err != nil { - return err - } - - gatherComplete := webrtc.GatheringCompletePromise(peerConnection) - if err = peerConnection.SetLocalDescription(answer); err != nil { - return err - } - <-gatherComplete - peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { ice := candidate.ToJSON() @@ -324,7 +310,21 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.sendToDevice(event.CallCandidates, candidateEvtContent) }) - // TODO: send any subsequent candidates we discover to the peer + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: offer.SDP, + }) + + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + return err + } + + gatherComplete := webrtc.GatheringCompletePromise(peerConnection) + if err = peerConnection.SetLocalDescription(answer); err != nil { + return err + } + <-gatherComplete answerSdp := peerConnection.LocalDescription().SDP From ba82de8f9c1bfa7a8daadda6815f1d173c94f838 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 20:07:28 +0100 Subject: [PATCH 021/124] experiment: stop namespacing on streamIDs --- focus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/focus.go b/focus.go index 3f7919b..dc70828 100644 --- a/focus.go +++ b/focus.go @@ -266,13 +266,13 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } c.conf.trackDetails[trackKey{ - streamID: trackRemote.StreamID(), + streamID: "unknown", trackID: trackRemote.ID(), }] = &trackDetail{ call: c, track: trackLocal, } - log.Printf("%s | published %s %s", c.callID, streamID, trackID) + log.Printf("%s | published %s %s", c.callID, "unknown", trackRemote.ID()) c.conf.trackDetailsMu.Unlock() copyRemoteToLocal(trackRemote, trackLocal) From 1dd777b195edfd099b95bcb2e0d5692c6e84eaff Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 20:48:20 +0100 Subject: [PATCH 022/124] moar logging --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 659bb7f..47c3893 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ func loadConfig(configFilePath string) (*config, error) { } func main() { + log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds) configFilePath := flag.String("config", "config.yaml", "Configuration file path") flag.Parse() From fba15af469e23a8c2bf3ad3720346b3647bf3742 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 20:49:46 +0100 Subject: [PATCH 023/124] don't barf on candidate completion --- focus.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/focus.go b/focus.go index dc70828..d4af698 100644 --- a/focus.go +++ b/focus.go @@ -193,7 +193,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { // its streams. if track != nil { - log.Printf("adding track %s", track.ID()) + log.Printf("%s | adding track %s", c.callID, track.ID()) if _, err := peerConnection.AddTrack(track); err != nil { panic(err) } @@ -283,8 +283,14 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { }) peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if (candidate == nil) { + return + } + ice := candidate.ToJSON() + log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) + // TODO: batch these up a bit candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ From 95cffd14469ad1406261ff9ccd53f11d0912e62c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 21:26:37 +0100 Subject: [PATCH 024/124] more debugging; handle stream lookup failure correctly --- focus.go | 51 +++++++++++++++++++++++++++++++-------------------- matrix.go | 3 ++- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/focus.go b/focus.go index d4af698..ffdcb07 100644 --- a/focus.go +++ b/focus.go @@ -19,8 +19,6 @@ import ( type trackDetail struct { call *call - trackID string - streamID string track *webrtc.TrackLocalStaticRTP } @@ -127,6 +125,7 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { } func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLocal, err error) { + log.Printf("localTrackLookup called for %s %s", streamID, trackID) c.trackDetailsMu.Lock() defer c.trackDetailsMu.Unlock() @@ -134,6 +133,9 @@ func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLoc streamID: streamID, trackID: trackID, }] + + log.Printf("localTrackLookup returning with trackDetail %+v", trackDetail) + if trackDetail == nil { return nil, errors.New("No such track") } else { @@ -145,6 +147,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { peerConnection := c.peerConnection sendError := func(errMsg string) { + log.Printf("%s | sending DC error %s", c.callID, errMsg) marshaled, err := json.Marshal(&dataChannelMessage{ Op: "error", Message: errMsg, @@ -170,43 +173,50 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Printf("%s | Received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) + // TODO: hook cascade back up. + // As we're not an AS, we'd rely on the client + // to send us a "connect" op to tell us how to + // connect to another focus in order to select + // its streams. + switch msg.Op { case "select": - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }); err != nil { - panic(err) - } - + var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { + log.Printf("%s | localTrackLookup", c.callID) track, err := c.conf.localTrackLookup(trackDesc.StreamID, trackDesc.TrackID) if err != nil { sendError("No Such Track") return + } else { + tracks = append(tracks, track) } + } - // TODO: hook cascade back up. - // As we're not an AS, we'd rely on the client - // to send us a "connect" op to tell us how to - // connect to another focus in order to select - // its streams. + log.Printf("%s | SetRemoteDescription", c.callID) + if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: msg.SDP, + }); err != nil { + panic(err) + } - if track != nil { - log.Printf("%s | adding track %s", c.callID, track.ID()) - if _, err := peerConnection.AddTrack(track); err != nil { - panic(err) - } + for _, track := range tracks { + log.Printf("%s | adding track %s", c.callID, track.ID()) + if _, err := peerConnection.AddTrack(track); err != nil { + panic(err) } } // TODO: hook up msg.Stop to unsubscribe from tracks + log.Printf("%s | CreateAnswer", c.callID) answer, err := peerConnection.CreateAnswer(nil) if err != nil { panic(err) } + log.Printf("%s | SetLocalDescription", c.callID) if err := peerConnection.SetLocalDescription(answer); err != nil { panic(err) } @@ -221,7 +231,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { panic(err) } - log.Printf("%s | Sending DC %s %s", c.callID, response.Op, response.SDP) + log.Printf("%s | Sending DC %s", c.callID, response.Op) if err = d.SendText(string(marshaled)); err != nil { panic(err) @@ -290,6 +300,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { ice := candidate.ToJSON() log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) + return // TODO: batch these up a bit candidateEvtContent := &event.Content{ diff --git a/matrix.go b/matrix.go index 2cc5604..295bffc 100644 --- a/matrix.go +++ b/matrix.go @@ -95,12 +95,13 @@ func initMatrix(config *config) error { case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() log.Printf("%s | Received to-device event %s", candidates.CallID, evt.Type.Type) + return true if conf, err = focus.getConf(candidates.ConfID, false); err != nil { log.Printf("Failed to find conf %s %+v", candidates.ConfID, err) return true } if call, err = conf.getCall(candidates.CallID, false); err != nil { - log.Printf("Failed to find call %s %+v", candidates.ConfID, err) + log.Printf("Failed to find call %s %+v", candidates.CallID, err) return true } call.onCandidates(candidates) From 8bd0d043924309ef77db884c75e05dee492fb655 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 21:27:02 +0100 Subject: [PATCH 025/124] reenable ICE --- focus.go | 1 - go.mod | 2 +- matrix.go | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/focus.go b/focus.go index ffdcb07..5b5eec4 100644 --- a/focus.go +++ b/focus.go @@ -300,7 +300,6 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { ice := candidate.ToJSON() log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) - return // TODO: batch these up a bit candidateEvtContent := &event.Content{ diff --git a/go.mod b/go.mod index cb4fab1..044b3ff 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( maunium.net/go/mautrix v0.11.0 ) -replace maunium.net/go/mautrix v0.11.0 => github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4 +replace maunium.net/go/mautrix v0.11.0 => ../mautrix-go require ( github.com/google/uuid v1.3.0 // indirect diff --git a/matrix.go b/matrix.go index 295bffc..5e6f67a 100644 --- a/matrix.go +++ b/matrix.go @@ -95,7 +95,6 @@ func initMatrix(config *config) error { case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() log.Printf("%s | Received to-device event %s", candidates.CallID, evt.Type.Type) - return true if conf, err = focus.getConf(candidates.ConfID, false); err != nil { log.Printf("Failed to find conf %s %+v", candidates.ConfID, err) return true From 0053c433e2679991c3dcfb8e5764c8d069b5f0aa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 5 Jun 2022 21:36:42 +0100 Subject: [PATCH 026/124] only send ICE candidates after we've calculated an answer --- focus.go | 68 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/focus.go b/focus.go index 5b5eec4..f81dda7 100644 --- a/focus.go +++ b/focus.go @@ -292,40 +292,6 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.dataChannelHandler(d) }) - peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if (candidate == nil) { - return - } - - ice := candidate.ToJSON() - - log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) - - // TODO: batch these up a bit - candidateEvtContent := &event.Content{ - Parsed: event.CallCandidatesEventContent{ - BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), - }, - Candidates: []event.CallCandidate{ - event.CallCandidate{ - Candidate: ice.Candidate, - SDPMLineIndex: int(*ice.SDPMLineIndex), - SDPMID: *ice.SDPMid, - // XXX: what about ice.UsernameFragment? - }, - }, - }, - } - c.sendToDevice(event.CallCandidates, candidateEvtContent) - }) - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: offer.SDP, @@ -336,6 +302,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { return err } + // TODO: trickle ICE for fast conn setup, rather than block here gatherComplete := webrtc.GatheringCompletePromise(peerConnection) if err = peerConnection.SetLocalDescription(answer); err != nil { return err @@ -363,6 +330,39 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } c.sendToDevice(event.CallAnswer, answerEvtContent) + peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + if (candidate == nil) { + return + } + + ice := candidate.ToJSON() + + log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) + + // TODO: batch these up a bit + candidateEvtContent := &event.Content{ + Parsed: event.CallCandidatesEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Candidates: []event.CallCandidate{ + event.CallCandidate{ + Candidate: ice.Candidate, + SDPMLineIndex: int(*ice.SDPMLineIndex), + SDPMID: *ice.SDPMid, + // XXX: what about ice.UsernameFragment? + }, + }, + }, + } + c.sendToDevice(event.CallCandidates, candidateEvtContent) + }) return err } From fb61684944d39269397b7727691de3b86aad7348 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 6 Jun 2022 02:57:47 +0100 Subject: [PATCH 027/124] remove debugging --- focus.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/focus.go b/focus.go index f81dda7..a36dbb3 100644 --- a/focus.go +++ b/focus.go @@ -183,7 +183,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { - log.Printf("%s | localTrackLookup", c.callID) track, err := c.conf.localTrackLookup(trackDesc.StreamID, trackDesc.TrackID) if err != nil { sendError("No Such Track") @@ -193,7 +192,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } } - log.Printf("%s | SetRemoteDescription", c.callID) if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: msg.SDP, @@ -210,13 +208,11 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { // TODO: hook up msg.Stop to unsubscribe from tracks - log.Printf("%s | CreateAnswer", c.callID) answer, err := peerConnection.CreateAnswer(nil) if err != nil { panic(err) } - log.Printf("%s | SetLocalDescription", c.callID) if err := peerConnection.SetLocalDescription(answer); err != nil { panic(err) } From f3d9cdbed34007eb7e33cd51b849249c9165d2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 17:36:10 +0200 Subject: [PATCH 028/124] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/focus.go b/focus.go index a36dbb3..999201c 100644 --- a/focus.go +++ b/focus.go @@ -18,12 +18,10 @@ import ( ) type trackDetail struct { - call *call - track *webrtc.TrackLocalStaticRTP + call *call + track *webrtc.TrackLocalStaticRTP } -type setTrackDetails func(call *call, track *webrtc.TrackLocal) - // stolen from matrix-js-sdk // TODO: actually use callState (will be needed for renegotiation) const ( @@ -118,7 +116,7 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { } c.calls.calls[callID] = ca } else { - return nil, errors.New("No such call") + return nil, errors.New("no such call") } } return ca, nil @@ -327,7 +325,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.sendToDevice(event.CallAnswer, answerEvtContent) peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if (candidate == nil) { + if candidate == nil { return } From 46c7e3ac7fe24de258603b17eefe4830823cb9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 17:37:03 +0200 Subject: [PATCH 029/124] Add more logging for DC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/focus.go b/focus.go index 999201c..6f10ea8 100644 --- a/focus.go +++ b/focus.go @@ -159,6 +159,18 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } } + d.OnOpen(func() { + log.Printf("DC opened on call %s", c.callID) + }) + + d.OnClose(func() { + log.Printf("DC closed on call %s", c.callID) + }) + + d.OnError(func(err error) { + log.Fatalf("DC error on call %s: %s", c.callID, err) + }) + d.OnMessage(func(m webrtc.DataChannelMessage) { if !m.IsString { log.Fatal("Inbound message is not string") From b7add84c19892d7c8144268d6f7174168d520a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 17:48:52 +0200 Subject: [PATCH 030/124] Add useful logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/focus.go b/focus.go index 6f10ea8..9c2ecc9 100644 --- a/focus.go +++ b/focus.go @@ -132,11 +132,10 @@ func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLoc trackID: trackID, }] - log.Printf("localTrackLookup returning with trackDetail %+v", trackDetail) - if trackDetail == nil { - return nil, errors.New("No such track") + return nil, errors.New("no such track") } else { + log.Printf("localTrackLookup returning %s track with StreamID %s", trackDetail.track.Kind(), trackDetail.track.StreamID()) return trackDetail.track, nil } } @@ -210,7 +209,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } for _, track := range tracks { - log.Printf("%s | adding track %s", c.callID, track.ID()) + log.Printf("%s | adding %s track with StreamID %s", c.callID, track.Kind(), track.StreamID()) if _, err := peerConnection.AddTrack(track); err != nil { panic(err) } From 0305ee0ab2b7b962b413de2cbcc48f78699d789e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 30 Jul 2022 17:49:11 +0200 Subject: [PATCH 031/124] Update `.mod` files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- go.mod | 8 ++++---- go.sum | 19 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 044b3ff..b61baaa 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require github.com/pion/webrtc/v3 v3.1.31 require ( github.com/pion/rtcp v1.2.9 - gopkg.in/yaml.v3 v3.0.0 + gopkg.in/yaml.v3 v3.0.1 maunium.net/go/mautrix v0.11.0 ) @@ -29,8 +29,8 @@ require ( github.com/pion/transport v0.13.0 // indirect github.com/pion/turn/v2 v2.0.8 // indirect github.com/pion/udp v0.1.1 // indirect - golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 // indirect - golang.org/x/net v0.0.0-20220513224357-95641704303c // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect + golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index e2d2ad4..94dfcef 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4 h1:FzlLAmPztXGN0gufupwO7gI78J+6MDGiEzL1fnJXtXo= -github.com/matrix-org/mautrix-go v0.11.1-0.20220603181555-80ebd9db03f4/go.mod h1:CiKpMhAx5QZFHK03jpWb0iKI3sGU8x6+LfsOjDrcO8I= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -83,15 +81,15 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9 h1:NUzdAbFtCJSXU20AOXgeqaUwg8Ypg4MPYmL+d+rsB5c= -golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -105,8 +103,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220513224357-95641704303c h1:nF9mHSvoKBLkQNQhJZNsc66z2UzAMUbLGjC95CF3pU0= -golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -122,8 +120,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -156,5 +155,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 2b4bafbea6b219bf2475e7220e17a4a9e130026a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 31 Jul 2022 21:27:48 +0200 Subject: [PATCH 032/124] Fix types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 47c3893..ee53a2f 100644 --- a/main.go +++ b/main.go @@ -3,10 +3,11 @@ package main import ( "flag" "fmt" - yaml "gopkg.in/yaml.v3" "io/ioutil" "log" + yaml "gopkg.in/yaml.v3" + "maunium.net/go/mautrix/id" ) @@ -53,7 +54,7 @@ type trackDesc struct { type dataChannelMessage struct { Op string `json:"op"` ID string `json:"id"` - Message string `json:"message,omitempty` + Message string `json:"message,omitempty"` // XXX: is this even needed? we know which conf a given call is for... ConfID string `json:"conf_id,omitempty"` Start []trackDesc `json:"start,omitempty"` From 831d7b09c559bd5fb9edcecedef53831cc65800d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 31 Jul 2022 21:29:30 +0200 Subject: [PATCH 033/124] Be more precise about `m.call.negotaite` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- matrix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix.go b/matrix.go index 5e6f67a..f4a5a7c 100644 --- a/matrix.go +++ b/matrix.go @@ -111,7 +111,7 @@ func initMatrix(config *config) error { case CallSelectAnswer.Type: log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) case CallNegotiate.Type: - log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + log.Printf("Ignoring event as %s should be handled over DC", evt.Type.Type) case CallHangup.Type: log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) default: From a3ec7d82e3839982bd46bcf9077844ce4e69f6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 31 Jul 2022 21:32:30 +0200 Subject: [PATCH 034/124] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/focus.go b/focus.go index 9c2ecc9..868e658 100644 --- a/focus.go +++ b/focus.go @@ -397,7 +397,7 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { SDPMid: &candidate.SDPMID, } if err := c.peerConnection.AddICECandidate(ice); err != nil { - log.Printf("Failed to add ICE candidate", content) + log.Print("Failed to add ICE candidate", content) return err } } diff --git a/main.go b/main.go index ee53a2f..6b1d493 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,7 @@ func loadConfig(configFilePath string) (*config, error) { } var config config if err := yaml.Unmarshal(file, &config); err != nil { - return nil, fmt.Errorf("Failed to unmarshal YAML: %s", err) + return nil, fmt.Errorf("failed to unmarshal YAML: %s", err) } return &config, nil } From ca48340f19ee7c112eb829f6c5325c42ffc6bbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 31 Jul 2022 21:40:25 +0200 Subject: [PATCH 035/124] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/focus.go b/focus.go index 868e658..d0f5e27 100644 --- a/focus.go +++ b/focus.go @@ -357,7 +357,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { Version: event.CallVersion("1"), }, Candidates: []event.CallCandidate{ - event.CallCandidate{ + { Candidate: ice.Candidate, SDPMLineIndex: int(*ice.SDPMLineIndex), SDPMID: *ice.SDPMid, From 52917877b72426838dda2a7f734bc7f68f0864cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 31 Jul 2022 21:44:15 +0200 Subject: [PATCH 036/124] Make basic group calling work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 117 +++++++++++++++++++++++++------------------------------ 1 file changed, 52 insertions(+), 65 deletions(-) diff --git a/focus.go b/focus.go index d0f5e27..bd90fd4 100644 --- a/focus.go +++ b/focus.go @@ -17,11 +17,6 @@ import ( "github.com/pion/webrtc/v3" ) -type trackDetail struct { - call *call - track *webrtc.TrackLocalStaticRTP -} - // stolen from matrix-js-sdk // TODO: actually use callState (will be needed for renegotiation) const ( @@ -56,17 +51,11 @@ type calls struct { calls map[string]*call } -// FIXME: for uniqueness, should we index tracks by {callID, streamID, trackID}? -type trackKey struct { - streamID string - trackID string -} - type conf struct { - confID string - calls calls - trackDetailsMu sync.RWMutex - trackDetails map[trackKey]*trackDetail // by trackID. + confID string + calls calls + tracksMu sync.RWMutex + tracks map[string][]*webrtc.TrackLocalStaticRTP // by streamId. } type confs struct { @@ -95,9 +84,9 @@ func (f *focus) getConf(confID string, create bool) (*conf, error) { } f.confs.confs[confID] = co co.calls.calls = make(map[string]*call) - co.trackDetails = make(map[trackKey]*trackDetail) + co.tracks = make(map[string][]*webrtc.TrackLocalStaticRTP) } else { - return nil, errors.New("No such conf") + return nil, errors.New("no such conf") } } return co, nil @@ -122,21 +111,20 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { return ca, nil } -func (c *conf) localTrackLookup(streamID, trackID string) (track webrtc.TrackLocal, err error) { - log.Printf("localTrackLookup called for %s %s", streamID, trackID) - c.trackDetailsMu.Lock() - defer c.trackDetailsMu.Unlock() +func (c *conf) getLocalTrackByStreamId(streamID string) (tracks []webrtc.TrackLocal, err error) { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() - trackDetail := c.trackDetails[trackKey{ - streamID: streamID, - trackID: trackID, - }] - - if trackDetail == nil { - return nil, errors.New("no such track") + foundTracks := c.tracks[streamID] + if foundTracks == nil { + log.Printf("Found no streams for %s", streamID) + return nil, errors.New("no such streams") } else { - log.Printf("localTrackLookup returning %s track with StreamID %s", trackDetail.track.Kind(), trackDetail.track.StreamID()) - return trackDetail.track, nil + tracksToReturn := []webrtc.TrackLocal{} + for _, track := range foundTracks { + tracksToReturn = append(tracksToReturn, track) + } + return tracksToReturn, nil } } @@ -192,22 +180,15 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { - track, err := c.conf.localTrackLookup(trackDesc.StreamID, trackDesc.TrackID) + foundTracks, err := c.conf.getLocalTrackByStreamId(trackDesc.StreamID) if err != nil { - sendError("No Such Track") + sendError("No Such Stream") return } else { - tracks = append(tracks, track) + tracks = append(tracks, foundTracks...) } } - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }); err != nil { - panic(err) - } - for _, track := range tracks { log.Printf("%s | adding %s track with StreamID %s", c.callID, track.Kind(), track.StreamID()) if _, err := peerConnection.AddTrack(track); err != nil { @@ -215,34 +196,40 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } } - // TODO: hook up msg.Stop to unsubscribe from tracks - - answer, err := peerConnection.CreateAnswer(nil) + offer, err := peerConnection.CreateOffer(nil) if err != nil { panic(err) } - - if err := peerConnection.SetLocalDescription(answer); err != nil { + err = peerConnection.SetLocalDescription(offer) + if err != nil { panic(err) } response := dataChannelMessage{ - Op: "answer", + Op: "offer", ID: msg.ID, - SDP: answer.SDP, + SDP: offer.SDP, } marshaled, err := json.Marshal(response) if err != nil { panic(err) } - - log.Printf("%s | Sending DC %s", c.callID, response.Op) - - if err = d.SendText(string(marshaled)); err != nil { + err = d.SendText(string(marshaled)) + if err != nil { panic(err) } + + log.Printf("%s | Sent DC %s", c.callID, response.Op) + + case "answer": + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: msg.SDP, + }) + default: log.Fatalf("Unknown operation %s", msg.Op) + // TODO: hook up msg.Stop to unsubscribe from tracks } }) } @@ -257,11 +244,8 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { c.peerConnection = peerConnection peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { - log.Printf("%s | discovered track on PC with id %s, streamID %s and codec %+v", c.callID, trackRemote.ID(), trackRemote.StreamID(), trackRemote.Codec()) - id := "audio" + log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) if strings.Contains(trackRemote.Codec().MimeType, "video") { - id = "video" - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval go func() { ticker := time.NewTicker(time.Millisecond * 200) @@ -274,21 +258,24 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } - c.conf.trackDetailsMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, id, fmt.Sprintf("%s-%s-%s", c.callID, c.deviceID, trackRemote.ID())) + c.conf.tracksMu.Lock() + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.Kind().String(), trackRemote.StreamID()) if err != nil { panic(err) } - c.conf.trackDetails[trackKey{ - streamID: "unknown", - trackID: trackRemote.ID(), - }] = &trackDetail{ - call: c, - track: trackLocal, + if c.conf.tracks[trackLocal.StreamID()] == nil { + receivedTracks := []*webrtc.TrackLocalStaticRTP{trackLocal} + c.conf.tracks[trackLocal.StreamID()] = receivedTracks + + } else { + receivedTracks := append(c.conf.tracks[trackLocal.StreamID()], trackLocal) + c.conf.tracks[trackLocal.StreamID()] = receivedTracks + } - log.Printf("%s | published %s %s", c.callID, "unknown", trackRemote.ID()) - c.conf.trackDetailsMu.Unlock() + + log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) + c.conf.tracksMu.Unlock() copyRemoteToLocal(trackRemote, trackLocal) }) From d0cb49bf6a682d07e6cae8cfed4eca676317375b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 1 Aug 2022 09:55:59 +0200 Subject: [PATCH 037/124] Handle events we don't care about nicer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- matrix.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/matrix.go b/matrix.go index f4a5a7c..e0ba1ee 100644 --- a/matrix.go +++ b/matrix.go @@ -104,16 +104,17 @@ func initMatrix(config *config) error { return true } call.onCandidates(candidates) - case CallAnswer.Type: - log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) - case CallReject.Type: - log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) case CallSelectAnswer.Type: log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) - case CallNegotiate.Type: - log.Printf("Ignoring event as %s should be handled over DC", evt.Type.Type) case CallHangup.Type: log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + + // Events we don't care about + case CallNegotiate.Type: + log.Printf("Ignoring event %s as should be handled over DC", evt.Type.Type) + case CallReject.Type: + case CallAnswer.Type: + log.Printf("Ignoring event %s as we are always the ones answering", evt.Type.Type) default: log.Printf("Ignoring unrecognised to-device event of type %s", evt.Type.Type) } From 8393552c4c3b949c92cf6aadbda78af8ba61b4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 1 Aug 2022 10:48:20 +0200 Subject: [PATCH 038/124] Handle `m.call.hangup` and `m.call.select_answer` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 17 ++++++++++++++++ matrix.go | 61 ++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/focus.go b/focus.go index bd90fd4..46a2779 100644 --- a/focus.go +++ b/focus.go @@ -358,6 +358,23 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { return err } +func (c *call) onSelectAnswer(content *event.CallSelectAnswerEventContent) { + selectedPartyId := content.SelectedPartyID + if selectedPartyId != string(c.client.DeviceID) { + c.terminate() + log.Printf("%s | Call was answered on a different device: %s", content.CallID, selectedPartyId) + } +} + +func (c *call) onHangup(content *event.CallHangupEventContent) { + c.terminate() +} + +func (c *call) terminate() error { + // TODO: Implement terminate + return nil +} + func (c *call) sendToDevice(callType event.Type, content *event.Content) error { log.Printf("%s | sending to device %s", c.callID, callType.Type) toDevice := &mautrix.ReqSendToDevice{ diff --git a/matrix.go b/matrix.go index e0ba1ee..e3027b0 100644 --- a/matrix.go +++ b/matrix.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "reflect" + "strings" "maunium.net/go/mautrix" "maunium.net/go/mautrix/event" @@ -51,9 +52,22 @@ func initMatrix(config *config) error { // TODO: E2EE - syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { - //log.Printf("synced %+v %+v", resp, since) + getExistingCall := func(confID string, callID string) (*call, error) { + var conf *conf + var call *call + + if conf, err = focus.getConf(confID, false); err != nil || conf == nil { + log.Printf("Failed to get conf %s %+v", confID, err) + return nil, err + } + if call, err = conf.getCall(callID, false); err != nil || call == nil { + log.Printf("Failed to get call %s %+v", callID, err) + return nil, err + } + return call, nil + } + syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { for _, evt := range resp.ToDevice.Events { evt.Type.Class = event.ToDeviceEventType err := evt.Content.ParseRaw(evt.Type) @@ -65,24 +79,25 @@ func initMatrix(config *config) error { var conf *conf var call *call + if strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { + log.Printf("%s | Received to-device event %s", evt.Content.Raw["call_id"], evt.Type.Type) + } else { + log.Printf("Received non-call to-device event %s", evt.Type.Type) + continue + } + + // TODO: check session IDs switch evt.Type.Type { case CallInvite.Type: invite := evt.Content.AsCallInvite() - log.Printf("%s | Received to-device event %s", invite.CallID, evt.Type.Type) - if conf, err = focus.getConf(invite.ConfID, true); err != nil { + if conf, err = focus.getConf(invite.ConfID, true); err != nil || conf == nil { log.Printf("Failed to create conf %s %+v", invite.ConfID, err) return true } - if conf == nil { - log.Fatal("Failed to create conf") - } - if call, err = conf.getCall(invite.CallID, true); err != nil { + if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { log.Printf("Failed to create call %s %+v", invite.CallID, err) return true } - if call == nil { - log.Fatal("Failed to create call") - } call.userID = evt.Sender call.deviceID = invite.DeviceID // XXX: hardcode the same sessionID for SFUs for now, as nobody should care @@ -90,24 +105,25 @@ func initMatrix(config *config) error { call.localSessionID = "sfu" call.remoteSessionID = invite.SenderSessionID call.client = client - // TODO: check session IDs call.onInvite(invite) case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() - log.Printf("%s | Received to-device event %s", candidates.CallID, evt.Type.Type) - if conf, err = focus.getConf(candidates.ConfID, false); err != nil { - log.Printf("Failed to find conf %s %+v", candidates.ConfID, err) - return true - } - if call, err = conf.getCall(candidates.CallID, false); err != nil { - log.Printf("Failed to find call %s %+v", candidates.CallID, err) + if call, err = getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil || call == nil { return true } call.onCandidates(candidates) case CallSelectAnswer.Type: - log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + selectAnswer := evt.Content.AsCallSelectAnswer() + if call, err = getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil || call == nil { + return true + } + call.onSelectAnswer(selectAnswer) case CallHangup.Type: - log.Printf("Ignoring unimplemented event of type %s", evt.Type.Type) + hangup := evt.Content.AsCallHangup() + if call, err = getExistingCall(hangup.ConfID, hangup.CallID); err != nil || call == nil { + return true + } + call.onHangup(hangup) // Events we don't care about case CallNegotiate.Type: @@ -123,8 +139,7 @@ func initMatrix(config *config) error { return true }) - err = client.Sync() - if err != nil { + if err = client.Sync(); err != nil { log.Panic("Sync failed", err) } From 699e36215cf376de3a6289fda7656fb75973a7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 1 Aug 2022 20:41:13 +0200 Subject: [PATCH 039/124] Allow lookup of tracks by info about them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 64 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/focus.go b/focus.go index 46a2779..9d7285b 100644 --- a/focus.go +++ b/focus.go @@ -46,6 +46,17 @@ type call struct { // we track the call's tracks via the conf object. } +type localTrackInfo struct { + streamID string + trackID string + call *call +} + +type localTrackWithInfo struct { + track *webrtc.TrackLocalStaticRTP + info localTrackInfo +} + type calls struct { callsMu sync.RWMutex calls map[string]*call @@ -55,7 +66,7 @@ type conf struct { confID string calls calls tracksMu sync.RWMutex - tracks map[string][]*webrtc.TrackLocalStaticRTP // by streamId. + tracks []localTrackWithInfo } type confs struct { @@ -84,7 +95,7 @@ func (f *focus) getConf(confID string, create bool) (*conf, error) { } f.confs.confs[confID] = co co.calls.calls = make(map[string]*call) - co.tracks = make(map[string][]*webrtc.TrackLocalStaticRTP) + co.tracks = []localTrackWithInfo{} } else { return nil, errors.New("no such conf") } @@ -111,20 +122,30 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { return ca, nil } -func (c *conf) getLocalTrackByStreamId(streamID string) (tracks []webrtc.TrackLocal, err error) { +func (c *conf) getLocalTrackByStreamId(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { c.tracksMu.Lock() defer c.tracksMu.Unlock() - foundTracks := c.tracks[streamID] - if foundTracks == nil { - log.Printf("Found no streams for %s", streamID) + foundTracks := []webrtc.TrackLocal{} + for _, track := range c.tracks { + info := track.info + if selectInfo.call != nil && selectInfo.call != info.call { + continue + } + if selectInfo.streamID != "" && selectInfo.streamID != info.streamID { + continue + } + if selectInfo.trackID != "" && selectInfo.trackID != info.trackID { + continue + } + foundTracks = append(foundTracks, track.track) + } + + if len(foundTracks) == 0 { + log.Printf("Found no tracks for %+v", selectInfo) return nil, errors.New("no such streams") } else { - tracksToReturn := []webrtc.TrackLocal{} - for _, track := range foundTracks { - tracksToReturn = append(tracksToReturn, track) - } - return tracksToReturn, nil + return foundTracks, nil } } @@ -180,7 +201,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { - foundTracks, err := c.conf.getLocalTrackByStreamId(trackDesc.StreamID) + foundTracks, err := c.conf.getLocalTrackByStreamId(localTrackInfo{streamID: trackDesc.StreamID}) if err != nil { sendError("No Such Stream") return @@ -259,20 +280,19 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } c.conf.tracksMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.Kind().String(), trackRemote.StreamID()) + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) if err != nil { panic(err) } - if c.conf.tracks[trackLocal.StreamID()] == nil { - receivedTracks := []*webrtc.TrackLocalStaticRTP{trackLocal} - c.conf.tracks[trackLocal.StreamID()] = receivedTracks - - } else { - receivedTracks := append(c.conf.tracks[trackLocal.StreamID()], trackLocal) - c.conf.tracks[trackLocal.StreamID()] = receivedTracks - - } + c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ + track: trackLocal, + info: localTrackInfo{ + trackID: trackLocal.ID(), + streamID: trackLocal.StreamID(), + call: c, + }, + }) log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) c.conf.tracksMu.Unlock() From d592bf3064910651f3f77ca20b5c994e01879e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 09:59:41 +0200 Subject: [PATCH 040/124] Make leaving confs work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- focus.go | 243 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 161 insertions(+), 82 deletions(-) diff --git a/focus.go b/focus.go index 9d7285b..dae8a48 100644 --- a/focus.go +++ b/focus.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "errors" - "fmt" "log" "strings" "sync" @@ -43,6 +42,7 @@ type call struct { peerConnection *webrtc.PeerConnection callState callState conf *conf + dataChannel *webrtc.DataChannel // we track the call's tracks via the conf object. } @@ -59,7 +59,7 @@ type localTrackWithInfo struct { type calls struct { callsMu sync.RWMutex - calls map[string]*call + calls map[string]*call // By callID } type conf struct { @@ -122,12 +122,9 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { return ca, nil } -func (c *conf) getLocalTrackByStreamId(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() - - foundTracks := []webrtc.TrackLocal{} - for _, track := range c.tracks { +func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int, err error) { + foundIndices := []int{} + for index, track := range c.tracks { info := track.info if selectInfo.call != nil && selectInfo.call != info.call { continue @@ -138,18 +135,73 @@ func (c *conf) getLocalTrackByStreamId(selectInfo localTrackInfo) (tracks []webr if selectInfo.trackID != "" && selectInfo.trackID != info.trackID { continue } - foundTracks = append(foundTracks, track.track) + foundIndices = append(foundIndices, index) } - if len(foundTracks) == 0 { + if len(foundIndices) == 0 { log.Printf("Found no tracks for %+v", selectInfo) - return nil, errors.New("no such streams") + return nil, errors.New("no such tracks") + } else { + return foundIndices, nil + } +} + +func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() + + indices, err := c.getLocalTrackIndicesByInfo(selectInfo) + if err != nil { + return nil, err + } + + foundTracks := []webrtc.TrackLocal{} + for _, index := range indices { + foundTracks = append(foundTracks, c.tracks[index].track) + } + + if len(foundTracks) == 0 { + log.Printf("No tracks") + return nil, errors.New("no such tracks") } else { return foundTracks, nil } } +func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() + + indices, err := c.getLocalTrackIndicesByInfo(removeInfo) + if err != nil { + return err + } + + // FIXME: the big O of this must be awful... + for _, index := range indices { + info := c.tracks[index].info + + for _, call := range c.calls.calls { + for _, sender := range call.peerConnection.GetSenders() { + if info.trackID == sender.Track().ID() { + log.Printf("%s | removing %s track with StreamID %s", call.callID, sender.Track().Kind(), info.streamID) + if err := sender.Stop(); err != nil { + log.Printf("%s | failed to stop sender: %s", call.callID, err) + } + if err := call.peerConnection.RemoveTrack(sender); err != nil { + log.Printf("%s | failed to remove track: %s", call.callID, err) + return err + } + } + } + } + } + + return nil +} + func (c *call) dataChannelHandler(d *webrtc.DataChannel) { + c.dataChannel = d peerConnection := c.peerConnection sendError := func(errMsg string) { @@ -168,15 +220,15 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } d.OnOpen(func() { - log.Printf("DC opened on call %s", c.callID) + log.Printf("%s | DC opened", c.callID) }) d.OnClose(func() { - log.Printf("DC closed on call %s", c.callID) + log.Printf("%s | DC closed", c.callID) }) d.OnError(func(err error) { - log.Fatalf("DC error on call %s: %s", c.callID, err) + log.Fatalf("%s | DC error: %s", c.callID, err) }) d.OnMessage(func(m webrtc.DataChannelMessage) { @@ -186,10 +238,10 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { msg := &dataChannelMessage{} if err := json.Unmarshal(m.Data, msg); err != nil { - log.Fatal(err) + log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) } - log.Printf("%s | Received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) + log.Printf("%s | received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -201,7 +253,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { - foundTracks, err := c.conf.getLocalTrackByStreamId(localTrackInfo{streamID: trackDesc.StreamID}) + foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID}) if err != nil { sendError("No Such Stream") return @@ -217,31 +269,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } } - offer, err := peerConnection.CreateOffer(nil) - if err != nil { - panic(err) - } - err = peerConnection.SetLocalDescription(offer) - if err != nil { - panic(err) - } - - response := dataChannelMessage{ - Op: "offer", - ID: msg.ID, - SDP: offer.SDP, - } - marshaled, err := json.Marshal(response) - if err != nil { - panic(err) - } - err = d.SendText(string(marshaled)) - if err != nil { - panic(err) - } - - log.Printf("%s | Sent DC %s", c.callID, response.Op) - case "answer": peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, @@ -255,6 +282,68 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) } +func (c *call) negotiationNeeded() { + log.Printf("%s | negotiation needed", c.callID) + + offer, err := c.peerConnection.CreateOffer(nil) + if err != nil { + panic(err) + } + err = c.peerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + response := dataChannelMessage{ + Op: "offer", + SDP: offer.SDP, + } + marshaled, err := json.Marshal(response) + if err != nil { + panic(err) + } + err = c.dataChannel.SendText(string(marshaled)) + if err != nil { + log.Printf("%s | failed to send over DC: %s", c.callID, err) + } + + log.Printf("%s | sent DC %s", c.callID, response.Op) +} + +func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { + if candidate == nil { + return + } + + ice := candidate.ToJSON() + + log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) + + // TODO: batch these up a bit + candidateEvtContent := &event.Content{ + Parsed: event.CallCandidatesEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Candidates: []event.CallCandidate{ + { + Candidate: ice.Candidate, + SDPMLineIndex: int(*ice.SDPMLineIndex), + SDPMID: *ice.SDPMid, + // XXX: what about ice.UsernameFragment? + }, + }, + }, + } + c.sendToDevice(event.CallCandidates, candidateEvtContent) +} + func (c *call) onInvite(content *event.CallInviteEventContent) error { offer := content.Offer @@ -271,12 +360,12 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { go func() { ticker := time.NewTicker(time.Millisecond * 200) for range ticker.C { - if errSend := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); errSend != nil { - fmt.Println(errSend) + if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) + break } } }() - } c.conf.tracksMu.Lock() @@ -303,6 +392,12 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { c.dataChannelHandler(d) }) + peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + c.iceCandidateHandler(candidate) + }) + peerConnection.OnNegotiationNeeded(func() { + c.negotiationNeeded() + }) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, @@ -342,39 +437,6 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } c.sendToDevice(event.CallAnswer, answerEvtContent) - peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - if candidate == nil { - return - } - - ice := candidate.ToJSON() - - log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) - - // TODO: batch these up a bit - candidateEvtContent := &event.Content{ - Parsed: event.CallCandidatesEventContent{ - BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), - }, - Candidates: []event.CallCandidate{ - { - Candidate: ice.Candidate, - SDPMLineIndex: int(*ice.SDPMLineIndex), - SDPMID: *ice.SDPMid, - // XXX: what about ice.UsernameFragment? - }, - }, - }, - } - c.sendToDevice(event.CallCandidates, candidateEvtContent) - }) return err } @@ -391,7 +453,22 @@ func (c *call) onHangup(content *event.CallHangupEventContent) { } func (c *call) terminate() error { - // TODO: Implement terminate + log.Printf("%s | Terminating call", c.callID) + + if err := c.peerConnection.Close(); err != nil { + log.Printf("%s | error closing peer connection: %s", c.callID, err) + } + + c.conf.calls.callsMu.Lock() + delete(c.conf.calls.calls, c.callID) + c.conf.calls.callsMu.Unlock() + + if err := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{call: c}); err != nil { + return err + } + + // TODO: Remove the tracks from conf.tracks + return nil } @@ -432,12 +509,14 @@ func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.Track buff := make([]byte, 1500) for { i, _, err := trackRemote.Read(buff) - if err != nil { - panic(err) + if err != nil || buff == nil { + log.Printf("ending read on track with StreamID %s: %s", trackRemote.StreamID(), err) + break } if _, err = trackLocal.Write(buff[:i]); err != nil { - panic(err) + log.Printf("ending write on track with StreamID %s: %s", trackLocal.StreamID(), err) + break } } From 0caefb1578d7a4ef4c703fe6db3a80ca107e68d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:03:19 +0200 Subject: [PATCH 041/124] Move golang files into `src` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- README.md | 2 +- cascade.go => src/cascade.go | 0 focus.go => src/focus.go | 0 main.go => src/main.go | 0 matrix.go => src/matrix.go | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename cascade.go => src/cascade.go (100%) rename focus.go => src/focus.go (100%) rename main.go => src/main.go (100%) rename matrix.go => src/matrix.go (100%) diff --git a/README.md b/README.md index c881d11..6965c71 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,6 @@ The client will respond to the `subscribe` with the answer. ## Running -* `go run *.go` +* `go run ./src/*.go` * Access at http://localhost:8080 diff --git a/cascade.go b/src/cascade.go similarity index 100% rename from cascade.go rename to src/cascade.go diff --git a/focus.go b/src/focus.go similarity index 100% rename from focus.go rename to src/focus.go diff --git a/main.go b/src/main.go similarity index 100% rename from main.go rename to src/main.go diff --git a/matrix.go b/src/matrix.go similarity index 100% rename from matrix.go rename to src/matrix.go From 209be906a317a40d3257c1daf1363fce78978460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:25:06 +0200 Subject: [PATCH 042/124] Reogranize source code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .gitignore | 2 +- README.md | 5 + build.sh | 1 + src/call.go | 333 ++++++++++++++++++++++++++++++++++ src/conf.go | 147 +++++++++++++++ src/focus.go | 503 --------------------------------------------------- src/utils.go | 24 +++ 7 files changed, 511 insertions(+), 504 deletions(-) create mode 100755 build.sh create mode 100644 src/call.go create mode 100644 src/conf.go create mode 100644 src/utils.go diff --git a/.gitignore b/.gitignore index 4adbf11..c1ffadc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ config.yaml -sfu-to-sfu +dist/ diff --git a/README.md b/README.md index 6965c71..1e45f7d 100644 --- a/README.md +++ b/README.md @@ -176,3 +176,8 @@ The client will respond to the `subscribe` with the answer. * `go run ./src/*.go` * Access at http://localhost:8080 +## Building + +* `./build.zsh` +* `./dist/bin` +* Access at diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..3990ed4 --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +go build -o dist/bin src/*.go \ No newline at end of file diff --git a/src/call.go b/src/call.go new file mode 100644 index 0000000..ff47848 --- /dev/null +++ b/src/call.go @@ -0,0 +1,333 @@ +package main + +import ( + "encoding/json" + "log" + "strings" + "time" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/pion/rtcp" + "github.com/pion/webrtc/v3" +) + +type call struct { + callID string + userID id.UserID + deviceID id.DeviceID + localSessionID string + remoteSessionID string + client *mautrix.Client + peerConnection *webrtc.PeerConnection + conf *conf + dataChannel *webrtc.DataChannel +} + +func (c *call) dataChannelHandler(d *webrtc.DataChannel) { + c.dataChannel = d + peerConnection := c.peerConnection + + sendError := func(errMsg string) { + log.Printf("%s | sending DC error %s", c.callID, errMsg) + marshaled, err := json.Marshal(&dataChannelMessage{ + Op: "error", + Message: errMsg, + }) + if err != nil { + panic(err) + } + + if err = d.SendText(string(marshaled)); err != nil { + panic(err) + } + } + + d.OnOpen(func() { + log.Printf("%s | DC opened", c.callID) + }) + + d.OnClose(func() { + log.Printf("%s | DC closed", c.callID) + }) + + d.OnError(func(err error) { + log.Fatalf("%s | DC error: %s", c.callID, err) + }) + + d.OnMessage(func(m webrtc.DataChannelMessage) { + if !m.IsString { + log.Fatal("Inbound message is not string") + } + + msg := &dataChannelMessage{} + if err := json.Unmarshal(m.Data, msg); err != nil { + log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) + } + + log.Printf("%s | received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) + + // TODO: hook cascade back up. + // As we're not an AS, we'd rely on the client + // to send us a "connect" op to tell us how to + // connect to another focus in order to select + // its streams. + + switch msg.Op { + case "select": + var tracks []webrtc.TrackLocal + for _, trackDesc := range msg.Start { + foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID}) + if err != nil { + sendError("No Such Stream") + return + } else { + tracks = append(tracks, foundTracks...) + } + } + + for _, track := range tracks { + log.Printf("%s | adding %s track with StreamID %s", c.callID, track.Kind(), track.StreamID()) + if _, err := peerConnection.AddTrack(track); err != nil { + panic(err) + } + } + + case "answer": + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: msg.SDP, + }) + + default: + log.Fatalf("Unknown operation %s", msg.Op) + // TODO: hook up msg.Stop to unsubscribe from tracks + } + }) +} + +func (c *call) negotiationNeededHandler() { + log.Printf("%s | negotiation needed", c.callID) + + offer, err := c.peerConnection.CreateOffer(nil) + if err != nil { + panic(err) + } + err = c.peerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + response := dataChannelMessage{ + Op: "offer", + SDP: offer.SDP, + } + marshaled, err := json.Marshal(response) + if err != nil { + panic(err) + } + err = c.dataChannel.SendText(string(marshaled)) + if err != nil { + log.Printf("%s | failed to send over DC: %s", c.callID, err) + } + + log.Printf("%s | sent DC %s", c.callID, response.Op) +} + +func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { + if candidate == nil { + return + } + + ice := candidate.ToJSON() + + log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) + + // TODO: batch these up a bit + candidateEvtContent := &event.Content{ + Parsed: event.CallCandidatesEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Candidates: []event.CallCandidate{ + { + Candidate: ice.Candidate, + SDPMLineIndex: int(*ice.SDPMLineIndex), + SDPMID: *ice.SDPMid, + // XXX: what about ice.UsernameFragment? + }, + }, + }, + } + c.sendToDevice(event.CallCandidates, candidateEvtContent) +} + +func (c *call) onInvite(content *event.CallInviteEventContent) error { + offer := content.Offer + + peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + return err + } + c.peerConnection = peerConnection + + peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) + if strings.Contains(trackRemote.Codec().MimeType, "video") { + // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + go func() { + ticker := time.NewTicker(time.Millisecond * 200) + for range ticker.C { + if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) + break + } + } + }() + } + + c.conf.tracksMu.Lock() + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) + if err != nil { + panic(err) + } + + c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ + track: trackLocal, + info: localTrackInfo{ + trackID: trackLocal.ID(), + streamID: trackLocal.StreamID(), + call: c, + }, + }) + + log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) + c.conf.tracksMu.Unlock() + + copyRemoteToLocal(trackRemote, trackLocal) + }) + + peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { + c.dataChannelHandler(d) + }) + peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { + c.iceCandidateHandler(candidate) + }) + peerConnection.OnNegotiationNeeded(func() { + c.negotiationNeededHandler() + }) + + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: offer.SDP, + }) + + answer, err := peerConnection.CreateAnswer(nil) + if err != nil { + return err + } + + // TODO: trickle ICE for fast conn setup, rather than block here + gatherComplete := webrtc.GatheringCompletePromise(peerConnection) + if err = peerConnection.SetLocalDescription(answer); err != nil { + return err + } + <-gatherComplete + + answerSdp := peerConnection.LocalDescription().SDP + + answerEvtContent := &event.Content{ + Parsed: event.CallAnswerEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Answer: event.CallData{ + Type: "answer", + SDP: answerSdp, + }, + }, + } + c.sendToDevice(event.CallAnswer, answerEvtContent) + + return err +} + +func (c *call) onSelectAnswer(content *event.CallSelectAnswerEventContent) { + selectedPartyId := content.SelectedPartyID + if selectedPartyId != string(c.client.DeviceID) { + c.terminate() + log.Printf("%s | Call was answered on a different device: %s", content.CallID, selectedPartyId) + } +} + +func (c *call) onHangup(content *event.CallHangupEventContent) { + c.terminate() +} + +func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { + for _, candidate := range content.Candidates { + sdpMLineIndex := uint16(candidate.SDPMLineIndex) + ice := webrtc.ICECandidateInit{ + Candidate: candidate.Candidate, + SDPMid: &candidate.SDPMID, + SDPMLineIndex: &sdpMLineIndex, + UsernameFragment: new(string), + } + if err := c.peerConnection.AddICECandidate(ice); err != nil { + log.Print("Failed to add ICE candidate", content) + return err + } + } + return nil +} + +func (c *call) terminate() error { + log.Printf("%s | Terminating call", c.callID) + + if err := c.peerConnection.Close(); err != nil { + log.Printf("%s | error closing peer connection: %s", c.callID, err) + } + + c.conf.calls.callsMu.Lock() + delete(c.conf.calls.calls, c.callID) + c.conf.calls.callsMu.Unlock() + + if err := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{call: c}); err != nil { + return err + } + + // TODO: Remove the tracks from conf.tracks + + return nil +} + +func (c *call) sendToDevice(callType event.Type, content *event.Content) error { + log.Printf("%s | sending to device %s", c.callID, callType.Type) + toDevice := &mautrix.ReqSendToDevice{ + Messages: map[id.UserID]map[id.DeviceID]*event.Content{ + c.userID: { + c.deviceID: content, + }, + }, + } + + // TODO: E2EE + // TODO: to-device reliability + c.client.SendToDevice(callType, toDevice) + + return nil +} diff --git a/src/conf.go b/src/conf.go new file mode 100644 index 0000000..6b3e4c2 --- /dev/null +++ b/src/conf.go @@ -0,0 +1,147 @@ +package main + +import ( + "errors" + "log" + "sync" + + "github.com/pion/webrtc/v3" +) + +type localTrackInfo struct { + streamID string + trackID string + call *call +} + +type localTrackWithInfo struct { + track *webrtc.TrackLocalStaticRTP + info localTrackInfo +} + +type calls struct { + callsMu sync.RWMutex + calls map[string]*call // By callID +} + +type conf struct { + confID string + calls calls + tracksMu sync.RWMutex + tracks []localTrackWithInfo +} + +func (f *focus) getConf(confID string, create bool) (*conf, error) { + f.confs.confsMu.Lock() + defer f.confs.confsMu.Unlock() + co := f.confs.confs[confID] + if co == nil { + if create { + co = &conf{ + confID: confID, + } + f.confs.confs[confID] = co + co.calls.calls = make(map[string]*call) + co.tracks = []localTrackWithInfo{} + } else { + return nil, errors.New("no such conf") + } + } + return co, nil +} + +func (c *conf) getCall(callID string, create bool) (*call, error) { + c.calls.callsMu.Lock() + defer c.calls.callsMu.Unlock() + ca := c.calls.calls[callID] + if ca == nil { + if create { + ca = &call{ + callID: callID, + conf: c, + } + c.calls.calls[callID] = ca + } else { + return nil, errors.New("no such call") + } + } + return ca, nil +} + +func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int, err error) { + foundIndices := []int{} + for index, track := range c.tracks { + info := track.info + if selectInfo.call != nil && selectInfo.call != info.call { + continue + } + if selectInfo.streamID != "" && selectInfo.streamID != info.streamID { + continue + } + if selectInfo.trackID != "" && selectInfo.trackID != info.trackID { + continue + } + foundIndices = append(foundIndices, index) + } + + if len(foundIndices) == 0 { + log.Printf("Found no tracks for %+v", selectInfo) + return nil, errors.New("no such tracks") + } else { + return foundIndices, nil + } +} + +func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() + + indices, err := c.getLocalTrackIndicesByInfo(selectInfo) + if err != nil { + return nil, err + } + + foundTracks := []webrtc.TrackLocal{} + for _, index := range indices { + foundTracks = append(foundTracks, c.tracks[index].track) + } + + if len(foundTracks) == 0 { + log.Printf("No tracks") + return nil, errors.New("no such tracks") + } else { + return foundTracks, nil + } +} + +func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() + + indices, err := c.getLocalTrackIndicesByInfo(removeInfo) + if err != nil { + return err + } + + // FIXME: the big O of this must be awful... + for _, index := range indices { + info := c.tracks[index].info + + for _, call := range c.calls.calls { + for _, sender := range call.peerConnection.GetSenders() { + if info.trackID == sender.Track().ID() { + log.Printf("%s | removing %s track with StreamID %s", call.callID, sender.Track().Kind(), info.streamID) + if err := sender.Stop(); err != nil { + log.Printf("%s | failed to stop sender: %s", call.callID, err) + } + if err := call.peerConnection.RemoveTrack(sender); err != nil { + log.Printf("%s | failed to remove track: %s", call.callID, err) + return err + } + } + } + } + } + + return nil +} diff --git a/src/focus.go b/src/focus.go index dae8a48..d0f4277 100644 --- a/src/focus.go +++ b/src/focus.go @@ -1,74 +1,9 @@ package main import ( - "encoding/json" - "errors" - "log" - "strings" "sync" - "time" - - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/event" - "maunium.net/go/mautrix/id" - - "github.com/pion/rtcp" - "github.com/pion/webrtc/v3" -) - -// stolen from matrix-js-sdk -// TODO: actually use callState (will be needed for renegotiation) -const ( - Fledgling = "fledgling" - InviteSent = "invite_sent" - WaitLocalMedia = "wait_local_media" - CreateOffer = "create_offer" - CreateAnswer = "create_answer" - Connecting = "connecting" - Connected = "connected" - Ringing = "ringing" - Ended = "ended" ) -type callState string - -type call struct { - callID string - userID id.UserID - deviceID id.DeviceID - localSessionID string - remoteSessionID string - client *mautrix.Client - peerConnection *webrtc.PeerConnection - callState callState - conf *conf - dataChannel *webrtc.DataChannel - // we track the call's tracks via the conf object. -} - -type localTrackInfo struct { - streamID string - trackID string - call *call -} - -type localTrackWithInfo struct { - track *webrtc.TrackLocalStaticRTP - info localTrackInfo -} - -type calls struct { - callsMu sync.RWMutex - calls map[string]*call // By callID -} - -type conf struct { - confID string - calls calls - tracksMu sync.RWMutex - tracks []localTrackWithInfo -} - type confs struct { confsMu sync.RWMutex confs map[string]*conf @@ -83,441 +18,3 @@ func (f *focus) Init(name string) { f.name = name f.confs.confs = make(map[string]*conf) } - -func (f *focus) getConf(confID string, create bool) (*conf, error) { - f.confs.confsMu.Lock() - defer f.confs.confsMu.Unlock() - co := f.confs.confs[confID] - if co == nil { - if create { - co = &conf{ - confID: confID, - } - f.confs.confs[confID] = co - co.calls.calls = make(map[string]*call) - co.tracks = []localTrackWithInfo{} - } else { - return nil, errors.New("no such conf") - } - } - return co, nil -} - -func (c *conf) getCall(callID string, create bool) (*call, error) { - c.calls.callsMu.Lock() - defer c.calls.callsMu.Unlock() - ca := c.calls.calls[callID] - if ca == nil { - if create { - ca = &call{ - callID: callID, - conf: c, - callState: WaitLocalMedia, - } - c.calls.calls[callID] = ca - } else { - return nil, errors.New("no such call") - } - } - return ca, nil -} - -func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int, err error) { - foundIndices := []int{} - for index, track := range c.tracks { - info := track.info - if selectInfo.call != nil && selectInfo.call != info.call { - continue - } - if selectInfo.streamID != "" && selectInfo.streamID != info.streamID { - continue - } - if selectInfo.trackID != "" && selectInfo.trackID != info.trackID { - continue - } - foundIndices = append(foundIndices, index) - } - - if len(foundIndices) == 0 { - log.Printf("Found no tracks for %+v", selectInfo) - return nil, errors.New("no such tracks") - } else { - return foundIndices, nil - } -} - -func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() - - indices, err := c.getLocalTrackIndicesByInfo(selectInfo) - if err != nil { - return nil, err - } - - foundTracks := []webrtc.TrackLocal{} - for _, index := range indices { - foundTracks = append(foundTracks, c.tracks[index].track) - } - - if len(foundTracks) == 0 { - log.Printf("No tracks") - return nil, errors.New("no such tracks") - } else { - return foundTracks, nil - } -} - -func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() - - indices, err := c.getLocalTrackIndicesByInfo(removeInfo) - if err != nil { - return err - } - - // FIXME: the big O of this must be awful... - for _, index := range indices { - info := c.tracks[index].info - - for _, call := range c.calls.calls { - for _, sender := range call.peerConnection.GetSenders() { - if info.trackID == sender.Track().ID() { - log.Printf("%s | removing %s track with StreamID %s", call.callID, sender.Track().Kind(), info.streamID) - if err := sender.Stop(); err != nil { - log.Printf("%s | failed to stop sender: %s", call.callID, err) - } - if err := call.peerConnection.RemoveTrack(sender); err != nil { - log.Printf("%s | failed to remove track: %s", call.callID, err) - return err - } - } - } - } - } - - return nil -} - -func (c *call) dataChannelHandler(d *webrtc.DataChannel) { - c.dataChannel = d - peerConnection := c.peerConnection - - sendError := func(errMsg string) { - log.Printf("%s | sending DC error %s", c.callID, errMsg) - marshaled, err := json.Marshal(&dataChannelMessage{ - Op: "error", - Message: errMsg, - }) - if err != nil { - panic(err) - } - - if err = d.SendText(string(marshaled)); err != nil { - panic(err) - } - } - - d.OnOpen(func() { - log.Printf("%s | DC opened", c.callID) - }) - - d.OnClose(func() { - log.Printf("%s | DC closed", c.callID) - }) - - d.OnError(func(err error) { - log.Fatalf("%s | DC error: %s", c.callID, err) - }) - - d.OnMessage(func(m webrtc.DataChannelMessage) { - if !m.IsString { - log.Fatal("Inbound message is not string") - } - - msg := &dataChannelMessage{} - if err := json.Unmarshal(m.Data, msg); err != nil { - log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) - } - - log.Printf("%s | received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) - - // TODO: hook cascade back up. - // As we're not an AS, we'd rely on the client - // to send us a "connect" op to tell us how to - // connect to another focus in order to select - // its streams. - - switch msg.Op { - case "select": - var tracks []webrtc.TrackLocal - for _, trackDesc := range msg.Start { - foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID}) - if err != nil { - sendError("No Such Stream") - return - } else { - tracks = append(tracks, foundTracks...) - } - } - - for _, track := range tracks { - log.Printf("%s | adding %s track with StreamID %s", c.callID, track.Kind(), track.StreamID()) - if _, err := peerConnection.AddTrack(track); err != nil { - panic(err) - } - } - - case "answer": - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeAnswer, - SDP: msg.SDP, - }) - - default: - log.Fatalf("Unknown operation %s", msg.Op) - // TODO: hook up msg.Stop to unsubscribe from tracks - } - }) -} - -func (c *call) negotiationNeeded() { - log.Printf("%s | negotiation needed", c.callID) - - offer, err := c.peerConnection.CreateOffer(nil) - if err != nil { - panic(err) - } - err = c.peerConnection.SetLocalDescription(offer) - if err != nil { - panic(err) - } - - response := dataChannelMessage{ - Op: "offer", - SDP: offer.SDP, - } - marshaled, err := json.Marshal(response) - if err != nil { - panic(err) - } - err = c.dataChannel.SendText(string(marshaled)) - if err != nil { - log.Printf("%s | failed to send over DC: %s", c.callID, err) - } - - log.Printf("%s | sent DC %s", c.callID, response.Op) -} - -func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { - if candidate == nil { - return - } - - ice := candidate.ToJSON() - - log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) - - // TODO: batch these up a bit - candidateEvtContent := &event.Content{ - Parsed: event.CallCandidatesEventContent{ - BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), - }, - Candidates: []event.CallCandidate{ - { - Candidate: ice.Candidate, - SDPMLineIndex: int(*ice.SDPMLineIndex), - SDPMID: *ice.SDPMid, - // XXX: what about ice.UsernameFragment? - }, - }, - }, - } - c.sendToDevice(event.CallCandidates, candidateEvtContent) -} - -func (c *call) onInvite(content *event.CallInviteEventContent) error { - offer := content.Offer - - peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) - if err != nil { - return err - } - c.peerConnection = peerConnection - - peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { - log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) - if strings.Contains(trackRemote.Codec().MimeType, "video") { - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval - go func() { - ticker := time.NewTicker(time.Millisecond * 200) - for range ticker.C { - if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { - log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) - break - } - } - }() - } - - c.conf.tracksMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) - if err != nil { - panic(err) - } - - c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ - track: trackLocal, - info: localTrackInfo{ - trackID: trackLocal.ID(), - streamID: trackLocal.StreamID(), - call: c, - }, - }) - - log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) - c.conf.tracksMu.Unlock() - - copyRemoteToLocal(trackRemote, trackLocal) - }) - - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - c.dataChannelHandler(d) - }) - peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - c.iceCandidateHandler(candidate) - }) - peerConnection.OnNegotiationNeeded(func() { - c.negotiationNeeded() - }) - - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: offer.SDP, - }) - - answer, err := peerConnection.CreateAnswer(nil) - if err != nil { - return err - } - - // TODO: trickle ICE for fast conn setup, rather than block here - gatherComplete := webrtc.GatheringCompletePromise(peerConnection) - if err = peerConnection.SetLocalDescription(answer); err != nil { - return err - } - <-gatherComplete - - answerSdp := peerConnection.LocalDescription().SDP - - answerEvtContent := &event.Content{ - Parsed: event.CallAnswerEventContent{ - BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), - Version: event.CallVersion("1"), - }, - Answer: event.CallData{ - Type: "answer", - SDP: answerSdp, - }, - }, - } - c.sendToDevice(event.CallAnswer, answerEvtContent) - - return err -} - -func (c *call) onSelectAnswer(content *event.CallSelectAnswerEventContent) { - selectedPartyId := content.SelectedPartyID - if selectedPartyId != string(c.client.DeviceID) { - c.terminate() - log.Printf("%s | Call was answered on a different device: %s", content.CallID, selectedPartyId) - } -} - -func (c *call) onHangup(content *event.CallHangupEventContent) { - c.terminate() -} - -func (c *call) terminate() error { - log.Printf("%s | Terminating call", c.callID) - - if err := c.peerConnection.Close(); err != nil { - log.Printf("%s | error closing peer connection: %s", c.callID, err) - } - - c.conf.calls.callsMu.Lock() - delete(c.conf.calls.calls, c.callID) - c.conf.calls.callsMu.Unlock() - - if err := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{call: c}); err != nil { - return err - } - - // TODO: Remove the tracks from conf.tracks - - return nil -} - -func (c *call) sendToDevice(callType event.Type, content *event.Content) error { - log.Printf("%s | sending to device %s", c.callID, callType.Type) - toDevice := &mautrix.ReqSendToDevice{ - Messages: map[id.UserID]map[id.DeviceID]*event.Content{ - c.userID: { - c.deviceID: content, - }, - }, - } - - // TODO: E2EE - // TODO: to-device reliability - c.client.SendToDevice(callType, toDevice) - - return nil -} - -func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { - for _, candidate := range content.Candidates { - sdpMLineIndex := uint16(candidate.SDPMLineIndex) - ice := webrtc.ICECandidateInit{ - Candidate: candidate.Candidate, - SDPMLineIndex: &sdpMLineIndex, - SDPMid: &candidate.SDPMID, - } - if err := c.peerConnection.AddICECandidate(ice); err != nil { - log.Print("Failed to add ICE candidate", content) - return err - } - } - return nil -} - -func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { - buff := make([]byte, 1500) - for { - i, _, err := trackRemote.Read(buff) - if err != nil || buff == nil { - log.Printf("ending read on track with StreamID %s: %s", trackRemote.StreamID(), err) - break - } - - if _, err = trackLocal.Write(buff[:i]); err != nil { - log.Printf("ending write on track with StreamID %s: %s", trackLocal.StreamID(), err) - break - } - } - -} diff --git a/src/utils.go b/src/utils.go new file mode 100644 index 0000000..ede6a58 --- /dev/null +++ b/src/utils.go @@ -0,0 +1,24 @@ +package main + +import ( + "log" + + "github.com/pion/webrtc/v3" +) + +func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { + buff := make([]byte, 1500) + for { + i, _, err := trackRemote.Read(buff) + if err != nil || buff == nil { + log.Printf("ending read on track with StreamID %s: %s", trackRemote.StreamID(), err) + break + } + + if _, err = trackLocal.Write(buff[:i]); err != nil { + log.Printf("ending write on track with StreamID %s: %s", trackLocal.StreamID(), err) + break + } + } + +} From 7d576806f01b669d21acc37e7b9a314d05ab6461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:27:55 +0200 Subject: [PATCH 043/124] Add newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 3990ed4..b1747a4 100755 --- a/build.sh +++ b/build.sh @@ -1 +1 @@ -go build -o dist/bin src/*.go \ No newline at end of file +go build -o dist/bin src/*.go From ab61cda63ee1eb327e7a07ea8f42a5d9af6ead41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:30:23 +0200 Subject: [PATCH 044/124] Remove `static` dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- static/index.html | 166 ---------------------------------------------- 1 file changed, 166 deletions(-) delete mode 100644 static/index.html diff --git a/static/index.html b/static/index.html deleted file mode 100644 index b09dabc..0000000 --- a/static/index.html +++ /dev/null @@ -1,166 +0,0 @@ - - - SFU to SFU - - - - - - Connection State: - New -
- -
-

Publish

- - - -
- - Do screenshare: - -

Subscribe

- - - - - -
- -
- -
- - - From 8eff32f68e413083fe6118ff9df95a616f5334c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:38:00 +0200 Subject: [PATCH 045/124] Delint `README.md` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- README.md | 102 +++++++++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 1e45f7d..c2e0e07 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ pub/sub semantics. The pub/sub streams are keyed by `foci`, `call_id`, `device_i Lets say you have a Matrix room where user `Alice` wishes to publish a screenshare to `Bob` and `Charlie`. -``` +```none * `Alice` establishes a session with a SFU * `Alice` publishes a screenshare feed with `call_id`, `device_id` and `purpose` * `Alice` publishes to the matrix room with the values `foci`, `call_id`, `device_id` and `purpose` @@ -32,7 +32,9 @@ Lets say you have a Matrix room where user `Alice` wishes to publish a screensha ``` ## How + ### Establishing a session + Client sends a POST with a WebRTC Offer that is datachannel only. Server responds with Answer. Server will open a datachannel called `signaling`. Clients can send publish/subscribe now. @@ -40,7 +42,8 @@ Server will open a datachannel called `signaling`. Clients can send publish/subs `POST /createSession` `Request` -``` + +```sdp o=- 6685856480478485828 2 IN IP4 127.0.0.1 s=- t=0 0 @@ -60,7 +63,8 @@ a=max-message-size:262144 ``` `Response` -``` + +```sdp o=- 1712750552704711910 2 IN IP4 127.0.0.1 s=- t=0 0 @@ -80,6 +84,7 @@ a=max-message-size:262144 ``` ### Publish a Stream + A user can start publish a stream by making a JSON request to publish with a new Offer. With the following keys. * `event` - Must be `publish` @@ -87,94 +92,95 @@ A user can start publish a stream by making a JSON request to publish with a new * Stream Identification - `call_id`, `device_id`, `purpose` * `sdp` - Offer frome the Peer. Any new additional tracks will belong to the stream. -``` +```json { - event: 'publish', - id: `ABC`, - call_id: 'AAA', - device_id: 'BBB', - purpose: 'DDD', - sdp: `...`, + "event": "publish", + "id": "ABC", + "call_id": "AAA", + "device_id": "BBB", + "purpose": "DDD", + "sdp": "...", } ``` -** Errors ** -* Stream already exists -* Server over capacity +* **Errors** + * Stream already exists + * Server over capacity The server will respond to the `subscribe` with the answer. -``` +```json { - event: 'publish', - id: `ABC`, - call_id: 'AAA', - device_id: 'BBB', - purpose: 'DDD', - sdp: `...`, + "event": "publish", + "id": "ABC", + "call_id": "AAA", + "device_id": "BBB", + "purpose": "DDD", + "sdp": "...", } ``` - ### Subscribe to a Stream + A user can subscribe to a stream by making a JSON request to subscribe with a new Offer. With the following keys. * `event` - Must be `subscribe` * `id` - Unique ID for this message. Allows server to respond with with errors * Stream Identification - `call_id`, `device_id`, `purpose` -``` +```json { - event: 'subscribe', - id: `ABC`, - call_id: 'AAA', - device_id: 'BBB', - purpose: 'DDD', - sdp: `...`, + "event": "subscribe", + "id": "ABC", + "call_id": "AAA", + "device_id": "BBB", + "purpose": "DDD", + "sdp": "...", } ``` The client will respond to the `subscribe` with the answer. -``` +```json { - event: 'subscribe', - id: `ABC`, - sdp: `...`, + "event": "subscribe", + "id": "ABC", + "sdp": "...", } ``` -** Errors ** -* Stream doesn'texist -* Server over capacity +* **Errors** + * Stream doesn't exist + * Server over capacity ### Unpublish a Stream -``` + +```json { - event: 'unpublish', - id: `ABC`, - call_id: 'AAA', - device_id: 'BBB', - purpose: 'DDD', + "event": "unpublish", + "id": "ABC", + "call_id": "AAA", + "device_id": "BBB", + "purpose": "DDD", } ``` ### Unsubscribe to a Stream -``` +```json { - event: 'unsubscribe', - id: `ABC`, - call_id: 'AAA', - device_id: 'BBB', - purpose: 'DDD', + "event": "unsubscribe", + "id": "ABC", + "call_id": "AAA", + "device_id": "BBB", + "purpose": "DDD", } ``` ## Running * `go run ./src/*.go` -* Access at http://localhost:8080 +* Access at ## Building From 484aed860afd81a65472096481b6b65d19b56405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 10:39:41 +0200 Subject: [PATCH 046/124] Add copyrights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 16 ++++++++++++++++ src/cascade.go | 16 ++++++++++++++++ src/conf.go | 16 ++++++++++++++++ src/focus.go | 16 ++++++++++++++++ src/main.go | 16 ++++++++++++++++ src/matrix.go | 16 ++++++++++++++++ src/utils.go | 16 ++++++++++++++++ 7 files changed, 112 insertions(+) diff --git a/src/call.go b/src/call.go index ff47848..5654116 100644 --- a/src/call.go +++ b/src/call.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/src/cascade.go b/src/cascade.go index 51ea4fc..cf8ca67 100644 --- a/src/cascade.go +++ b/src/cascade.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main /* diff --git a/src/conf.go b/src/conf.go index 6b3e4c2..ffb7c2e 100644 --- a/src/conf.go +++ b/src/conf.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/src/focus.go b/src/focus.go index d0f4277..5688806 100644 --- a/src/focus.go +++ b/src/focus.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/src/main.go b/src/main.go index 6b1d493..64ae80d 100644 --- a/src/main.go +++ b/src/main.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/src/matrix.go b/src/matrix.go index e3027b0..68faca9 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/src/utils.go b/src/utils.go index ede6a58..242786f 100644 --- a/src/utils.go +++ b/src/utils.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( From 36593665caa6e857d91dccad05714bc2555336da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 19:23:10 +0200 Subject: [PATCH 047/124] Extract `onTrack` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 73 ++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/src/call.go b/src/call.go index 5654116..be16cc1 100644 --- a/src/call.go +++ b/src/call.go @@ -186,6 +186,42 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { c.sendToDevice(event.CallCandidates, candidateEvtContent) } +func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { + log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) + if strings.Contains(trackRemote.Codec().MimeType, "video") { + // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + go func() { + ticker := time.NewTicker(time.Millisecond * 200) + for range ticker.C { + if err := c.peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) + break + } + } + }() + } + + c.conf.tracksMu.Lock() + trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) + if err != nil { + panic(err) + } + + c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ + track: trackLocal, + info: localTrackInfo{ + trackID: trackLocal.ID(), + streamID: trackLocal.StreamID(), + call: c, + }, + }) + + log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) + c.conf.tracksMu.Unlock() + + copyRemoteToLocal(trackRemote, trackLocal) +} + func (c *call) onInvite(content *event.CallInviteEventContent) error { offer := content.Offer @@ -195,42 +231,9 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { } c.peerConnection = peerConnection - peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { - log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) - if strings.Contains(trackRemote.Codec().MimeType, "video") { - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval - go func() { - ticker := time.NewTicker(time.Millisecond * 200) - for range ticker.C { - if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { - log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) - break - } - } - }() - } - - c.conf.tracksMu.Lock() - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) - if err != nil { - panic(err) - } - - c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ - track: trackLocal, - info: localTrackInfo{ - trackID: trackLocal.ID(), - streamID: trackLocal.StreamID(), - call: c, - }, - }) - - log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) - c.conf.tracksMu.Unlock() - - copyRemoteToLocal(trackRemote, trackLocal) + peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { + c.trackHandler(track, receiver) }) - peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { c.dataChannelHandler(d) }) From c6904317ad44ee08e458a3b38e4e119486100508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 19:28:37 +0200 Subject: [PATCH 048/124] Extract `sendDataChannelMessage()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/call.go b/src/call.go index be16cc1..bdd3d5b 100644 --- a/src/call.go +++ b/src/call.go @@ -136,20 +136,10 @@ func (c *call) negotiationNeededHandler() { panic(err) } - response := dataChannelMessage{ + c.sendDataChannelMessage(dataChannelMessage{ Op: "offer", SDP: offer.SDP, - } - marshaled, err := json.Marshal(response) - if err != nil { - panic(err) - } - err = c.dataChannel.SendText(string(marshaled)) - if err != nil { - log.Printf("%s | failed to send over DC: %s", c.callID, err) - } - - log.Printf("%s | sent DC %s", c.callID, response.Op) + }) } func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { @@ -350,3 +340,20 @@ func (c *call) sendToDevice(callType event.Type, content *event.Content) error { return nil } + +func (c *call) sendDataChannelMessage(msg dataChannelMessage) { + msg.ConfID = c.conf.confID + // TODO: Set ID + + marshaled, err := json.Marshal(msg) + if err != nil { + panic(err) + } + + err = c.dataChannel.SendText(string(marshaled)) + if err != nil { + log.Printf("%s | failed to send over DC: %s", c.callID, err) + } + + log.Printf("%s | sent DC %s", c.callID, msg.Op) +} From 2d8cf74c982b9078f74884ee41038a1ca2ed7b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 19:41:12 +0200 Subject: [PATCH 049/124] Add support for `publish` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/call.go b/src/call.go index bdd3d5b..0db2de0 100644 --- a/src/call.go +++ b/src/call.go @@ -111,6 +111,26 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } } + case "publish": + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: msg.SDP, + }) + + offer, err := c.peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + err = c.peerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + c.sendDataChannelMessage(dataChannelMessage{ + Op: "answer", + SDP: offer.SDP, + }) + case "answer": peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, From 5b62e458f62ef98873f4e68a091f6553260123c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 2 Aug 2022 19:58:20 +0200 Subject: [PATCH 050/124] Extract `sendDataChannelError()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/call.go b/src/call.go index 0db2de0..d124eb7 100644 --- a/src/call.go +++ b/src/call.go @@ -46,21 +46,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { c.dataChannel = d peerConnection := c.peerConnection - sendError := func(errMsg string) { - log.Printf("%s | sending DC error %s", c.callID, errMsg) - marshaled, err := json.Marshal(&dataChannelMessage{ - Op: "error", - Message: errMsg, - }) - if err != nil { - panic(err) - } - - if err = d.SendText(string(marshaled)); err != nil { - panic(err) - } - } - d.OnOpen(func() { log.Printf("%s | DC opened", c.callID) }) @@ -97,7 +82,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { for _, trackDesc := range msg.Start { foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID}) if err != nil { - sendError("No Such Stream") + c.sendDataChannelError("No Such Stream") return } else { tracks = append(tracks, foundTracks...) @@ -377,3 +362,18 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { log.Printf("%s | sent DC %s", c.callID, msg.Op) } + +func (c *call) sendDataChannelError(errMsg string) { + log.Printf("%s | sending DC error %s", c.callID, errMsg) + marshaled, err := json.Marshal(&dataChannelMessage{ + Op: "error", + Message: errMsg, + }) + if err != nil { + panic(err) + } + + if err = c.dataChannel.SendText(string(marshaled)); err != nil { + panic(err) + } +} From a78c8ec4fd4f5223c14da4226e4847c97a3cbc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 10:27:56 +0200 Subject: [PATCH 051/124] Correctly remove tracks when a person leaves a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 8 +++++--- src/conf.go | 30 +++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/call.go b/src/call.go index d124eb7..62cbcd4 100644 --- a/src/call.go +++ b/src/call.go @@ -320,11 +320,13 @@ func (c *call) terminate() error { delete(c.conf.calls.calls, c.callID) c.conf.calls.callsMu.Unlock() - if err := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{call: c}); err != nil { + info := localTrackInfo{call: c} + if err := c.conf.removeTracksFromPeerConnectionsByInfo(info); err != nil { + return err + } + if err := c.conf.removeTracksFromConfByInfo(info); err != nil { return err } - - // TODO: Remove the tracks from conf.tracks return nil } diff --git a/src/conf.go b/src/conf.go index ffb7c2e..e63508a 100644 --- a/src/conf.go +++ b/src/conf.go @@ -131,9 +131,6 @@ func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.T } func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() - indices, err := c.getLocalTrackIndicesByInfo(removeInfo) if err != nil { return err @@ -161,3 +158,30 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) return nil } + +func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) error { + c.tracksMu.Lock() + defer c.tracksMu.Unlock() + + indicesToRemove, err := c.getLocalTrackIndicesByInfo(removeInfo) + if err != nil { + return err + } + + newTracks := []localTrackWithInfo{} + for index, track := range c.tracks { + keep := true + for _, indexToRemove := range indicesToRemove { + if indexToRemove == index { + keep = false + } + } + if keep { + newTracks = append(newTracks, track) + } + } + + c.tracks = newTracks + + return nil +} From 37b1c35e4396c274a83986495e9eaeca5135f6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 11:04:34 +0200 Subject: [PATCH 052/124] Update scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- build.sh | 2 ++ run.sh | 3 +++ 2 files changed, 5 insertions(+) create mode 100755 run.sh diff --git a/build.sh b/build.sh index b1747a4..5011d8c 100755 --- a/build.sh +++ b/build.sh @@ -1 +1,3 @@ +#!/usr/bin/env bash + go build -o dist/bin src/*.go diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..4cc62aa --- /dev/null +++ b/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +clear && go run ./src/*.go \ No newline at end of file From 6d91759fbc8d98b29692a871eb591a0dae860093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 11:14:35 +0200 Subject: [PATCH 053/124] Move scripts into a separete dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- build.sh => scripts/build.sh | 0 run.sh => scripts/run.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename build.sh => scripts/build.sh (100%) rename run.sh => scripts/run.sh (100%) diff --git a/build.sh b/scripts/build.sh similarity index 100% rename from build.sh rename to scripts/build.sh diff --git a/run.sh b/scripts/run.sh similarity index 100% rename from run.sh rename to scripts/run.sh From 1ad36e9eddb77fcca8c98db07368310de8dd6e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 11:15:23 +0200 Subject: [PATCH 054/124] Update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2e0e07..fbfea3d 100644 --- a/README.md +++ b/README.md @@ -179,11 +179,11 @@ The client will respond to the `subscribe` with the answer. ## Running -* `go run ./src/*.go` +* `./scripts/run.sh` * Access at ## Building -* `./build.zsh` +* `./scripts/build.sh` * `./dist/bin` * Access at From ee5a4c38e1e6f67c32b73d7fd64abda61a059728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 14:31:11 +0200 Subject: [PATCH 055/124] Select by both `streamId` and `trackId` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index 62cbcd4..364fd23 100644 --- a/src/call.go +++ b/src/call.go @@ -80,7 +80,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { - foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID}) + foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID, trackID: trackDesc.TrackID}) if err != nil { c.sendDataChannelError("No Such Stream") return From 805244ed927dc768094c38bc6c901e146baa2594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 15:23:50 +0200 Subject: [PATCH 056/124] Improve logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 8 +++----- src/conf.go | 2 +- src/matrix.go | 24 ++++++++++++++---------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/call.go b/src/call.go index 364fd23..c08ee32 100644 --- a/src/call.go +++ b/src/call.go @@ -90,7 +90,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } for _, track := range tracks { - log.Printf("%s | adding %s track with StreamID %s", c.callID, track.Kind(), track.StreamID()) + log.Printf("%s | adding %s track with %s", c.callID, track.Kind(), track.ID()) if _, err := peerConnection.AddTrack(track); err != nil { panic(err) } @@ -154,8 +154,6 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { ice := candidate.ToJSON() - log.Printf("%s | discovered local candidate %s", c.callID, ice.Candidate) - // TODO: batch these up a bit candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ @@ -310,7 +308,7 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { } func (c *call) terminate() error { - log.Printf("%s | Terminating call", c.callID) + log.Printf("%s | terminating call", c.callID) if err := c.peerConnection.Close(); err != nil { log.Printf("%s | error closing peer connection: %s", c.callID, err) @@ -359,7 +357,7 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { err = c.dataChannel.SendText(string(marshaled)) if err != nil { - log.Printf("%s | failed to send over DC: %s", c.callID, err) + log.Printf("%s | failed to send %s over DC: %s", c.callID, msg.Op, err) } log.Printf("%s | sent DC %s", c.callID, msg.Op) diff --git a/src/conf.go b/src/conf.go index e63508a..a7926ea 100644 --- a/src/conf.go +++ b/src/conf.go @@ -101,7 +101,7 @@ func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []i } if len(foundIndices) == 0 { - log.Printf("Found no tracks for %+v", selectInfo) + log.Printf("found no tracks for %+v", selectInfo) return nil, errors.New("no such tracks") } else { return foundIndices, nil diff --git a/src/matrix.go b/src/matrix.go index 68faca9..442a5f5 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -73,11 +73,11 @@ func initMatrix(config *config) error { var call *call if conf, err = focus.getConf(confID, false); err != nil || conf == nil { - log.Printf("Failed to get conf %s %+v", confID, err) + log.Printf("%s | failed to get conf %+v", confID, err) return nil, err } if call, err = conf.getCall(callID, false); err != nil || call == nil { - log.Printf("Failed to get call %s %+v", callID, err) + log.Printf("%s | failed to get call %+v", callID, err) return nil, err } return call, nil @@ -88,7 +88,7 @@ func initMatrix(config *config) error { evt.Type.Class = event.ToDeviceEventType err := evt.Content.ParseRaw(evt.Type) if err != nil { - log.Printf("Failed to parse to-device event of type %s: %v", evt.Type.Type, err) + log.Printf("failed to parse to-device event of type %s: %v", evt.Type.Type, err) continue } @@ -96,9 +96,9 @@ func initMatrix(config *config) error { var call *call if strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { - log.Printf("%s | Received to-device event %s", evt.Content.Raw["call_id"], evt.Type.Type) + log.Printf("%s | received to-device event %s", evt.Content.Raw["call_id"], evt.Type.Type) } else { - log.Printf("Received non-call to-device event %s", evt.Type.Type) + log.Printf("received non-call to-device event %s", evt.Type.Type) continue } @@ -107,11 +107,11 @@ func initMatrix(config *config) error { case CallInvite.Type: invite := evt.Content.AsCallInvite() if conf, err = focus.getConf(invite.ConfID, true); err != nil || conf == nil { - log.Printf("Failed to create conf %s %+v", invite.ConfID, err) + log.Printf("%s | failed to create conf %s: %+v", invite.CallID, invite.ConfID, err) return true } if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { - log.Printf("Failed to create call %s %+v", invite.CallID, err) + log.Printf("%s | failed to create call: %+v", invite.CallID, err) return true } call.userID = evt.Sender @@ -143,12 +143,16 @@ func initMatrix(config *config) error { // Events we don't care about case CallNegotiate.Type: - log.Printf("Ignoring event %s as should be handled over DC", evt.Type.Type) + negotiate := evt.Content.AsCallNegotiate() + log.Printf("%s | ignoring event %s as should be handled over DC", negotiate.CallID, evt.Type.Type) case CallReject.Type: + reject := evt.Content.AsCallReject() + log.Printf("%s | ignoring event %s as we are always the ones answering", reject.CallID, evt.Type.Type) case CallAnswer.Type: - log.Printf("Ignoring event %s as we are always the ones answering", evt.Type.Type) + answer := evt.Content.AsCallAnswer() + log.Printf("%s | ignoring event %s as we are always the ones answering", answer.CallID, evt.Type.Type) default: - log.Printf("Ignoring unrecognised to-device event of type %s", evt.Type.Type) + log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Content.Raw["call_id"], evt.Type.Type) } } From cb7c76f7f6ceaf6b33160488cf2d08a90c7fa44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 15:30:30 +0200 Subject: [PATCH 057/124] Correctly log `select` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index c08ee32..5464625 100644 --- a/src/call.go +++ b/src/call.go @@ -68,7 +68,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) } - log.Printf("%s | received DC %s confId=%s start=%+v", c.callID, msg.Op, msg.ConfID, msg.Start) + log.Printf("%s | received DC: %s", c.callID, msg.Op) // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -78,6 +78,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { switch msg.Op { case "select": + log.Printf("%s | selected: %+v", c.callID, msg.Start) + var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID, trackID: trackDesc.TrackID}) From 83981196ce6c0e5249ab66eec8aed1353f9ce68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 15:36:46 +0200 Subject: [PATCH 058/124] Log `userID`s instead of `callID`s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 30 +++++++++++++++--------------- src/conf.go | 6 +++--- src/matrix.go | 20 ++++++++------------ 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/call.go b/src/call.go index 5464625..a502aba 100644 --- a/src/call.go +++ b/src/call.go @@ -47,11 +47,11 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { peerConnection := c.peerConnection d.OnOpen(func() { - log.Printf("%s | DC opened", c.callID) + log.Printf("%s | DC opened", c.userID) }) d.OnClose(func() { - log.Printf("%s | DC closed", c.callID) + log.Printf("%s | DC closed", c.userID) }) d.OnError(func(err error) { @@ -68,7 +68,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) } - log.Printf("%s | received DC: %s", c.callID, msg.Op) + log.Printf("%s | received DC: %s", c.userID, msg.Op) // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -78,7 +78,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { switch msg.Op { case "select": - log.Printf("%s | selected: %+v", c.callID, msg.Start) + log.Printf("%s | selected: %+v", c.userID, msg.Start) var tracks []webrtc.TrackLocal for _, trackDesc := range msg.Start { @@ -92,7 +92,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } for _, track := range tracks { - log.Printf("%s | adding %s track with %s", c.callID, track.Kind(), track.ID()) + log.Printf("%s | adding %s track with %s", c.userID, track.Kind(), track.ID()) if _, err := peerConnection.AddTrack(track); err != nil { panic(err) } @@ -132,7 +132,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } func (c *call) negotiationNeededHandler() { - log.Printf("%s | negotiation needed", c.callID) + log.Printf("%s | negotiation needed", c.userID) offer, err := c.peerConnection.CreateOffer(nil) if err != nil { @@ -182,14 +182,14 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { } func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { - log.Printf("%s | discovered track with streamID %s and kind %s", c.callID, trackRemote.StreamID(), trackRemote.Kind()) + log.Printf("%s | discovered track with streamID %s and kind %s", c.userID, trackRemote.StreamID(), trackRemote.Kind()) if strings.Contains(trackRemote.Codec().MimeType, "video") { // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval go func() { ticker := time.NewTicker(time.Millisecond * 200) for range ticker.C { if err := c.peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { - log.Printf("%s | failed to write RTCP on trackID %s: %s", c.callID, trackRemote.ID(), err) + log.Printf("%s | failed to write RTCP on trackID %s: %s", c.userID, trackRemote.ID(), err) break } } @@ -211,7 +211,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }, }) - log.Printf("%s | published track with streamID %s and kind %s", c.callID, trackLocal.StreamID(), trackLocal.Kind()) + log.Printf("%s | published track with streamID %s and kind %s", c.userID, trackLocal.StreamID(), trackLocal.Kind()) c.conf.tracksMu.Unlock() copyRemoteToLocal(trackRemote, trackLocal) @@ -284,7 +284,7 @@ func (c *call) onSelectAnswer(content *event.CallSelectAnswerEventContent) { selectedPartyId := content.SelectedPartyID if selectedPartyId != string(c.client.DeviceID) { c.terminate() - log.Printf("%s | Call was answered on a different device: %s", content.CallID, selectedPartyId) + log.Printf("%s | Call was answered on a different device: %s", c.userID, selectedPartyId) } } @@ -313,7 +313,7 @@ func (c *call) terminate() error { log.Printf("%s | terminating call", c.callID) if err := c.peerConnection.Close(); err != nil { - log.Printf("%s | error closing peer connection: %s", c.callID, err) + log.Printf("%s | error closing peer connection: %s", c.userID, err) } c.conf.calls.callsMu.Lock() @@ -332,7 +332,7 @@ func (c *call) terminate() error { } func (c *call) sendToDevice(callType event.Type, content *event.Content) error { - log.Printf("%s | sending to device %s", c.callID, callType.Type) + log.Printf("%s | sending to device %s", c.userID, callType.Type) toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ c.userID: { @@ -359,14 +359,14 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { err = c.dataChannel.SendText(string(marshaled)) if err != nil { - log.Printf("%s | failed to send %s over DC: %s", c.callID, msg.Op, err) + log.Printf("%s | failed to send %s over DC: %s", c.userID, msg.Op, err) } - log.Printf("%s | sent DC %s", c.callID, msg.Op) + log.Printf("%s | sent DC %s", c.userID, msg.Op) } func (c *call) sendDataChannelError(errMsg string) { - log.Printf("%s | sending DC error %s", c.callID, errMsg) + log.Printf("%s | sending DC error: %s", c.userID, errMsg) marshaled, err := json.Marshal(&dataChannelMessage{ Op: "error", Message: errMsg, diff --git a/src/conf.go b/src/conf.go index a7926ea..8231d4a 100644 --- a/src/conf.go +++ b/src/conf.go @@ -143,12 +143,12 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) for _, call := range c.calls.calls { for _, sender := range call.peerConnection.GetSenders() { if info.trackID == sender.Track().ID() { - log.Printf("%s | removing %s track with StreamID %s", call.callID, sender.Track().Kind(), info.streamID) + log.Printf("%s | removing %s track with StreamID %s", call.userID, sender.Track().Kind(), info.streamID) if err := sender.Stop(); err != nil { - log.Printf("%s | failed to stop sender: %s", call.callID, err) + log.Printf("%s | failed to stop sender: %s", call.userID, err) } if err := call.peerConnection.RemoveTrack(sender); err != nil { - log.Printf("%s | failed to remove track: %s", call.callID, err) + log.Printf("%s | failed to remove track: %s", call.userID, err) return err } } diff --git a/src/matrix.go b/src/matrix.go index 442a5f5..8372503 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -73,11 +73,11 @@ func initMatrix(config *config) error { var call *call if conf, err = focus.getConf(confID, false); err != nil || conf == nil { - log.Printf("%s | failed to get conf %+v", confID, err) + log.Printf("failed to get conf %s: %s", confID, err) return nil, err } if call, err = conf.getCall(callID, false); err != nil || call == nil { - log.Printf("%s | failed to get call %+v", callID, err) + log.Printf("failed to get call %s: %s", callID, err) return nil, err } return call, nil @@ -96,7 +96,7 @@ func initMatrix(config *config) error { var call *call if strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { - log.Printf("%s | received to-device event %s", evt.Content.Raw["call_id"], evt.Type.Type) + log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) } else { log.Printf("received non-call to-device event %s", evt.Type.Type) continue @@ -107,11 +107,11 @@ func initMatrix(config *config) error { case CallInvite.Type: invite := evt.Content.AsCallInvite() if conf, err = focus.getConf(invite.ConfID, true); err != nil || conf == nil { - log.Printf("%s | failed to create conf %s: %+v", invite.CallID, invite.ConfID, err) + log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) return true } if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { - log.Printf("%s | failed to create call: %+v", invite.CallID, err) + log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) return true } call.userID = evt.Sender @@ -143,16 +143,12 @@ func initMatrix(config *config) error { // Events we don't care about case CallNegotiate.Type: - negotiate := evt.Content.AsCallNegotiate() - log.Printf("%s | ignoring event %s as should be handled over DC", negotiate.CallID, evt.Type.Type) + log.Printf("%s | ignoring event %s as should be handled over DC", evt.Sender.String(), evt.Type.Type) case CallReject.Type: - reject := evt.Content.AsCallReject() - log.Printf("%s | ignoring event %s as we are always the ones answering", reject.CallID, evt.Type.Type) case CallAnswer.Type: - answer := evt.Content.AsCallAnswer() - log.Printf("%s | ignoring event %s as we are always the ones answering", answer.CallID, evt.Type.Type) + log.Printf("%s | ignoring event %s as we are always the ones answering", evt.Sender.String(), evt.Type.Type) default: - log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Content.Raw["call_id"], evt.Type.Type) + log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Sender.String(), evt.Type.Type) } } From cd3076be3090cfe2705e7e634829b1b169cbeaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 17:47:37 +0200 Subject: [PATCH 059/124] Fix terminate call log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index a502aba..d329c2c 100644 --- a/src/call.go +++ b/src/call.go @@ -310,7 +310,7 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { } func (c *call) terminate() error { - log.Printf("%s | terminating call", c.callID) + log.Printf("%s | terminating call", c.userID) if err := c.peerConnection.Close(); err != nil { log.Printf("%s | error closing peer connection: %s", c.userID, err) From b999bda7e7ea0d26f8709188b7b6097be3b4b3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 19:57:00 +0200 Subject: [PATCH 060/124] Fix more logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/call.go b/src/call.go index d329c2c..e4ec5d8 100644 --- a/src/call.go +++ b/src/call.go @@ -182,7 +182,6 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { } func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { - log.Printf("%s | discovered track with streamID %s and kind %s", c.userID, trackRemote.StreamID(), trackRemote.Kind()) if strings.Contains(trackRemote.Codec().MimeType, "video") { // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval go func() { @@ -211,7 +210,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }, }) - log.Printf("%s | published track with streamID %s and kind %s", c.userID, trackLocal.StreamID(), trackLocal.Kind()) + log.Printf("%s | published track with trackID %s and kind %s", c.userID, trackLocal.ID(), trackLocal.Kind()) c.conf.tracksMu.Unlock() copyRemoteToLocal(trackRemote, trackLocal) From 0d373b02ff3de8fc6e6e6478e906474313f63209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 20:22:45 +0200 Subject: [PATCH 061/124] Don't error if we found no tracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 +--- src/conf.go | 41 ++++++++--------------------------------- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/src/call.go b/src/call.go index e4ec5d8..4d033c5 100644 --- a/src/call.go +++ b/src/call.go @@ -323,9 +323,7 @@ func (c *call) terminate() error { if err := c.conf.removeTracksFromPeerConnectionsByInfo(info); err != nil { return err } - if err := c.conf.removeTracksFromConfByInfo(info); err != nil { - return err - } + c.conf.removeTracksFromConfByInfo(info) return nil } diff --git a/src/conf.go b/src/conf.go index 8231d4a..abe49c7 100644 --- a/src/conf.go +++ b/src/conf.go @@ -84,7 +84,7 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { return ca, nil } -func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int, err error) { +func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int) { foundIndices := []int{} for index, track := range c.tracks { info := track.info @@ -100,41 +100,21 @@ func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []i foundIndices = append(foundIndices, index) } - if len(foundIndices) == 0 { - log.Printf("found no tracks for %+v", selectInfo) - return nil, errors.New("no such tracks") - } else { - return foundIndices, nil - } + return foundIndices } -func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal, err error) { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() - - indices, err := c.getLocalTrackIndicesByInfo(selectInfo) - if err != nil { - return nil, err - } - +func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal) { + indices := c.getLocalTrackIndicesByInfo(selectInfo) foundTracks := []webrtc.TrackLocal{} for _, index := range indices { foundTracks = append(foundTracks, c.tracks[index].track) } - if len(foundTracks) == 0 { - log.Printf("No tracks") - return nil, errors.New("no such tracks") - } else { - return foundTracks, nil - } + return foundTracks } func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { - indices, err := c.getLocalTrackIndicesByInfo(removeInfo) - if err != nil { - return err - } + indices := c.getLocalTrackIndicesByInfo(removeInfo) // FIXME: the big O of this must be awful... for _, index := range indices { @@ -159,14 +139,11 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) return nil } -func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) error { +func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { c.tracksMu.Lock() defer c.tracksMu.Unlock() - indicesToRemove, err := c.getLocalTrackIndicesByInfo(removeInfo) - if err != nil { - return err - } + indicesToRemove := c.getLocalTrackIndicesByInfo(removeInfo) newTracks := []localTrackWithInfo{} for index, track := range c.tracks { @@ -182,6 +159,4 @@ func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) error { } c.tracks = newTracks - - return nil } From 5657c6d7c850ae30ea6de9f1e95a0816b04f639d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 20:33:45 +0200 Subject: [PATCH 062/124] Try to fix race condition where track is later than state event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 78 ++++++++++++++++++++++++++++++----------------------- src/conf.go | 1 + 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/call.go b/src/call.go index 4d033c5..5457f60 100644 --- a/src/call.go +++ b/src/call.go @@ -20,6 +20,7 @@ import ( "encoding/json" "log" "strings" + "sync" "time" "maunium.net/go/mautrix" @@ -30,16 +31,22 @@ import ( "github.com/pion/webrtc/v3" ) +type subscribedTracks struct { + mutex sync.RWMutex + tracks []localTrackInfo +} + type call struct { - callID string - userID id.UserID - deviceID id.DeviceID - localSessionID string - remoteSessionID string - client *mautrix.Client - peerConnection *webrtc.PeerConnection - conf *conf - dataChannel *webrtc.DataChannel + callID string + userID id.UserID + deviceID id.DeviceID + localSessionID string + remoteSessionID string + client *mautrix.Client + peerConnection *webrtc.PeerConnection + conf *conf + dataChannel *webrtc.DataChannel + subscribedTracks subscribedTracks } func (c *call) dataChannelHandler(d *webrtc.DataChannel) { @@ -80,23 +87,16 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "select": log.Printf("%s | selected: %+v", c.userID, msg.Start) - var tracks []webrtc.TrackLocal + c.subscribedTracks.mutex.Lock() for _, trackDesc := range msg.Start { - foundTracks, err := c.conf.getLocalTrackByInfo(localTrackInfo{streamID: trackDesc.StreamID, trackID: trackDesc.TrackID}) - if err != nil { - c.sendDataChannelError("No Such Stream") - return - } else { - tracks = append(tracks, foundTracks...) - } + c.subscribedTracks.tracks = append(c.subscribedTracks.tracks, localTrackInfo{ + streamID: trackDesc.StreamID, + trackID: trackDesc.TrackID, + }) } + c.subscribedTracks.mutex.Unlock() - for _, track := range tracks { - log.Printf("%s | adding %s track with %s", c.userID, track.Kind(), track.ID()) - if _, err := peerConnection.AddTrack(track); err != nil { - panic(err) - } - } + c.addSubscribedTracksToPeerConnection() case "publish": peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -213,6 +213,8 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece log.Printf("%s | published track with trackID %s and kind %s", c.userID, trackLocal.ID(), trackLocal.Kind()) c.conf.tracksMu.Unlock() + c.addSubscribedTracksToPeerConnection() + copyRemoteToLocal(trackRemote, trackLocal) } @@ -362,17 +364,27 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { log.Printf("%s | sent DC %s", c.userID, msg.Op) } -func (c *call) sendDataChannelError(errMsg string) { - log.Printf("%s | sending DC error: %s", c.userID, errMsg) - marshaled, err := json.Marshal(&dataChannelMessage{ - Op: "error", - Message: errMsg, - }) - if err != nil { - panic(err) +func (c *call) addSubscribedTracksToPeerConnection() { + newSubscribedTracks := []localTrackInfo{} + tracksToAddToPeerConnection := []webrtc.TrackLocal{} + + c.subscribedTracks.mutex.Lock() + for _, trackInfo := range c.subscribedTracks.tracks { + foundTracks := c.conf.getLocalTrackByInfo(trackInfo) + if len(foundTracks) == 0 { + log.Printf("%s | no track found for %+v", c.userID, trackInfo) + newSubscribedTracks = append(newSubscribedTracks, trackInfo) + } else { + tracksToAddToPeerConnection = append(tracksToAddToPeerConnection, foundTracks...) + } } + c.subscribedTracks.tracks = newSubscribedTracks + c.subscribedTracks.mutex.Unlock() - if err = c.dataChannel.SendText(string(marshaled)); err != nil { - panic(err) + for _, track := range tracksToAddToPeerConnection { + log.Printf("%s | adding %s track with %s", c.userID, track.Kind(), track.ID()) + if _, err := c.peerConnection.AddTrack(track); err != nil { + panic(err) + } } } diff --git a/src/conf.go b/src/conf.go index abe49c7..51f1f57 100644 --- a/src/conf.go +++ b/src/conf.go @@ -76,6 +76,7 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { callID: callID, conf: c, } + ca.subscribedTracks.tracks = []localTrackInfo{} c.calls.calls[callID] = ca } else { return nil, errors.New("no such call") From 060a5286effd6d37b7119f5614933e4ae0853af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 3 Aug 2022 20:40:40 +0200 Subject: [PATCH 063/124] Fix `tracks` mutex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 6 +++--- src/conf.go | 28 ++++++++++++++++------------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/call.go b/src/call.go index 5457f60..d569e74 100644 --- a/src/call.go +++ b/src/call.go @@ -195,13 +195,13 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }() } - c.conf.tracksMu.Lock() + c.conf.tracks.mutex.Lock() trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) if err != nil { panic(err) } - c.conf.tracks = append(c.conf.tracks, localTrackWithInfo{ + c.conf.tracks.tracks = append(c.conf.tracks.tracks, localTrackWithInfo{ track: trackLocal, info: localTrackInfo{ trackID: trackLocal.ID(), @@ -211,7 +211,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }) log.Printf("%s | published track with trackID %s and kind %s", c.userID, trackLocal.ID(), trackLocal.Kind()) - c.conf.tracksMu.Unlock() + c.conf.tracks.mutex.Unlock() c.addSubscribedTracksToPeerConnection() diff --git a/src/conf.go b/src/conf.go index 51f1f57..7bdfd8a 100644 --- a/src/conf.go +++ b/src/conf.go @@ -40,11 +40,15 @@ type calls struct { calls map[string]*call // By callID } +type tracks struct { + mutex sync.RWMutex + tracks []localTrackWithInfo +} + type conf struct { - confID string - calls calls - tracksMu sync.RWMutex - tracks []localTrackWithInfo + confID string + calls calls + tracks tracks } func (f *focus) getConf(confID string, create bool) (*conf, error) { @@ -58,7 +62,7 @@ func (f *focus) getConf(confID string, create bool) (*conf, error) { } f.confs.confs[confID] = co co.calls.calls = make(map[string]*call) - co.tracks = []localTrackWithInfo{} + co.tracks.tracks = []localTrackWithInfo{} } else { return nil, errors.New("no such conf") } @@ -87,7 +91,7 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int) { foundIndices := []int{} - for index, track := range c.tracks { + for index, track := range c.tracks.tracks { info := track.info if selectInfo.call != nil && selectInfo.call != info.call { continue @@ -108,7 +112,7 @@ func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.T indices := c.getLocalTrackIndicesByInfo(selectInfo) foundTracks := []webrtc.TrackLocal{} for _, index := range indices { - foundTracks = append(foundTracks, c.tracks[index].track) + foundTracks = append(foundTracks, c.tracks.tracks[index].track) } return foundTracks @@ -119,7 +123,7 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) // FIXME: the big O of this must be awful... for _, index := range indices { - info := c.tracks[index].info + info := c.tracks.tracks[index].info for _, call := range c.calls.calls { for _, sender := range call.peerConnection.GetSenders() { @@ -141,13 +145,13 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) } func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { - c.tracksMu.Lock() - defer c.tracksMu.Unlock() + c.tracks.mutex.Lock() + defer c.tracks.mutex.Unlock() indicesToRemove := c.getLocalTrackIndicesByInfo(removeInfo) newTracks := []localTrackWithInfo{} - for index, track := range c.tracks { + for index, track := range c.tracks.tracks { keep := true for _, indexToRemove := range indicesToRemove { if indexToRemove == index { @@ -159,5 +163,5 @@ func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { } } - c.tracks = newTracks + c.tracks.tracks = newTracks } From c591a4a6b2ac44337b47b65b9904affbc1aeb6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 19:17:41 +0200 Subject: [PATCH 064/124] Handle no tracks to remove better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 +--- src/conf.go | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/call.go b/src/call.go index d569e74..5d8b184 100644 --- a/src/call.go +++ b/src/call.go @@ -322,9 +322,7 @@ func (c *call) terminate() error { c.conf.calls.callsMu.Unlock() info := localTrackInfo{call: c} - if err := c.conf.removeTracksFromPeerConnectionsByInfo(info); err != nil { - return err - } + c.conf.removeTracksFromPeerConnectionsByInfo(info) c.conf.removeTracksFromConfByInfo(info) return nil diff --git a/src/conf.go b/src/conf.go index 7bdfd8a..d5ab9c2 100644 --- a/src/conf.go +++ b/src/conf.go @@ -118,7 +118,7 @@ func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.T return foundTracks } -func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) error { +func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) int { indices := c.getLocalTrackIndicesByInfo(removeInfo) // FIXME: the big O of this must be awful... @@ -134,14 +134,13 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) } if err := call.peerConnection.RemoveTrack(sender); err != nil { log.Printf("%s | failed to remove track: %s", call.userID, err) - return err } } } } } - return nil + return len(indices) } func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { From 6063f2cbecd45c23d3a8e62f5a1befe007b4c4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 19:19:31 +0200 Subject: [PATCH 065/124] Better logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index 5d8b184..058b55e 100644 --- a/src/call.go +++ b/src/call.go @@ -210,7 +210,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }, }) - log.Printf("%s | published track with trackID %s and kind %s", c.userID, trackLocal.ID(), trackLocal.Kind()) + log.Printf("%s | published track with streamID %s trackID %s and kind %s", c.userID, trackLocal.StreamID(), trackLocal.ID(), trackLocal.Kind()) c.conf.tracks.mutex.Unlock() c.addSubscribedTracksToPeerConnection() From 646efa70f82687c8768b9f57bb723ba71ff4e52e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 4 Aug 2022 21:07:22 +0200 Subject: [PATCH 066/124] Add `unpublish` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/call.go b/src/call.go index 058b55e..347df91 100644 --- a/src/call.go +++ b/src/call.go @@ -118,6 +118,38 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { SDP: offer.SDP, }) + case "unpublish": + log.Printf("%s | unpublished: %+v", c.userID, msg.Stop) + + for _, trackDesc := range msg.Stop { + if removedTracksCount := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{ + streamID: trackDesc.StreamID, + trackID: trackDesc.TrackID, + }); removedTracksCount == 0 { + log.Printf("%s | no tracks to remove for: %+v", c.userID, msg.Stop) + } + + } + + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: msg.SDP, + }) + + offer, err := c.peerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + err = c.peerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + c.sendDataChannelMessage(dataChannelMessage{ + Op: "answer", + SDP: offer.SDP, + }) + case "answer": peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, From ca546c6122ef747a4c1c0393560bb6a432c115da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 10:53:27 +0200 Subject: [PATCH 067/124] Remove old call with device id when we get a new one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/conf.go | 9 +++++++++ src/matrix.go | 1 + 2 files changed, 10 insertions(+) diff --git a/src/conf.go b/src/conf.go index d5ab9c2..0cfcfb1 100644 --- a/src/conf.go +++ b/src/conf.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/pion/webrtc/v3" + "maunium.net/go/mautrix/id" ) type localTrackInfo struct { @@ -164,3 +165,11 @@ func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { c.tracks.tracks = newTracks } + +func (c *conf) removeOldCallByDeviceId(deviceId id.DeviceID) { + for _, call := range c.calls.calls { + if call.deviceID == deviceId { + call.terminate() + } + } +} diff --git a/src/matrix.go b/src/matrix.go index 8372503..21a6f79 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -110,6 +110,7 @@ func initMatrix(config *config) error { log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) return true } + conf.removeOldCallByDeviceId(invite.DeviceID) if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) return true From 477857395b269ac64ba20565b8cfd1fd3779708a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 6 Aug 2022 13:43:44 +0200 Subject: [PATCH 068/124] Fix `addSubscribedTracksToPeerConnection()` being called for incorrect calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/call.go b/src/call.go index 347df91..9ce79b6 100644 --- a/src/call.go +++ b/src/call.go @@ -241,11 +241,15 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece call: c, }, }) + c.conf.tracks.mutex.Unlock() log.Printf("%s | published track with streamID %s trackID %s and kind %s", c.userID, trackLocal.StreamID(), trackLocal.ID(), trackLocal.Kind()) - c.conf.tracks.mutex.Unlock() - c.addSubscribedTracksToPeerConnection() + for _, call := range c.conf.calls.calls { + if call.callID != c.callID { + call.addSubscribedTracksToPeerConnection() + } + } copyRemoteToLocal(trackRemote, trackLocal) } From 928604e99e8e97f481fd0301cceb91452a082815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 7 Aug 2022 11:39:39 +0200 Subject: [PATCH 069/124] Add comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/call.go b/src/call.go index 9ce79b6..bbe0bf4 100644 --- a/src/call.go +++ b/src/call.go @@ -214,6 +214,7 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { } func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { + // FIXME: This is a potential performance killer if strings.Contains(trackRemote.Codec().MimeType, "video") { // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval go func() { From b8479f0dbded3398a1d9893785805e115bdceddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 7 Aug 2022 16:56:07 +0200 Subject: [PATCH 070/124] Remove empty line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.go b/src/utils.go index 242786f..698bc9b 100644 --- a/src/utils.go +++ b/src/utils.go @@ -36,5 +36,4 @@ func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.Track break } } - } From 24a3e0b9c310bd328e537e86772b287205bfdc15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 7 Aug 2022 16:58:22 +0200 Subject: [PATCH 071/124] Support profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- go.mod | 3 +++ go.sum | 9 +++++++++ scripts/profile.sh | 3 +++ src/main.go | 20 +++++++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100755 scripts/profile.sh diff --git a/go.mod b/go.mod index b61baaa..2f54c4d 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,10 @@ require ( replace maunium.net/go/mautrix v0.11.0 => ../mautrix-go require ( + github.com/chzyer/readline v1.5.0 // indirect + github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 // indirect github.com/pion/datachannel v1.5.2 // indirect github.com/pion/dtls/v2 v2.1.3 // indirect github.com/pion/ice/v2 v2.2.3 // indirect diff --git a/go.sum b/go.sum index 94dfcef..254fb11 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -17,9 +21,13 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ= +github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -121,6 +129,7 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/scripts/profile.sh b/scripts/profile.sh new file mode 100755 index 0000000..478e5ff --- /dev/null +++ b/scripts/profile.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +clear && go run ./src/*.go --profile \ No newline at end of file diff --git a/src/main.go b/src/main.go index 64ae80d..2bc258a 100644 --- a/src/main.go +++ b/src/main.go @@ -21,12 +21,23 @@ import ( "fmt" "io/ioutil" "log" + "net/http" yaml "gopkg.in/yaml.v3" "maunium.net/go/mautrix/id" + + _ "net/http/pprof" ) +func initProfiling() { + log.Printf("Initializing profiling") + + go func() { + http.ListenAndServe(":1234", nil) + }() +} + func loadConfig(configFilePath string) (*config, error) { log.Printf("Loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) @@ -41,7 +52,14 @@ func loadConfig(configFilePath string) (*config, error) { } func main() { - log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds) + profilingEnabled := flag.Bool("profile", false, "profiling mode") + flag.Parse() + + if *profilingEnabled { + initProfiling() + } + + log.SetFlags(log.Ldate | log.Ltime) configFilePath := flag.String("config", "config.yaml", "Configuration file path") flag.Parse() From e5bec68b3fe610c59449cf973ad076c307bb5812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 8 Aug 2022 11:06:24 +0200 Subject: [PATCH 072/124] Better cpu and memory profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .gitignore | 1 + scripts/profile.sh | 2 +- src/main.go | 91 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index c1ffadc..3a13c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ config.yaml dist/ +*.pprof \ No newline at end of file diff --git a/scripts/profile.sh b/scripts/profile.sh index 478e5ff..c38a9f8 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --profile \ No newline at end of file +clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof \ No newline at end of file diff --git a/src/main.go b/src/main.go index 2bc258a..ed136b9 100644 --- a/src/main.go +++ b/src/main.go @@ -21,7 +21,11 @@ import ( "fmt" "io/ioutil" "log" - "net/http" + "os" + "os/signal" + "runtime" + "runtime/pprof" + "syscall" yaml "gopkg.in/yaml.v3" @@ -30,19 +34,56 @@ import ( _ "net/http/pprof" ) -func initProfiling() { - log.Printf("Initializing profiling") +var configFilePath = flag.String("config", "config.yaml", "Configuration file path") +var cpuProfile = flag.String("cpuProfile", "", "write CPU profile to `file`") +var memProfile = flag.String("memProfile", "", "write memory profile to `file`") - go func() { - http.ListenAndServe(":1234", nil) - }() +func initCpuProfiling(cpuProfile *string) func() { + log.Print("initializing CPU profiling") + + f, err := os.Create(*cpuProfile) + if err != nil { + log.Fatalf("could not create CPU profile: %s", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatalf("could not start CPU profile: %s", err) + } + + return func() { + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + log.Fatalf("could not close CPU profile: %s", err) + } + } +} + +func initMemoryProfiling(memProfile *string) func() { + log.Print("initializing memory profiling") + + return func() { + f, err := os.Create(*memProfile) + if err != nil { + log.Fatalf("could not create memory profile: %s", err) + } + runtime.GC() + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatalf("could not write memory profile: %s", err) + } + if err = f.Close(); err != nil { + log.Fatalf("could not close memory profile: %s", err) + } + } +} + +func initLogging() { + log.SetFlags(log.Ldate | log.Ltime) } func loadConfig(configFilePath string) (*config, error) { - log.Printf("Loading %s", configFilePath) + log.Printf("loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { - log.Fatal("Failed to read config", err) + log.Fatalf("failed to read config: %s", err) } var config config if err := yaml.Unmarshal(file, &config); err != nil { @@ -51,26 +92,44 @@ func loadConfig(configFilePath string) (*config, error) { return &config, nil } +func onKill(c chan os.Signal, beforeExit []func()) { + select { + case <-c: + log.Printf("ending program") + + for _, function := range beforeExit { + function() + } + defer os.Exit(0) + } +} + func main() { - profilingEnabled := flag.Bool("profile", false, "profiling mode") + initLogging() + flag.Parse() - if *profilingEnabled { - initProfiling() + beforeExit := []func(){} + if *cpuProfile != "" { + beforeExit = append(beforeExit, initCpuProfiling(cpuProfile)) + } + if *memProfile != "" { + beforeExit = append(beforeExit, initMemoryProfiling(memProfile)) } - log.SetFlags(log.Ldate | log.Ltime) - configFilePath := flag.String("config", "config.yaml", "Configuration file path") - flag.Parse() + // try to handle os interrupt(signal terminated) + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go onKill(c, beforeExit) var config *config var err error if config, err = loadConfig(*configFilePath); err != nil { - log.Fatal("Failed to load config file", err) + log.Fatalf("failed to load config file: %s", err) } if err := initMatrix(config); err != nil { - log.Fatal("Failed to init Matrix", err) + log.Fatalf("failed to init Matrix: %s", err) } } From d80dadad740908436069d3bf1fdaca94b68f631c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 8 Aug 2022 13:15:19 +0200 Subject: [PATCH 073/124] Implement sessions ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 ++-- src/conf.go | 12 +++++++++--- src/matrix.go | 18 +++++++++++++----- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/call.go b/src/call.go index bbe0bf4..dc94876 100644 --- a/src/call.go +++ b/src/call.go @@ -40,8 +40,8 @@ type call struct { callID string userID id.UserID deviceID id.DeviceID - localSessionID string - remoteSessionID string + localSessionID id.SessionID + remoteSessionID id.SessionID client *mautrix.Client peerConnection *webrtc.PeerConnection conf *conf diff --git a/src/conf.go b/src/conf.go index 0cfcfb1..7950148 100644 --- a/src/conf.go +++ b/src/conf.go @@ -166,10 +166,16 @@ func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { c.tracks.tracks = newTracks } -func (c *conf) removeOldCallByDeviceId(deviceId id.DeviceID) { +func (c *conf) removeOldCallsByDeviceAndSessionIds(deviceID id.DeviceID, sessionID id.SessionID) error { + var err error for _, call := range c.calls.calls { - if call.deviceID == deviceId { - call.terminate() + if call.deviceID == deviceID { + if call.remoteSessionID == sessionID { + err = errors.New("found existing call with equal DeviceID and SessionID") + } else { + call.terminate() + } } } + return err } diff --git a/src/matrix.go b/src/matrix.go index 21a6f79..a707038 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -26,6 +26,8 @@ import ( "maunium.net/go/mautrix/event" ) +const localSessionID = "sfu" + func initMatrix(config *config) error { client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) if err != nil { @@ -102,7 +104,11 @@ func initMatrix(config *config) error { continue } - // TODO: check session IDs + if evt.Content.Raw["dest_session_id"] != localSessionID { + log.Printf("%s | SessionID %s does not match our SessionID %s - ignoring", evt.Content.Raw["dest_session_id"], localSessionID, err) + continue + } + switch evt.Type.Type { case CallInvite.Type: invite := evt.Content.AsCallInvite() @@ -110,16 +116,18 @@ func initMatrix(config *config) error { log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) return true } - conf.removeOldCallByDeviceId(invite.DeviceID) + if err := conf.removeOldCallsByDeviceAndSessionIds(invite.DeviceID, invite.SenderSessionID); err != nil { + log.Printf("%s | error removing old calls - ignoring call: %+v", evt.Sender.String(), err) + return true + } if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) return true } call.userID = evt.Sender call.deviceID = invite.DeviceID - // XXX: hardcode the same sessionID for SFUs for now, as nobody should care - // much if they get restarted(?) - call.localSessionID = "sfu" + // XXX: What if an SFU gets restarted? + call.localSessionID = localSessionID call.remoteSessionID = invite.SenderSessionID call.client = client call.onInvite(invite) From f0282d8bf9541a02aa0b4de48363831e76e75925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 11:21:23 +0200 Subject: [PATCH 074/124] Theoretical perf improvement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/call.go b/src/call.go index dc94876..099c4bf 100644 --- a/src/call.go +++ b/src/call.go @@ -400,6 +400,10 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { } func (c *call) addSubscribedTracksToPeerConnection() { + if len(c.subscribedTracks.tracks) == 0 { + return + } + newSubscribedTracks := []localTrackInfo{} tracksToAddToPeerConnection := []webrtc.TrackLocal{} From ebf4e102b67ad75b70326309e2e4af7ffc50e3ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 11:24:43 +0200 Subject: [PATCH 075/124] Use gorutines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/call.go b/src/call.go index 099c4bf..2b8b0dd 100644 --- a/src/call.go +++ b/src/call.go @@ -96,7 +96,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } c.subscribedTracks.mutex.Unlock() - c.addSubscribedTracksToPeerConnection() + go c.addSubscribedTracksToPeerConnection() case "publish": peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -248,11 +248,11 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece for _, call := range c.conf.calls.calls { if call.callID != c.callID { - call.addSubscribedTracksToPeerConnection() + go call.addSubscribedTracksToPeerConnection() } } - copyRemoteToLocal(trackRemote, trackLocal) + go copyRemoteToLocal(trackRemote, trackLocal) } func (c *call) onInvite(content *event.CallInviteEventContent) error { From becdee74e95680664bd9ff6330c87cfbd9660683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Aug 2022 11:29:05 +0200 Subject: [PATCH 076/124] Lock later MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index 2b8b0dd..e3062c7 100644 --- a/src/call.go +++ b/src/call.go @@ -228,12 +228,12 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }() } - c.conf.tracks.mutex.Lock() trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) if err != nil { panic(err) } + c.conf.tracks.mutex.Lock() c.conf.tracks.tracks = append(c.conf.tracks.tracks, localTrackWithInfo{ track: trackLocal, info: localTrackInfo{ From 7eb8480e2c6fc84bed3066cb05b5804ea279f4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 13:47:10 +0200 Subject: [PATCH 077/124] Move `getConf()` to the correct place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/conf.go | 19 ------------------- src/focus.go | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/conf.go b/src/conf.go index 7950148..33752a5 100644 --- a/src/conf.go +++ b/src/conf.go @@ -52,25 +52,6 @@ type conf struct { tracks tracks } -func (f *focus) getConf(confID string, create bool) (*conf, error) { - f.confs.confsMu.Lock() - defer f.confs.confsMu.Unlock() - co := f.confs.confs[confID] - if co == nil { - if create { - co = &conf{ - confID: confID, - } - f.confs.confs[confID] = co - co.calls.calls = make(map[string]*call) - co.tracks.tracks = []localTrackWithInfo{} - } else { - return nil, errors.New("no such conf") - } - } - return co, nil -} - func (c *conf) getCall(callID string, create bool) (*call, error) { c.calls.callsMu.Lock() defer c.calls.callsMu.Unlock() diff --git a/src/focus.go b/src/focus.go index 5688806..f0f51fe 100644 --- a/src/focus.go +++ b/src/focus.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "errors" "sync" ) @@ -34,3 +35,22 @@ func (f *focus) Init(name string) { f.name = name f.confs.confs = make(map[string]*conf) } + +func (f *focus) getConf(confID string, create bool) (*conf, error) { + f.confs.confsMu.Lock() + defer f.confs.confsMu.Unlock() + co := f.confs.confs[confID] + if co == nil { + if create { + co = &conf{ + confID: confID, + } + f.confs.confs[confID] = co + co.calls.calls = make(map[string]*call) + co.tracks.tracks = []localTrackWithInfo{} + } else { + return nil, errors.New("no such conf") + } + } + return co, nil +} From c7cda4d86f9b846c69eabbb4ad8af08a0b475d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 16:40:08 +0200 Subject: [PATCH 078/124] Implement timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- config.yaml.sample | 1 + src/call.go | 54 ++++++++++++++++++++++++++++++++++++++++------ src/main.go | 17 +++++++++------ src/matrix.go | 10 ++++----- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/config.yaml.sample b/config.yaml.sample index fa19e40..0b1b4df 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -1,3 +1,4 @@ homeserverurl: "http://localhost:8008" userid: "@sfu:shadowfax" accesstoken: "..." +timeout: 30 diff --git a/src/call.go b/src/call.go index e3062c7..0a081d8 100644 --- a/src/call.go +++ b/src/call.go @@ -47,6 +47,7 @@ type call struct { conf *conf dataChannel *webrtc.DataChannel subscribedTracks subscribedTracks + lastKeepAliveTs time.Time } func (c *call) dataChannelHandler(d *webrtc.DataChannel) { @@ -75,7 +76,9 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) } - log.Printf("%s | received DC: %s", c.userID, msg.Op) + if msg.Op != "alive" { + log.Printf("%s | received DC: %s", c.userID, msg.Op) + } // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client @@ -156,6 +159,9 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { SDP: msg.SDP, }) + case "alive": + c.lastKeepAliveTs = time.UnixMilli(int64(msg.Timestamp)) + default: log.Fatalf("Unknown operation %s", msg.Op) // TODO: hook up msg.Stop to unsubscribe from tracks @@ -255,6 +261,12 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece go copyRemoteToLocal(trackRemote, trackLocal) } +func (c *call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { + if state == webrtc.ICEConnectionStateCompleted || state == webrtc.ICEConnectionStateConnected { + go c.checkKeepAliveTimestamp() + } +} + func (c *call) onInvite(content *event.CallInviteEventContent) error { offer := content.Offer @@ -276,6 +288,9 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { peerConnection.OnNegotiationNeeded(func() { c.negotiationNeededHandler() }) + peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) { + c.iceConnectionStateHandler(state) + }) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, @@ -347,7 +362,7 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { return nil } -func (c *call) terminate() error { +func (c *call) terminate() { log.Printf("%s | terminating call", c.userID) if err := c.peerConnection.Close(); err != nil { @@ -362,10 +377,28 @@ func (c *call) terminate() error { c.conf.removeTracksFromPeerConnectionsByInfo(info) c.conf.removeTracksFromConfByInfo(info) - return nil } -func (c *call) sendToDevice(callType event.Type, content *event.Content) error { +func (c *call) hangup(reason event.CallHangupReason) { + hangupEvtContent := &event.Content{ + Parsed: event.CallHangupEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.callID, + ConfID: c.conf.confID, + DeviceID: c.client.DeviceID, + SenderSessionID: c.localSessionID, + DestSessionID: c.remoteSessionID, + PartyID: string(c.client.DeviceID), + Version: event.CallVersion("1"), + }, + Reason: reason, + }, + } + c.sendToDevice(event.CallHangup, hangupEvtContent) + c.terminate() +} + +func (c *call) sendToDevice(callType event.Type, content *event.Content) { log.Printf("%s | sending to device %s", c.userID, callType.Type) toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ @@ -378,8 +411,6 @@ func (c *call) sendToDevice(callType event.Type, content *event.Content) error { // TODO: E2EE // TODO: to-device reliability c.client.SendToDevice(callType, toDevice) - - return nil } func (c *call) sendDataChannelMessage(msg dataChannelMessage) { @@ -427,3 +458,14 @@ func (c *call) addSubscribedTracksToPeerConnection() { } } } + +func (c *call) checkKeepAliveTimestamp() { + timeout := time.Second * time.Duration(configInstance.Timeout) + for range time.Tick(timeout / 2 * 3) { + if c.lastKeepAliveTs.Add(timeout).Before(time.Now()) { + log.Printf("%s | call timed out", c.userID) + c.hangup(event.CallHangupKeepAliveTimeout) + break + } + } +} diff --git a/src/main.go b/src/main.go index ed136b9..bb422ae 100644 --- a/src/main.go +++ b/src/main.go @@ -34,6 +34,8 @@ import ( _ "net/http/pprof" ) +var configInstance *config + var configFilePath = flag.String("config", "config.yaml", "Configuration file path") var cpuProfile = flag.String("cpuProfile", "", "write CPU profile to `file`") var memProfile = flag.String("memProfile", "", "write memory profile to `file`") @@ -122,13 +124,12 @@ func main() { signal.Notify(c, os.Interrupt, syscall.SIGTERM) go onKill(c, beforeExit) - var config *config var err error - if config, err = loadConfig(*configFilePath); err != nil { + if configInstance, err = loadConfig(*configFilePath); err != nil { log.Fatalf("failed to load config file: %s", err) } - if err := initMatrix(config); err != nil { + if err := initMatrix(); err != nil { log.Fatalf("failed to init Matrix: %s", err) } } @@ -137,6 +138,7 @@ type config struct { UserID id.UserID HomeserverURL string AccessToken string + Timeout int64 } type trackDesc struct { @@ -149,8 +151,9 @@ type dataChannelMessage struct { ID string `json:"id"` Message string `json:"message,omitempty"` // XXX: is this even needed? we know which conf a given call is for... - ConfID string `json:"conf_id,omitempty"` - Start []trackDesc `json:"start,omitempty"` - Stop []trackDesc `json:"stop,omitempty"` - SDP string `json:"sdp,omitempty"` + ConfID string `json:"conf_id,omitempty"` + Start []trackDesc `json:"start,omitempty"` + Stop []trackDesc `json:"stop,omitempty"` + SDP string `json:"sdp,omitempty"` + Timestamp int `json:"ts,omitempty"` } diff --git a/src/matrix.go b/src/matrix.go index a707038..57e23a1 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -28,8 +28,8 @@ import ( const localSessionID = "sfu" -func initMatrix(config *config) error { - client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) +func initMatrix() error { + client, err := mautrix.NewClient(configInstance.HomeserverURL, configInstance.UserID, configInstance.AccessToken) if err != nil { log.Fatal("Failed to create client", err) } @@ -38,14 +38,14 @@ func initMatrix(config *config) error { if err != nil { log.Fatal("Failed to identify SFU user", err) } - if config.UserID != whoami.UserID { - log.Fatalf("Access token is for the wrong user: %s", config.UserID) + if configInstance.UserID != whoami.UserID { + log.Fatalf("Access token is for the wrong user: %s", configInstance.UserID) } log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID focus := new(focus) - focus.Init(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) + focus.Init(fmt.Sprintf("%s (%s)", configInstance.UserID, client.DeviceID)) syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true From 6149d7f33b19676ee7e32fd52857a189be2f24a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 16:59:37 +0200 Subject: [PATCH 079/124] Remove some debug logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/call.go b/src/call.go index 0a081d8..7a7e7b7 100644 --- a/src/call.go +++ b/src/call.go @@ -170,8 +170,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } func (c *call) negotiationNeededHandler() { - log.Printf("%s | negotiation needed", c.userID) - offer, err := c.peerConnection.CreateOffer(nil) if err != nil { panic(err) @@ -399,7 +397,9 @@ func (c *call) hangup(reason event.CallHangupReason) { } func (c *call) sendToDevice(callType event.Type, content *event.Content) { - log.Printf("%s | sending to device %s", c.userID, callType.Type) + if callType.Type != event.CallCandidates.Type { + log.Printf("%s | sending to device %s", c.userID, callType.Type) + } toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ c.userID: { From d3b4ba31f5a120109561982812e60ed96a6bc8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:14:18 +0200 Subject: [PATCH 080/124] Add vscode to gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3a13c8d..397fbfa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ config.yaml dist/ -*.pprof \ No newline at end of file +*.pprof +.vscode/ \ No newline at end of file From 80e9192f5e112d20993af6d8273958af43e9804c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:21:02 +0200 Subject: [PATCH 081/124] Fix timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 31 ++++++++++++++++--------------- src/main.go | 11 +++++------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/call.go b/src/call.go index 7a7e7b7..620003b 100644 --- a/src/call.go +++ b/src/call.go @@ -37,17 +37,17 @@ type subscribedTracks struct { } type call struct { - callID string - userID id.UserID - deviceID id.DeviceID - localSessionID id.SessionID - remoteSessionID id.SessionID - client *mautrix.Client - peerConnection *webrtc.PeerConnection - conf *conf - dataChannel *webrtc.DataChannel - subscribedTracks subscribedTracks - lastKeepAliveTs time.Time + callID string + userID id.UserID + deviceID id.DeviceID + localSessionID id.SessionID + remoteSessionID id.SessionID + client *mautrix.Client + peerConnection *webrtc.PeerConnection + conf *conf + dataChannel *webrtc.DataChannel + subscribedTracks subscribedTracks + lastKeepAliveTimestamp time.Time } func (c *call) dataChannelHandler(d *webrtc.DataChannel) { @@ -160,7 +160,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) case "alive": - c.lastKeepAliveTs = time.UnixMilli(int64(msg.Timestamp)) + c.lastKeepAliveTimestamp = time.Now() default: log.Fatalf("Unknown operation %s", msg.Op) @@ -261,6 +261,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece func (c *call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { if state == webrtc.ICEConnectionStateCompleted || state == webrtc.ICEConnectionStateConnected { + c.lastKeepAliveTimestamp = time.Now() go c.checkKeepAliveTimestamp() } } @@ -461,9 +462,9 @@ func (c *call) addSubscribedTracksToPeerConnection() { func (c *call) checkKeepAliveTimestamp() { timeout := time.Second * time.Duration(configInstance.Timeout) - for range time.Tick(timeout / 2 * 3) { - if c.lastKeepAliveTs.Add(timeout).Before(time.Now()) { - log.Printf("%s | call timed out", c.userID) + for range time.Tick(timeout) { + if c.lastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { + log.Printf("%s | did not get keep-alive message in the last %s:", c.userID, timeout) c.hangup(event.CallHangupKeepAliveTimeout) break } diff --git a/src/main.go b/src/main.go index bb422ae..b98792d 100644 --- a/src/main.go +++ b/src/main.go @@ -138,7 +138,7 @@ type config struct { UserID id.UserID HomeserverURL string AccessToken string - Timeout int64 + Timeout int } type trackDesc struct { @@ -151,9 +151,8 @@ type dataChannelMessage struct { ID string `json:"id"` Message string `json:"message,omitempty"` // XXX: is this even needed? we know which conf a given call is for... - ConfID string `json:"conf_id,omitempty"` - Start []trackDesc `json:"start,omitempty"` - Stop []trackDesc `json:"stop,omitempty"` - SDP string `json:"sdp,omitempty"` - Timestamp int `json:"ts,omitempty"` + ConfID string `json:"conf_id,omitempty"` + Start []trackDesc `json:"start,omitempty"` + Stop []trackDesc `json:"stop,omitempty"` + SDP string `json:"sdp,omitempty"` } From 3f563749378d70d57a4a151ae6079b6e185fc1ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:28:51 +0200 Subject: [PATCH 082/124] Make logging time optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- scripts/profile.sh | 2 +- scripts/run.sh | 2 +- src/main.go | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/profile.sh b/scripts/profile.sh index c38a9f8..1f53c9d 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof \ No newline at end of file +clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof --logTime \ No newline at end of file diff --git a/scripts/run.sh b/scripts/run.sh index 4cc62aa..6a2d664 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go \ No newline at end of file +clear && go run ./src/*.go --logTime \ No newline at end of file diff --git a/src/main.go b/src/main.go index b98792d..e3d2dd4 100644 --- a/src/main.go +++ b/src/main.go @@ -36,7 +36,8 @@ import ( var configInstance *config -var configFilePath = flag.String("config", "config.yaml", "Configuration file path") +var logTime = flag.Bool("logTime", false, "whether or not to print time and date in logs") +var configFilePath = flag.String("config", "config.yaml", "configuration file path") var cpuProfile = flag.String("cpuProfile", "", "write CPU profile to `file`") var memProfile = flag.String("memProfile", "", "write memory profile to `file`") @@ -77,8 +78,10 @@ func initMemoryProfiling(memProfile *string) func() { } } -func initLogging() { - log.SetFlags(log.Ldate | log.Ltime) +func initLogging(logTime *bool) { + if *logTime { + log.SetFlags(log.Ldate | log.Ltime) + } } func loadConfig(configFilePath string) (*config, error) { @@ -107,10 +110,10 @@ func onKill(c chan os.Signal, beforeExit []func()) { } func main() { - initLogging() - flag.Parse() + initLogging(logTime) + beforeExit := []func(){} if *cpuProfile != "" { beforeExit = append(beforeExit, initCpuProfiling(cpuProfile)) From 4845a7ea4abb4ade19f0dd3ee54d0b2801b04e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:29:35 +0200 Subject: [PATCH 083/124] Fix warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/main.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main.go b/src/main.go index e3d2dd4..8365ebb 100644 --- a/src/main.go +++ b/src/main.go @@ -98,15 +98,12 @@ func loadConfig(configFilePath string) (*config, error) { } func onKill(c chan os.Signal, beforeExit []func()) { - select { - case <-c: - log.Printf("ending program") - - for _, function := range beforeExit { - function() - } - defer os.Exit(0) + <-c + log.Printf("ending program") + for _, function := range beforeExit { + function() } + defer os.Exit(0) } func main() { From bdd0bbea2b9c742d8a4bb680e1d0e9f87d4b5106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:30:05 +0200 Subject: [PATCH 084/124] Remove unused packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- go.mod | 3 --- 1 file changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 2f54c4d..b61baaa 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,7 @@ require ( replace maunium.net/go/mautrix v0.11.0 => ../mautrix-go require ( - github.com/chzyer/readline v1.5.0 // indirect - github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 // indirect github.com/pion/datachannel v1.5.2 // indirect github.com/pion/dtls/v2 v2.1.3 // indirect github.com/pion/ice/v2 v2.2.3 // indirect From 9c1e4796e6983c255f05a76bb385bd24170e18f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 11 Aug 2022 21:35:18 +0200 Subject: [PATCH 085/124] Fix disabling loggin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.go b/src/main.go index 8365ebb..185b801 100644 --- a/src/main.go +++ b/src/main.go @@ -79,6 +79,7 @@ func initMemoryProfiling(memProfile *string) func() { } func initLogging(logTime *bool) { + log.SetFlags(0) if *logTime { log.SetFlags(log.Ldate | log.Ltime) } From 4dfd79eb59dec4e378b478bd10b2984463a5984e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 16:13:08 +0200 Subject: [PATCH 086/124] Don't log some events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/matrix.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix.go b/src/matrix.go index 57e23a1..9b60991 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -97,7 +97,9 @@ func initMatrix() error { var conf *conf var call *call - if strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { + if (strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.")) && + evt.Type.Type != CallCandidates.Type && + evt.Type.Type != CallSelectAnswer.Type { log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) } else { log.Printf("received non-call to-device event %s", evt.Type.Type) From b80edd60254f4494b70ed81e29e578d488b07710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 17:46:41 +0200 Subject: [PATCH 087/124] Improve logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 28 +++++++++------------------- src/conf.go | 2 +- src/matrix.go | 8 +++----- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/call.go b/src/call.go index 620003b..e0358f1 100644 --- a/src/call.go +++ b/src/call.go @@ -54,14 +54,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { c.dataChannel = d peerConnection := c.peerConnection - d.OnOpen(func() { - log.Printf("%s | DC opened", c.userID) - }) - - d.OnClose(func() { - log.Printf("%s | DC closed", c.userID) - }) - d.OnError(func(err error) { log.Fatalf("%s | DC error: %s", c.callID, err) }) @@ -76,10 +68,6 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) } - if msg.Op != "alive" { - log.Printf("%s | received DC: %s", c.userID, msg.Op) - } - // TODO: hook cascade back up. // As we're not an AS, we'd rely on the client // to send us a "connect" op to tell us how to @@ -88,10 +76,9 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { switch msg.Op { case "select": - log.Printf("%s | selected: %+v", c.userID, msg.Start) - c.subscribedTracks.mutex.Lock() for _, trackDesc := range msg.Start { + log.Printf("%s | selecting StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) c.subscribedTracks.tracks = append(c.subscribedTracks.tracks, localTrackInfo{ streamID: trackDesc.StreamID, trackID: trackDesc.TrackID, @@ -102,6 +89,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { go c.addSubscribedTracksToPeerConnection() case "publish": + log.Printf("%s | received DC publish", c.userID) + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: msg.SDP, @@ -122,9 +111,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) case "unpublish": - log.Printf("%s | unpublished: %+v", c.userID, msg.Stop) - for _, trackDesc := range msg.Stop { + log.Printf("%s | unpublishing StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) if removedTracksCount := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{ streamID: trackDesc.StreamID, trackID: trackDesc.TrackID, @@ -154,6 +142,8 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) case "answer": + log.Printf("%s | received DC answer", c.userID) + peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, SDP: msg.SDP, @@ -248,7 +238,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece }) c.conf.tracks.mutex.Unlock() - log.Printf("%s | published track with streamID %s trackID %s and kind %s", c.userID, trackLocal.StreamID(), trackLocal.ID(), trackLocal.Kind()) + log.Printf("%s | published %s StreamID %s TrackID %s", c.userID, trackLocal.Kind(), trackLocal.StreamID(), trackLocal.ID()) for _, call := range c.conf.calls.calls { if call.callID != c.callID { @@ -443,7 +433,7 @@ func (c *call) addSubscribedTracksToPeerConnection() { for _, trackInfo := range c.subscribedTracks.tracks { foundTracks := c.conf.getLocalTrackByInfo(trackInfo) if len(foundTracks) == 0 { - log.Printf("%s | no track found for %+v", c.userID, trackInfo) + log.Printf("%s | no track found StreamID %s TrackID %s", c.userID, trackInfo.streamID, trackInfo.trackID) newSubscribedTracks = append(newSubscribedTracks, trackInfo) } else { tracksToAddToPeerConnection = append(tracksToAddToPeerConnection, foundTracks...) @@ -453,7 +443,7 @@ func (c *call) addSubscribedTracksToPeerConnection() { c.subscribedTracks.mutex.Unlock() for _, track := range tracksToAddToPeerConnection { - log.Printf("%s | adding %s track with %s", c.userID, track.Kind(), track.ID()) + log.Printf("%s | adding %s StreamID %s TrackID %s", c.userID, track.Kind(), track.StreamID(), track.ID()) if _, err := c.peerConnection.AddTrack(track); err != nil { panic(err) } diff --git a/src/conf.go b/src/conf.go index 33752a5..ffaba9a 100644 --- a/src/conf.go +++ b/src/conf.go @@ -110,7 +110,7 @@ func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) for _, call := range c.calls.calls { for _, sender := range call.peerConnection.GetSenders() { if info.trackID == sender.Track().ID() { - log.Printf("%s | removing %s track with StreamID %s", call.userID, sender.Track().Kind(), info.streamID) + log.Printf("%s | removing %s StreamID %s TrackID %s", call.userID, sender.Track().Kind(), sender.Track().StreamID(), sender.Track().ID()) if err := sender.Stop(); err != nil { log.Printf("%s | failed to stop sender: %s", call.userID, err) } diff --git a/src/matrix.go b/src/matrix.go index 9b60991..d93727b 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -97,13 +97,11 @@ func initMatrix() error { var conf *conf var call *call - if (strings.HasPrefix(evt.Type.Type, "m.call.") || strings.HasPrefix(evt.Type.Type, "org.matrix.call.")) && - evt.Type.Type != CallCandidates.Type && - evt.Type.Type != CallSelectAnswer.Type { - log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) - } else { + if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { log.Printf("received non-call to-device event %s", evt.Type.Type) continue + } else if evt.Type.Type != CallCandidates.Type && evt.Type.Type != CallSelectAnswer.Type { + log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) } if evt.Content.Raw["dest_session_id"] != localSessionID { From 02e553c2ad10d9c7717814085b8ce8f48b563c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 14:32:16 +0200 Subject: [PATCH 088/124] Switch to data-channel for sending metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 90 ++++++++++++++++++++++++---------------------------- src/conf.go | 72 ++++++++++++++++++++++++++++++++++++++--- src/focus.go | 3 ++ src/main.go | 10 +++--- 4 files changed, 119 insertions(+), 56 deletions(-) diff --git a/src/call.go b/src/call.go index e0358f1..2c00d87 100644 --- a/src/call.go +++ b/src/call.go @@ -20,7 +20,6 @@ import ( "encoding/json" "log" "strings" - "sync" "time" "maunium.net/go/mautrix" @@ -31,11 +30,6 @@ import ( "github.com/pion/webrtc/v3" ) -type subscribedTracks struct { - mutex sync.RWMutex - tracks []localTrackInfo -} - type call struct { callID string userID id.UserID @@ -46,7 +40,6 @@ type call struct { peerConnection *webrtc.PeerConnection conf *conf dataChannel *webrtc.DataChannel - subscribedTracks subscribedTracks lastKeepAliveTimestamp time.Time } @@ -54,6 +47,10 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { c.dataChannel = d peerConnection := c.peerConnection + d.OnOpen(func() { + c.sendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + }) + d.OnError(func(err error) { log.Fatalf("%s | DC error: %s", c.callID, err) }) @@ -74,19 +71,33 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { // connect to another focus in order to select // its streams. + if msg.Metadata != nil { + c.conf.updateSDPStreamMetadata(c.deviceID, msg.Metadata) + } + switch msg.Op { case "select": - c.subscribedTracks.mutex.Lock() + if len(msg.Start) == 0 { + return + } + for _, trackDesc := range msg.Start { log.Printf("%s | selecting StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) - c.subscribedTracks.tracks = append(c.subscribedTracks.tracks, localTrackInfo{ + foundTracks := c.conf.getLocalTrackByInfo(localTrackInfo{ streamID: trackDesc.StreamID, trackID: trackDesc.TrackID, }) + if len(foundTracks) == 0 { + log.Printf("%s | no track found StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) + continue + } + for _, track := range foundTracks { + log.Printf("%s | adding %s StreamID %s TrackID %s", c.userID, track.Kind(), track.StreamID(), track.ID()) + if _, err := c.peerConnection.AddTrack(track); err != nil { + panic(err) + } + } } - c.subscribedTracks.mutex.Unlock() - - go c.addSubscribedTracksToPeerConnection() case "publish": log.Printf("%s | received DC publish", c.userID) @@ -152,6 +163,11 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { case "alive": c.lastKeepAliveTimestamp = time.Now() + case "metadata": + log.Printf("%s | received DC metadata", c.userID) + + c.conf.sendUpdatedMetadataFromCall(c.callID) + default: log.Fatalf("Unknown operation %s", msg.Op) // TODO: hook up msg.Stop to unsubscribe from tracks @@ -240,12 +256,7 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece log.Printf("%s | published %s StreamID %s TrackID %s", c.userID, trackLocal.Kind(), trackLocal.StreamID(), trackLocal.ID()) - for _, call := range c.conf.calls.calls { - if call.callID != c.callID { - go call.addSubscribedTracksToPeerConnection() - } - } - + go c.conf.sendUpdatedMetadataFromCall(c.callID) go copyRemoteToLocal(trackRemote, trackLocal) } @@ -257,6 +268,7 @@ func (c *call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { } func (c *call) onInvite(content *event.CallInviteEventContent) error { + c.conf.updateSDPStreamMetadata(c.deviceID, content.SDPStreamMetadata) offer := content.Offer peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) @@ -315,6 +327,7 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { Type: "answer", SDP: answerSdp, }, + SDPStreamMetadata: c.conf.getRemoteMetadataForDevice(c.deviceID), }, } c.sendToDevice(event.CallAnswer, answerEvtContent) @@ -365,7 +378,8 @@ func (c *call) terminate() { info := localTrackInfo{call: c} c.conf.removeTracksFromPeerConnectionsByInfo(info) c.conf.removeTracksFromConfByInfo(info) - + c.conf.removeMetadataByDeviceId(c.deviceID) + c.conf.sendUpdatedMetadataFromCall(c.callID) } func (c *call) hangup(reason event.CallHangupReason) { @@ -405,9 +419,18 @@ func (c *call) sendToDevice(callType event.Type, content *event.Content) { } func (c *call) sendDataChannelMessage(msg dataChannelMessage) { + if c.dataChannel == nil { + return + } + msg.ConfID = c.conf.confID + msg.Metadata = c.conf.getRemoteMetadataForDevice(c.deviceID) // TODO: Set ID + if msg.Op == "metadata" && len(msg.Metadata) == 0 { + return + } + marshaled, err := json.Marshal(msg) if err != nil { panic(err) @@ -421,35 +444,6 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { log.Printf("%s | sent DC %s", c.userID, msg.Op) } -func (c *call) addSubscribedTracksToPeerConnection() { - if len(c.subscribedTracks.tracks) == 0 { - return - } - - newSubscribedTracks := []localTrackInfo{} - tracksToAddToPeerConnection := []webrtc.TrackLocal{} - - c.subscribedTracks.mutex.Lock() - for _, trackInfo := range c.subscribedTracks.tracks { - foundTracks := c.conf.getLocalTrackByInfo(trackInfo) - if len(foundTracks) == 0 { - log.Printf("%s | no track found StreamID %s TrackID %s", c.userID, trackInfo.streamID, trackInfo.trackID) - newSubscribedTracks = append(newSubscribedTracks, trackInfo) - } else { - tracksToAddToPeerConnection = append(tracksToAddToPeerConnection, foundTracks...) - } - } - c.subscribedTracks.tracks = newSubscribedTracks - c.subscribedTracks.mutex.Unlock() - - for _, track := range tracksToAddToPeerConnection { - log.Printf("%s | adding %s StreamID %s TrackID %s", c.userID, track.Kind(), track.StreamID(), track.ID()) - if _, err := c.peerConnection.AddTrack(track); err != nil { - panic(err) - } - } -} - func (c *call) checkKeepAliveTimestamp() { timeout := time.Second * time.Duration(configInstance.Timeout) for range time.Tick(timeout) { diff --git a/src/conf.go b/src/conf.go index ffaba9a..9ea2776 100644 --- a/src/conf.go +++ b/src/conf.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/pion/webrtc/v3" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) @@ -46,10 +47,16 @@ type tracks struct { tracks []localTrackWithInfo } +type metadata struct { + mutex sync.RWMutex + metadata event.CallSDPStreamMetadata +} + type conf struct { - confID string - calls calls - tracks tracks + confID string + calls calls + tracks tracks + metadata metadata } func (c *conf) getCall(callID string, create bool) (*call, error) { @@ -62,7 +69,6 @@ func (c *conf) getCall(callID string, create bool) (*call, error) { callID: callID, conf: c, } - ca.subscribedTracks.tracks = []localTrackInfo{} c.calls.calls[callID] = ca } else { return nil, errors.New("no such call") @@ -160,3 +166,61 @@ func (c *conf) removeOldCallsByDeviceAndSessionIds(deviceID id.DeviceID, session } return err } + +func (c *conf) updateSDPStreamMetadata(deviceId id.DeviceID, metadata event.CallSDPStreamMetadata) { + c.metadata.mutex.Lock() + // Update existing and add new + for streamId, info := range metadata { + c.metadata.metadata[streamId] = info + } + // Remove removed + for streamId, info := range c.metadata.metadata { + _, exists := metadata[streamId] + if info.DeviceID == deviceId && !exists { + delete(c.metadata.metadata, streamId) + } + } + c.metadata.mutex.Unlock() +} + +func (c *conf) getRemoteMetadataForDevice(deviceId id.DeviceID) event.CallSDPStreamMetadata { + metadata := make(event.CallSDPStreamMetadata) + c.metadata.mutex.Lock() + for streamId, info := range c.metadata.metadata { + metadata[streamId] = info + } + c.metadata.mutex.Unlock() + for streamId, info := range metadata { + if info.DeviceID == deviceId { + delete(metadata, streamId) + continue + } + for trackId := range info.Tracks { + if len(c.getLocalTrackIndicesByInfo(localTrackInfo{ + streamID: streamId, + trackID: trackId, + })) == 0 { + delete(metadata, streamId) + break + } + } + } + + return metadata +} + +func (c *conf) removeMetadataByDeviceId(deviceId id.DeviceID) { + for streamId, info := range c.metadata.metadata { + if info.DeviceID == deviceId { + delete(c.metadata.metadata, streamId) + } + } +} + +func (c *conf) sendUpdatedMetadataFromCall(callID string) { + for _, call := range c.calls.calls { + if call.callID != callID { + call.sendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + } + } +} diff --git a/src/focus.go b/src/focus.go index f0f51fe..00cd8a0 100644 --- a/src/focus.go +++ b/src/focus.go @@ -19,6 +19,8 @@ package main import ( "errors" "sync" + + "maunium.net/go/mautrix/event" ) type confs struct { @@ -48,6 +50,7 @@ func (f *focus) getConf(confID string, create bool) (*conf, error) { f.confs.confs[confID] = co co.calls.calls = make(map[string]*call) co.tracks.tracks = []localTrackWithInfo{} + co.metadata.metadata = make(event.CallSDPStreamMetadata) } else { return nil, errors.New("no such conf") } diff --git a/src/main.go b/src/main.go index 185b801..3d6243c 100644 --- a/src/main.go +++ b/src/main.go @@ -29,6 +29,7 @@ import ( yaml "gopkg.in/yaml.v3" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" _ "net/http/pprof" @@ -152,8 +153,9 @@ type dataChannelMessage struct { ID string `json:"id"` Message string `json:"message,omitempty"` // XXX: is this even needed? we know which conf a given call is for... - ConfID string `json:"conf_id,omitempty"` - Start []trackDesc `json:"start,omitempty"` - Stop []trackDesc `json:"stop,omitempty"` - SDP string `json:"sdp,omitempty"` + ConfID string `json:"conf_id,omitempty"` + Start []trackDesc `json:"start,omitempty"` + Stop []trackDesc `json:"stop,omitempty"` + SDP string `json:"sdp,omitempty"` + Metadata event.CallSDPStreamMetadata `json:"metadata,omitempty"` } From 02825eac81eff210b028261d02ebd574f173ac29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:04:17 +0200 Subject: [PATCH 089/124] Update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- README.md | 198 ++++++++---------------------------------------------- 1 file changed, 27 insertions(+), 171 deletions(-) diff --git a/README.md b/README.md index fbfea3d..c4c57c3 100644 --- a/README.md +++ b/README.md @@ -2,187 +2,43 @@ ## Why -`SFU-to-SFU` is an example of a cascaded decentralised SFU. The intention is to be a implementation of Matrix's [MSC3401: Native Group VoIP signalling](https://github.com/matrix-org/matrix-spec-proposals/blob/matthew/group-voip/proposals/3401-group-voip.md). -This example is self contained and doesn't require any external software. The project was informed by the following goals. - -* **Easy Scaling** - SFU count can be grown/shrunk as users arrive. We don't scale on the dimension of calls making things easier. -* **Shorter Last Mile** - Users can connect to SFUs closest to them. Links `SFU <-> SFU` are higher quality then public hops. -* **Flexibility in WebRTC server choice** - All communication takes place using standard protocols/formats. You can use whatever server software best fits your needs. -* **Client Simplicity** - Clients will need to be created on lots of platforms. We should aim to use native WebRTC features as much as possible. - -The SFUs themselves have no concept of conference calls/rooms etc... All of this is communicated in the Matrix room. The SFUs themselves just operate off of -pub/sub semantics. The pub/sub streams are keyed by `foci`, `call_id`, `device_id` and `purpose` these keys come from [MSC3401](https://github.com/matrix-org/matrix-spec-proposals/blob/matthew/group-voip/proposals/3401-group-voip.md). - -Lets say you have a Matrix room where user `Alice` wishes to publish a screenshare to `Bob` and `Charlie`. - -```none -* `Alice` establishes a session with a SFU -* `Alice` publishes a screenshare feed with `call_id`, `device_id` and `purpose` -* `Alice` publishes to the matrix room with the values `foci`, `call_id`, `device_id` and `purpose` - -# Connecting directly to publishers FOCI -* `Bob` connects directly to `foci` and establishes a session. -* `Bob` requests a stream with values `foci`, `call_id`, `device_id` and `purpose`. - -# Connect to FOCI through different SFU -* `Charlie` connects to a SFU they run on a remote host. -* `Charlie` requests a stream with values `foci`, `call_id`, `device_id` and `purpose`. -* `Charlie`'s SFU connects to `foci` and requests the stream. -* `Alice`'s stream arrives to Charlie via `Alice -> FOCI -> Charlie's SFU -> Charlie` -``` +`SFU-to-SFU` is an example of a cascaded decentralised SFU. The intention is to +be a implementation of Matrix's [MSC3401: Native Group VoIP +signalling](https://github.com/matrix-org/matrix-spec-proposals/blob/matthew/group-voip/proposals/3401-group-voip.md). +This example is self contained and doesn't require any external software. The +project was informed by the following goals. + +* **Easy Scaling** - SFU count can be grown/shrunk as users arrive. We don't + scale on the dimension of calls making things easier. +* **Shorter Last Mile** - Users can connect to SFUs closest to them. Links `SFU + <-> SFU` are higher quality then public hops. +* **Flexibility in WebRTC server choice** - All communication takes place using + standard protocols/formats. You can use whatever server software best fits + your needs. +* **Client Simplicity** - Clients will need to be created on lots of platforms. + We should aim to use native WebRTC features as much as possible. + +This implements the MSC only roughly - given the current experimental nature of +this projects, it deviates in certain areas from the MSC. ## How -### Establishing a session - -Client sends a POST with a WebRTC Offer that is datachannel only. Server responds with Answer. - -Server will open a datachannel called `signaling`. Clients can send publish/subscribe now. - -`POST /createSession` - -`Request` - -```sdp -o=- 6685856480478485828 2 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 -a=extmap-allow-mixed -a=msid-semantic: WMS -m=application 9 UDP/DTLS/SCTP webrtc-datachannel -c=IN IP4 0.0.0.0 -a=ice-ufrag:gLSF -a=ice-pwd:xuxSHK0uJuSb607uYunnzlCQ -a=ice-options:trickle -a=fingerprint:sha-256 C2:1F:9B:A1:C2:DF:7E:13:E4:F9:64:F5:EC:4D:17:A1:89:21:0E:32:61:2A:B7:A5:A7:2A:7C:06:AC:FB:B2:A1 -a=setup:actpass -a=mid:0 -a=sctp-port:5000 -a=max-message-size:262144 -``` - -`Response` - -```sdp -o=- 1712750552704711910 2 IN IP4 127.0.0.1 -s=- -t=0 0 -a=group:BUNDLE 0 -a=extmap-allow-mixed -a=msid-semantic: WMS -m=application 9 UDP/DTLS/SCTP webrtc-datachannel -c=IN IP4 0.0.0.0 -a=ice-ufrag:90cu -a=ice-pwd:PARVC6h9kLvvgCqxSocjrXYZ -a=ice-options:trickle -a=fingerprint:sha-256 7F:79:0F:50:FF:D1:3F:DF:CA:BD:06:89:2B:C8:05:2E:EC:7D:EF:66:AF:A8:6E:D8:70:C6:74:68:E6:5C:47:D7 -a=setup:active -a=mid:0 -a=sctp-port:5000 -a=max-message-size:262144 -``` - -### Publish a Stream - -A user can start publish a stream by making a JSON request to publish with a new Offer. With the following keys. - -* `event` - Must be `publish` -* `id` - Unique ID for this message. Allows server to respond with with errors -* Stream Identification - `call_id`, `device_id`, `purpose` -* `sdp` - Offer frome the Peer. Any new additional tracks will belong to the stream. +### Configuration -```json -{ - "event": "publish", - "id": "ABC", - "call_id": "AAA", - "device_id": "BBB", - "purpose": "DDD", - "sdp": "...", -} -``` +* `cp config.yaml.sample config.yaml` +* Fill in `config.yaml` -* **Errors** - * Stream already exists - * Server over capacity +### Running -The server will respond to the `subscribe` with the answer. - -```json -{ - "event": "publish", - "id": "ABC", - "call_id": "AAA", - "device_id": "BBB", - "purpose": "DDD", - "sdp": "...", -} -``` - -### Subscribe to a Stream - -A user can subscribe to a stream by making a JSON request to subscribe with a new Offer. With the following keys. - -* `event` - Must be `subscribe` -* `id` - Unique ID for this message. Allows server to respond with with errors -* Stream Identification - `call_id`, `device_id`, `purpose` - -```json -{ - "event": "subscribe", - "id": "ABC", - "call_id": "AAA", - "device_id": "BBB", - "purpose": "DDD", - "sdp": "...", -} -``` - -The client will respond to the `subscribe` with the answer. - -```json -{ - "event": "subscribe", - "id": "ABC", - "sdp": "...", -} -``` - -* **Errors** - * Stream doesn't exist - * Server over capacity - -### Unpublish a Stream - -```json -{ - "event": "unpublish", - "id": "ABC", - "call_id": "AAA", - "device_id": "BBB", - "purpose": "DDD", -} -``` - -### Unsubscribe to a Stream - -```json -{ - "event": "unsubscribe", - "id": "ABC", - "call_id": "AAA", - "device_id": "BBB", - "purpose": "DDD", -} -``` +* `./scripts/run.sh` +* Access at -## Running +### Profiling -* `./scripts/run.sh` +* `./scripts/profile.sh` * Access at -## Building +### Building * `./scripts/build.sh` * `./dist/bin` From e513dd18ad7b833cf4e92609aa728af940da2bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:06:11 +0200 Subject: [PATCH 090/124] Add new lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- .gitignore | 2 +- scripts/profile.sh | 2 +- scripts/run.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 397fbfa..5525cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ config.yaml dist/ *.pprof -.vscode/ \ No newline at end of file +.vscode/ diff --git a/scripts/profile.sh b/scripts/profile.sh index 1f53c9d..bb106cf 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof --logTime \ No newline at end of file +clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof --logTime diff --git a/scripts/run.sh b/scripts/run.sh index 6a2d664..5c721ee 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --logTime \ No newline at end of file +clear && go run ./src/*.go --logTime From e473cd9fa53c757ff47a9239731228c0510ab1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:07:31 +0200 Subject: [PATCH 091/124] Remove cascade - will be readded later MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/cascade.go | 146 ------------------------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 src/cascade.go diff --git a/src/cascade.go b/src/cascade.go deleted file mode 100644 index cf8ca67..0000000 --- a/src/cascade.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -/* -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "strings" - - "github.com/pion/webrtc/v3" -) - -// Given a FOCI + CallID + DeviceID + Purpose establish a session and Subscribe. Take -// the media from the remote and copy it to a `webrtc.TrackLocal` so we can re-send -func remoteStreamLookup(msg dataChannelMessage) (webrtc.TrackLocal, webrtc.TrackLocal) { - audioTrack, videoTrack := make(chan webrtc.TrackLocal, 1), make(chan webrtc.TrackLocal, 1) - - peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) - if err != nil { - panic(err) - } - - peerConnection.OnTrack(func(trackRemote *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { - trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) - if err != nil { - panic(err) - } - - if strings.Contains(trackRemote.Codec().MimeType, "video") { - videoTrack <- trackLocal - } else { - audioTrack <- trackLocal - } - - copyRemoteToLocal(trackRemote, trackLocal) - }) - - dataChannel, err := peerConnection.CreateDataChannel("signaling", nil) - if err != nil { - panic(err) - } - - dataChannel.OnOpen(func() { - if _, err := peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionRecvonly}); err != nil { - panic(err) - } - - if _, err := peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionRecvonly}); err != nil { - panic(err) - } - - offer, err := peerConnection.CreateOffer(nil) - if err != nil { - panic(err) - } - - if err := peerConnection.SetLocalDescription(offer); err != nil { - panic(err) - } - - msg.SDP = offer.SDP - marshaled, err := json.Marshal(msg) - if err != nil { - panic(err) - } - - if err = dataChannel.SendText(string(marshaled)); err != nil { - panic(err) - } - }) - - dataChannel.OnMessage(func(m webrtc.DataChannelMessage) { - if !m.IsString { - log.Fatal("Inbound message is not string") - } - - cascadedMsg := &dataChannelMessage{} - if err := json.Unmarshal(m.Data, cascadedMsg); err != nil { - log.Fatal(err) - } - - switch cascadedMsg.Event { - case "error": - audioTrack <- nil - videoTrack <- nil - case "subscribe": - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: cascadedMsg.SDP}); err != nil { - panic(err) - } - - default: - log.Fatalf("Unknown msg Event type %s", msg.Event) - } - }) - - offer, err := peerConnection.CreateOffer(nil) - if err != nil { - panic(err) - } - - if err := peerConnection.SetLocalDescription(offer); err != nil { - panic(err) - } - - resp, err := http.Post("http://"+msg.FOCI+"/createSession", "application/text", bytes.NewBuffer([]byte(offer.SDP))) - if err != nil { - panic(err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("Got HTTP Status code %d", resp.StatusCode)) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err) - } - - if err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: string(body)}); err != nil { - panic(err) - } - - return <-audioTrack, <-videoTrack - -} -*/ From 7dbe06b8a379085784958d08316f1fa4ffe8a1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:23:24 +0200 Subject: [PATCH 092/124] Follow naming conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 264 +++++++++++++++++++++++----------------------- src/conf.go | 226 --------------------------------------- src/conference.go | 226 +++++++++++++++++++++++++++++++++++++++ src/focus.go | 24 ++--- src/main.go | 22 ++-- src/matrix.go | 42 ++++---- src/utils.go | 2 +- 7 files changed, 403 insertions(+), 403 deletions(-) delete mode 100644 src/conf.go create mode 100644 src/conference.go diff --git a/src/call.go b/src/call.go index 2c00d87..bf51807 100644 --- a/src/call.go +++ b/src/call.go @@ -30,29 +30,29 @@ import ( "github.com/pion/webrtc/v3" ) -type call struct { - callID string - userID id.UserID - deviceID id.DeviceID - localSessionID id.SessionID - remoteSessionID id.SessionID - client *mautrix.Client - peerConnection *webrtc.PeerConnection - conf *conf - dataChannel *webrtc.DataChannel - lastKeepAliveTimestamp time.Time +type Call struct { + CallID string + UserID id.UserID + DeviceID id.DeviceID + LocalSessionID id.SessionID + RemoteSessionID id.SessionID + Client *mautrix.Client + PeerConnection *webrtc.PeerConnection + Conf *Conference + DataChannel *webrtc.DataChannel + LastKeepAliveTimestamp time.Time } -func (c *call) dataChannelHandler(d *webrtc.DataChannel) { - c.dataChannel = d - peerConnection := c.peerConnection +func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { + c.DataChannel = d + peerConnection := c.PeerConnection d.OnOpen(func() { - c.sendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + c.SendDataChannelMessage(dataChannelMessage{Op: "metadata"}) }) d.OnError(func(err error) { - log.Fatalf("%s | DC error: %s", c.callID, err) + log.Fatalf("%s | DC error: %s", c.CallID, err) }) d.OnMessage(func(m webrtc.DataChannelMessage) { @@ -62,7 +62,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { msg := &dataChannelMessage{} if err := json.Unmarshal(m.Data, msg); err != nil { - log.Fatalf("%s | failed to unmarshal: %s", c.callID, err) + log.Fatalf("%s | failed to unmarshal: %s", c.CallID, err) } // TODO: hook cascade back up. @@ -72,7 +72,7 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { // its streams. if msg.Metadata != nil { - c.conf.updateSDPStreamMetadata(c.deviceID, msg.Metadata) + c.Conf.UpdateSDPStreamMetadata(c.DeviceID, msg.Metadata) } switch msg.Op { @@ -82,53 +82,53 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { } for _, trackDesc := range msg.Start { - log.Printf("%s | selecting StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) - foundTracks := c.conf.getLocalTrackByInfo(localTrackInfo{ - streamID: trackDesc.StreamID, - trackID: trackDesc.TrackID, + log.Printf("%s | selecting StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) + foundTracks := c.Conf.GetLocalTrackByInfo(LocalTrackInfo{ + StreamID: trackDesc.StreamID, + TrackID: trackDesc.TrackID, }) if len(foundTracks) == 0 { - log.Printf("%s | no track found StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) + log.Printf("%s | no track found StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) continue } for _, track := range foundTracks { - log.Printf("%s | adding %s StreamID %s TrackID %s", c.userID, track.Kind(), track.StreamID(), track.ID()) - if _, err := c.peerConnection.AddTrack(track); err != nil { + log.Printf("%s | adding %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) + if _, err := c.PeerConnection.AddTrack(track); err != nil { panic(err) } } } case "publish": - log.Printf("%s | received DC publish", c.userID) + log.Printf("%s | received DC publish", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: msg.SDP, }) - offer, err := c.peerConnection.CreateAnswer(nil) + offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { panic(err) } - err = c.peerConnection.SetLocalDescription(offer) + err = c.PeerConnection.SetLocalDescription(offer) if err != nil { panic(err) } - c.sendDataChannelMessage(dataChannelMessage{ + c.SendDataChannelMessage(dataChannelMessage{ Op: "answer", SDP: offer.SDP, }) case "unpublish": for _, trackDesc := range msg.Stop { - log.Printf("%s | unpublishing StreamID %s TrackID %s", c.userID, trackDesc.StreamID, trackDesc.TrackID) - if removedTracksCount := c.conf.removeTracksFromPeerConnectionsByInfo(localTrackInfo{ - streamID: trackDesc.StreamID, - trackID: trackDesc.TrackID, + log.Printf("%s | unpublishing StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) + if removedTracksCount := c.Conf.RemoveTracksFromPeerConnectionsByInfo(LocalTrackInfo{ + StreamID: trackDesc.StreamID, + TrackID: trackDesc.TrackID, }); removedTracksCount == 0 { - log.Printf("%s | no tracks to remove for: %+v", c.userID, msg.Stop) + log.Printf("%s | no tracks to remove for: %+v", c.UserID, msg.Stop) } } @@ -138,22 +138,22 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { SDP: msg.SDP, }) - offer, err := c.peerConnection.CreateAnswer(nil) + offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { panic(err) } - err = c.peerConnection.SetLocalDescription(offer) + err = c.PeerConnection.SetLocalDescription(offer) if err != nil { panic(err) } - c.sendDataChannelMessage(dataChannelMessage{ + c.SendDataChannelMessage(dataChannelMessage{ Op: "answer", SDP: offer.SDP, }) case "answer": - log.Printf("%s | received DC answer", c.userID) + log.Printf("%s | received DC answer", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, @@ -161,12 +161,12 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) case "alive": - c.lastKeepAliveTimestamp = time.Now() + c.LastKeepAliveTimestamp = time.Now() case "metadata": - log.Printf("%s | received DC metadata", c.userID) + log.Printf("%s | received DC metadata", c.UserID) - c.conf.sendUpdatedMetadataFromCall(c.callID) + c.Conf.SendUpdatedMetadataFromCall(c.CallID) default: log.Fatalf("Unknown operation %s", msg.Op) @@ -175,23 +175,23 @@ func (c *call) dataChannelHandler(d *webrtc.DataChannel) { }) } -func (c *call) negotiationNeededHandler() { - offer, err := c.peerConnection.CreateOffer(nil) +func (c *Call) NegotiationNeededHandler() { + offer, err := c.PeerConnection.CreateOffer(nil) if err != nil { panic(err) } - err = c.peerConnection.SetLocalDescription(offer) + err = c.PeerConnection.SetLocalDescription(offer) if err != nil { panic(err) } - c.sendDataChannelMessage(dataChannelMessage{ + c.SendDataChannelMessage(dataChannelMessage{ Op: "offer", SDP: offer.SDP, }) } -func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { +func (c *Call) IceCandidateHandler(candidate *webrtc.ICECandidate) { if candidate == nil { return } @@ -202,12 +202,12 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), + CallID: c.CallID, + ConfID: c.Conf.ConfID, + DeviceID: c.Client.DeviceID, + SenderSessionID: c.LocalSessionID, + DestSessionID: c.RemoteSessionID, + PartyID: string(c.Client.DeviceID), Version: event.CallVersion("1"), }, Candidates: []event.CallCandidate{ @@ -220,18 +220,18 @@ func (c *call) iceCandidateHandler(candidate *webrtc.ICECandidate) { }, }, } - c.sendToDevice(event.CallCandidates, candidateEvtContent) + c.SendToDevice(event.CallCandidates, candidateEvtContent) } -func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { +func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { // FIXME: This is a potential performance killer if strings.Contains(trackRemote.Codec().MimeType, "video") { // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval go func() { ticker := time.NewTicker(time.Millisecond * 200) for range ticker.C { - if err := c.peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { - log.Printf("%s | failed to write RTCP on trackID %s: %s", c.userID, trackRemote.ID(), err) + if err := c.PeerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + log.Printf("%s | failed to write RTCP on trackID %s: %s", c.UserID, trackRemote.ID(), err) break } } @@ -243,54 +243,54 @@ func (c *call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece panic(err) } - c.conf.tracks.mutex.Lock() - c.conf.tracks.tracks = append(c.conf.tracks.tracks, localTrackWithInfo{ - track: trackLocal, - info: localTrackInfo{ - trackID: trackLocal.ID(), - streamID: trackLocal.StreamID(), - call: c, + c.Conf.Tracks.Mutex.Lock() + c.Conf.Tracks.Tracks = append(c.Conf.Tracks.Tracks, LocalTrackWithInfo{ + Track: trackLocal, + Info: LocalTrackInfo{ + TrackID: trackLocal.ID(), + StreamID: trackLocal.StreamID(), + Call: c, }, }) - c.conf.tracks.mutex.Unlock() + c.Conf.Tracks.Mutex.Unlock() - log.Printf("%s | published %s StreamID %s TrackID %s", c.userID, trackLocal.Kind(), trackLocal.StreamID(), trackLocal.ID()) + log.Printf("%s | published %s StreamID %s TrackID %s", c.UserID, trackLocal.Kind(), trackLocal.StreamID(), trackLocal.ID()) - go c.conf.sendUpdatedMetadataFromCall(c.callID) - go copyRemoteToLocal(trackRemote, trackLocal) + go c.Conf.SendUpdatedMetadataFromCall(c.CallID) + go CopyRemoteToLocal(trackRemote, trackLocal) } -func (c *call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { +func (c *Call) IceConnectionStateHandler(state webrtc.ICEConnectionState) { if state == webrtc.ICEConnectionStateCompleted || state == webrtc.ICEConnectionStateConnected { - c.lastKeepAliveTimestamp = time.Now() - go c.checkKeepAliveTimestamp() + c.LastKeepAliveTimestamp = time.Now() + go c.CheckKeepAliveTimestamp() } } -func (c *call) onInvite(content *event.CallInviteEventContent) error { - c.conf.updateSDPStreamMetadata(c.deviceID, content.SDPStreamMetadata) +func (c *Call) OnInvite(content *event.CallInviteEventContent) error { + c.Conf.UpdateSDPStreamMetadata(c.DeviceID, content.SDPStreamMetadata) offer := content.Offer peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) if err != nil { return err } - c.peerConnection = peerConnection + c.PeerConnection = peerConnection peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - c.trackHandler(track, receiver) + c.TrackHandler(track, receiver) }) peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - c.dataChannelHandler(d) + c.DataChannelHandler(d) }) peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - c.iceCandidateHandler(candidate) + c.IceCandidateHandler(candidate) }) peerConnection.OnNegotiationNeeded(func() { - c.negotiationNeededHandler() + c.NegotiationNeededHandler() }) peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) { - c.iceConnectionStateHandler(state) + c.IceConnectionStateHandler(state) }) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -315,39 +315,39 @@ func (c *call) onInvite(content *event.CallInviteEventContent) error { answerEvtContent := &event.Content{ Parsed: event.CallAnswerEventContent{ BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), + CallID: c.CallID, + ConfID: c.Conf.ConfID, + DeviceID: c.Client.DeviceID, + SenderSessionID: c.LocalSessionID, + DestSessionID: c.RemoteSessionID, + PartyID: string(c.Client.DeviceID), Version: event.CallVersion("1"), }, Answer: event.CallData{ Type: "answer", SDP: answerSdp, }, - SDPStreamMetadata: c.conf.getRemoteMetadataForDevice(c.deviceID), + SDPStreamMetadata: c.Conf.GetRemoteMetadataForDevice(c.DeviceID), }, } - c.sendToDevice(event.CallAnswer, answerEvtContent) + c.SendToDevice(event.CallAnswer, answerEvtContent) return err } -func (c *call) onSelectAnswer(content *event.CallSelectAnswerEventContent) { - selectedPartyId := content.SelectedPartyID - if selectedPartyId != string(c.client.DeviceID) { - c.terminate() - log.Printf("%s | Call was answered on a different device: %s", c.userID, selectedPartyId) +func (c *Call) OnSelectAnswer(content *event.CallSelectAnswerEventContent) { + selectedPartyID := content.SelectedPartyID + if selectedPartyID != string(c.Client.DeviceID) { + c.Terminate() + log.Printf("%s | Call was answered on a different device: %s", c.UserID, selectedPartyID) } } -func (c *call) onHangup(content *event.CallHangupEventContent) { - c.terminate() +func (c *Call) OnHangup(content *event.CallHangupEventContent) { + c.Terminate() } -func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { +func (c *Call) OnCandidates(content *event.CallCandidatesEventContent) error { for _, candidate := range content.Candidates { sdpMLineIndex := uint16(candidate.SDPMLineIndex) ice := webrtc.ICECandidateInit{ @@ -356,7 +356,7 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { SDPMLineIndex: &sdpMLineIndex, UsernameFragment: new(string), } - if err := c.peerConnection.AddICECandidate(ice); err != nil { + if err := c.PeerConnection.AddICECandidate(ice); err != nil { log.Print("Failed to add ICE candidate", content) return err } @@ -364,67 +364,67 @@ func (c *call) onCandidates(content *event.CallCandidatesEventContent) error { return nil } -func (c *call) terminate() { - log.Printf("%s | terminating call", c.userID) +func (c *Call) Terminate() { + log.Printf("%s | terminating call", c.UserID) - if err := c.peerConnection.Close(); err != nil { - log.Printf("%s | error closing peer connection: %s", c.userID, err) + if err := c.PeerConnection.Close(); err != nil { + log.Printf("%s | error closing peer connection: %s", c.UserID, err) } - c.conf.calls.callsMu.Lock() - delete(c.conf.calls.calls, c.callID) - c.conf.calls.callsMu.Unlock() + c.Conf.Calls.CallsMu.Lock() + delete(c.Conf.Calls.Calls, c.CallID) + c.Conf.Calls.CallsMu.Unlock() - info := localTrackInfo{call: c} - c.conf.removeTracksFromPeerConnectionsByInfo(info) - c.conf.removeTracksFromConfByInfo(info) - c.conf.removeMetadataByDeviceId(c.deviceID) - c.conf.sendUpdatedMetadataFromCall(c.callID) + info := LocalTrackInfo{Call: c} + c.Conf.RemoveTracksFromPeerConnectionsByInfo(info) + c.Conf.RemoveTracksFromConfByInfo(info) + c.Conf.RemoveMetadataByDeviceID(c.DeviceID) + c.Conf.SendUpdatedMetadataFromCall(c.CallID) } -func (c *call) hangup(reason event.CallHangupReason) { +func (c *Call) Hangup(reason event.CallHangupReason) { hangupEvtContent := &event.Content{ Parsed: event.CallHangupEventContent{ BaseCallEventContent: event.BaseCallEventContent{ - CallID: c.callID, - ConfID: c.conf.confID, - DeviceID: c.client.DeviceID, - SenderSessionID: c.localSessionID, - DestSessionID: c.remoteSessionID, - PartyID: string(c.client.DeviceID), + CallID: c.CallID, + ConfID: c.Conf.ConfID, + DeviceID: c.Client.DeviceID, + SenderSessionID: c.LocalSessionID, + DestSessionID: c.RemoteSessionID, + PartyID: string(c.Client.DeviceID), Version: event.CallVersion("1"), }, Reason: reason, }, } - c.sendToDevice(event.CallHangup, hangupEvtContent) - c.terminate() + c.SendToDevice(event.CallHangup, hangupEvtContent) + c.Terminate() } -func (c *call) sendToDevice(callType event.Type, content *event.Content) { +func (c *Call) SendToDevice(callType event.Type, content *event.Content) { if callType.Type != event.CallCandidates.Type { - log.Printf("%s | sending to device %s", c.userID, callType.Type) + log.Printf("%s | sending to device %s", c.UserID, callType.Type) } toDevice := &mautrix.ReqSendToDevice{ Messages: map[id.UserID]map[id.DeviceID]*event.Content{ - c.userID: { - c.deviceID: content, + c.UserID: { + c.DeviceID: content, }, }, } // TODO: E2EE // TODO: to-device reliability - c.client.SendToDevice(callType, toDevice) + c.Client.SendToDevice(callType, toDevice) } -func (c *call) sendDataChannelMessage(msg dataChannelMessage) { - if c.dataChannel == nil { +func (c *Call) SendDataChannelMessage(msg dataChannelMessage) { + if c.DataChannel == nil { return } - msg.ConfID = c.conf.confID - msg.Metadata = c.conf.getRemoteMetadataForDevice(c.deviceID) + msg.ConfID = c.Conf.ConfID + msg.Metadata = c.Conf.GetRemoteMetadataForDevice(c.DeviceID) // TODO: Set ID if msg.Op == "metadata" && len(msg.Metadata) == 0 { @@ -436,20 +436,20 @@ func (c *call) sendDataChannelMessage(msg dataChannelMessage) { panic(err) } - err = c.dataChannel.SendText(string(marshaled)) + err = c.DataChannel.SendText(string(marshaled)) if err != nil { - log.Printf("%s | failed to send %s over DC: %s", c.userID, msg.Op, err) + log.Printf("%s | failed to send %s over DC: %s", c.UserID, msg.Op, err) } - log.Printf("%s | sent DC %s", c.userID, msg.Op) + log.Printf("%s | sent DC %s", c.UserID, msg.Op) } -func (c *call) checkKeepAliveTimestamp() { +func (c *Call) CheckKeepAliveTimestamp() { timeout := time.Second * time.Duration(configInstance.Timeout) for range time.Tick(timeout) { - if c.lastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { - log.Printf("%s | did not get keep-alive message in the last %s:", c.userID, timeout) - c.hangup(event.CallHangupKeepAliveTimeout) + if c.LastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { + log.Printf("%s | did not get keep-alive message in the last %s:", c.UserID, timeout) + c.Hangup(event.CallHangupKeepAliveTimeout) break } } diff --git a/src/conf.go b/src/conf.go deleted file mode 100644 index 9ea2776..0000000 --- a/src/conf.go +++ /dev/null @@ -1,226 +0,0 @@ -/* -Copyright 2022 The Matrix.org Foundation C.I.C. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "log" - "sync" - - "github.com/pion/webrtc/v3" - "maunium.net/go/mautrix/event" - "maunium.net/go/mautrix/id" -) - -type localTrackInfo struct { - streamID string - trackID string - call *call -} - -type localTrackWithInfo struct { - track *webrtc.TrackLocalStaticRTP - info localTrackInfo -} - -type calls struct { - callsMu sync.RWMutex - calls map[string]*call // By callID -} - -type tracks struct { - mutex sync.RWMutex - tracks []localTrackWithInfo -} - -type metadata struct { - mutex sync.RWMutex - metadata event.CallSDPStreamMetadata -} - -type conf struct { - confID string - calls calls - tracks tracks - metadata metadata -} - -func (c *conf) getCall(callID string, create bool) (*call, error) { - c.calls.callsMu.Lock() - defer c.calls.callsMu.Unlock() - ca := c.calls.calls[callID] - if ca == nil { - if create { - ca = &call{ - callID: callID, - conf: c, - } - c.calls.calls[callID] = ca - } else { - return nil, errors.New("no such call") - } - } - return ca, nil -} - -func (c *conf) getLocalTrackIndicesByInfo(selectInfo localTrackInfo) (tracks []int) { - foundIndices := []int{} - for index, track := range c.tracks.tracks { - info := track.info - if selectInfo.call != nil && selectInfo.call != info.call { - continue - } - if selectInfo.streamID != "" && selectInfo.streamID != info.streamID { - continue - } - if selectInfo.trackID != "" && selectInfo.trackID != info.trackID { - continue - } - foundIndices = append(foundIndices, index) - } - - return foundIndices -} - -func (c *conf) getLocalTrackByInfo(selectInfo localTrackInfo) (tracks []webrtc.TrackLocal) { - indices := c.getLocalTrackIndicesByInfo(selectInfo) - foundTracks := []webrtc.TrackLocal{} - for _, index := range indices { - foundTracks = append(foundTracks, c.tracks.tracks[index].track) - } - - return foundTracks -} - -func (c *conf) removeTracksFromPeerConnectionsByInfo(removeInfo localTrackInfo) int { - indices := c.getLocalTrackIndicesByInfo(removeInfo) - - // FIXME: the big O of this must be awful... - for _, index := range indices { - info := c.tracks.tracks[index].info - - for _, call := range c.calls.calls { - for _, sender := range call.peerConnection.GetSenders() { - if info.trackID == sender.Track().ID() { - log.Printf("%s | removing %s StreamID %s TrackID %s", call.userID, sender.Track().Kind(), sender.Track().StreamID(), sender.Track().ID()) - if err := sender.Stop(); err != nil { - log.Printf("%s | failed to stop sender: %s", call.userID, err) - } - if err := call.peerConnection.RemoveTrack(sender); err != nil { - log.Printf("%s | failed to remove track: %s", call.userID, err) - } - } - } - } - } - - return len(indices) -} - -func (c *conf) removeTracksFromConfByInfo(removeInfo localTrackInfo) { - c.tracks.mutex.Lock() - defer c.tracks.mutex.Unlock() - - indicesToRemove := c.getLocalTrackIndicesByInfo(removeInfo) - - newTracks := []localTrackWithInfo{} - for index, track := range c.tracks.tracks { - keep := true - for _, indexToRemove := range indicesToRemove { - if indexToRemove == index { - keep = false - } - } - if keep { - newTracks = append(newTracks, track) - } - } - - c.tracks.tracks = newTracks -} - -func (c *conf) removeOldCallsByDeviceAndSessionIds(deviceID id.DeviceID, sessionID id.SessionID) error { - var err error - for _, call := range c.calls.calls { - if call.deviceID == deviceID { - if call.remoteSessionID == sessionID { - err = errors.New("found existing call with equal DeviceID and SessionID") - } else { - call.terminate() - } - } - } - return err -} - -func (c *conf) updateSDPStreamMetadata(deviceId id.DeviceID, metadata event.CallSDPStreamMetadata) { - c.metadata.mutex.Lock() - // Update existing and add new - for streamId, info := range metadata { - c.metadata.metadata[streamId] = info - } - // Remove removed - for streamId, info := range c.metadata.metadata { - _, exists := metadata[streamId] - if info.DeviceID == deviceId && !exists { - delete(c.metadata.metadata, streamId) - } - } - c.metadata.mutex.Unlock() -} - -func (c *conf) getRemoteMetadataForDevice(deviceId id.DeviceID) event.CallSDPStreamMetadata { - metadata := make(event.CallSDPStreamMetadata) - c.metadata.mutex.Lock() - for streamId, info := range c.metadata.metadata { - metadata[streamId] = info - } - c.metadata.mutex.Unlock() - for streamId, info := range metadata { - if info.DeviceID == deviceId { - delete(metadata, streamId) - continue - } - for trackId := range info.Tracks { - if len(c.getLocalTrackIndicesByInfo(localTrackInfo{ - streamID: streamId, - trackID: trackId, - })) == 0 { - delete(metadata, streamId) - break - } - } - } - - return metadata -} - -func (c *conf) removeMetadataByDeviceId(deviceId id.DeviceID) { - for streamId, info := range c.metadata.metadata { - if info.DeviceID == deviceId { - delete(c.metadata.metadata, streamId) - } - } -} - -func (c *conf) sendUpdatedMetadataFromCall(callID string) { - for _, call := range c.calls.calls { - if call.callID != callID { - call.sendDataChannelMessage(dataChannelMessage{Op: "metadata"}) - } - } -} diff --git a/src/conference.go b/src/conference.go new file mode 100644 index 0000000..854dab9 --- /dev/null +++ b/src/conference.go @@ -0,0 +1,226 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "log" + "sync" + + "github.com/pion/webrtc/v3" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type LocalTrackInfo struct { + StreamID string + TrackID string + Call *Call +} + +type LocalTrackWithInfo struct { + Track *webrtc.TrackLocalStaticRTP + Info LocalTrackInfo +} + +type Calls struct { + CallsMu sync.RWMutex + Calls map[string]*Call // By callID +} + +type Tracks struct { + Mutex sync.RWMutex + Tracks []LocalTrackWithInfo +} + +type Metadata struct { + Mutex sync.RWMutex + Metadata event.CallSDPStreamMetadata +} + +type Conference struct { + ConfID string + Calls Calls + Tracks Tracks + Metadata Metadata +} + +func (c *Conference) GetCall(callID string, create bool) (*Call, error) { + c.Calls.CallsMu.Lock() + defer c.Calls.CallsMu.Unlock() + ca := c.Calls.Calls[callID] + if ca == nil { + if create { + ca = &Call{ + CallID: callID, + Conf: c, + } + c.Calls.Calls[callID] = ca + } else { + return nil, errors.New("no such call") + } + } + return ca, nil +} + +func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (tracks []int) { + foundIndices := []int{} + for index, track := range c.Tracks.Tracks { + info := track.Info + if selectInfo.Call != nil && selectInfo.Call != info.Call { + continue + } + if selectInfo.StreamID != "" && selectInfo.StreamID != info.StreamID { + continue + } + if selectInfo.TrackID != "" && selectInfo.TrackID != info.TrackID { + continue + } + foundIndices = append(foundIndices, index) + } + + return foundIndices +} + +func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []webrtc.TrackLocal) { + indices := c.GetLocalTrackIndicesByInfo(selectInfo) + foundTracks := []webrtc.TrackLocal{} + for _, index := range indices { + foundTracks = append(foundTracks, c.Tracks.Tracks[index].Track) + } + + return foundTracks +} + +func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrackInfo) int { + indices := c.GetLocalTrackIndicesByInfo(removeInfo) + + // FIXME: the big O of this must be awful... + for _, index := range indices { + info := c.Tracks.Tracks[index].Info + + for _, call := range c.Calls.Calls { + for _, sender := range call.PeerConnection.GetSenders() { + if info.TrackID == sender.Track().ID() { + log.Printf("%s | removing %s StreamID %s TrackID %s", call.UserID, sender.Track().Kind(), sender.Track().StreamID(), sender.Track().ID()) + if err := sender.Stop(); err != nil { + log.Printf("%s | failed to stop sender: %s", call.UserID, err) + } + if err := call.PeerConnection.RemoveTrack(sender); err != nil { + log.Printf("%s | failed to remove track: %s", call.UserID, err) + } + } + } + } + } + + return len(indices) +} + +func (c *Conference) RemoveTracksFromConfByInfo(removeInfo LocalTrackInfo) { + c.Tracks.Mutex.Lock() + defer c.Tracks.Mutex.Unlock() + + indicesToRemove := c.GetLocalTrackIndicesByInfo(removeInfo) + + newTracks := []LocalTrackWithInfo{} + for index, track := range c.Tracks.Tracks { + keep := true + for _, indexToRemove := range indicesToRemove { + if indexToRemove == index { + keep = false + } + } + if keep { + newTracks = append(newTracks, track) + } + } + + c.Tracks.Tracks = newTracks +} + +func (c *Conference) RemoveOldCallsByDeviceAndSessionIDs(deviceID id.DeviceID, sessionID id.SessionID) error { + var err error + for _, call := range c.Calls.Calls { + if call.DeviceID == deviceID { + if call.RemoteSessionID == sessionID { + err = errors.New("found existing call with equal DeviceID and SessionID") + } else { + call.Terminate() + } + } + } + return err +} + +func (c *Conference) UpdateSDPStreamMetadata(deviceID id.DeviceID, metadata event.CallSDPStreamMetadata) { + c.Metadata.Mutex.Lock() + // Update existing and add new + for streamID, info := range metadata { + c.Metadata.Metadata[streamID] = info + } + // Remove removed + for streamID, info := range c.Metadata.Metadata { + _, exists := metadata[streamID] + if info.DeviceID == deviceID && !exists { + delete(c.Metadata.Metadata, streamID) + } + } + c.Metadata.Mutex.Unlock() +} + +func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.CallSDPStreamMetadata { + metadata := make(event.CallSDPStreamMetadata) + c.Metadata.Mutex.Lock() + for streamID, info := range c.Metadata.Metadata { + metadata[streamID] = info + } + c.Metadata.Mutex.Unlock() + for streamID, info := range metadata { + if info.DeviceID == deviceID { + delete(metadata, streamID) + continue + } + for trackID := range info.Tracks { + if len(c.GetLocalTrackIndicesByInfo(LocalTrackInfo{ + StreamID: streamID, + TrackID: trackID, + })) == 0 { + delete(metadata, streamID) + break + } + } + } + + return metadata +} + +func (c *Conference) RemoveMetadataByDeviceID(deviceID id.DeviceID) { + for streamID, info := range c.Metadata.Metadata { + if info.DeviceID == deviceID { + delete(c.Metadata.Metadata, streamID) + } + } +} + +func (c *Conference) SendUpdatedMetadataFromCall(callID string) { + for _, call := range c.Calls.Calls { + if call.CallID != callID { + call.SendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + } + } +} diff --git a/src/focus.go b/src/focus.go index 00cd8a0..64ac40d 100644 --- a/src/focus.go +++ b/src/focus.go @@ -23,34 +23,34 @@ import ( "maunium.net/go/mautrix/event" ) -type confs struct { +type Confs struct { confsMu sync.RWMutex - confs map[string]*conf + confs map[string]*Conference } -type focus struct { +type Focus struct { name string - confs confs + confs Confs } -func (f *focus) Init(name string) { +func (f *Focus) Init(name string) { f.name = name - f.confs.confs = make(map[string]*conf) + f.confs.confs = make(map[string]*Conference) } -func (f *focus) getConf(confID string, create bool) (*conf, error) { +func (f *Focus) GetConf(confID string, create bool) (*Conference, error) { f.confs.confsMu.Lock() defer f.confs.confsMu.Unlock() co := f.confs.confs[confID] if co == nil { if create { - co = &conf{ - confID: confID, + co = &Conference{ + ConfID: confID, } f.confs.confs[confID] = co - co.calls.calls = make(map[string]*call) - co.tracks.tracks = []localTrackWithInfo{} - co.metadata.metadata = make(event.CallSDPStreamMetadata) + co.Calls.Calls = make(map[string]*Call) + co.Tracks.Tracks = []LocalTrackWithInfo{} + co.Metadata.Metadata = make(event.CallSDPStreamMetadata) } else { return nil, errors.New("no such conf") } diff --git a/src/main.go b/src/main.go index 3d6243c..439c5c0 100644 --- a/src/main.go +++ b/src/main.go @@ -42,7 +42,7 @@ var configFilePath = flag.String("config", "config.yaml", "configuration file pa var cpuProfile = flag.String("cpuProfile", "", "write CPU profile to `file`") var memProfile = flag.String("memProfile", "", "write memory profile to `file`") -func initCpuProfiling(cpuProfile *string) func() { +func InitCpuProfiling(cpuProfile *string) func() { log.Print("initializing CPU profiling") f, err := os.Create(*cpuProfile) @@ -61,7 +61,7 @@ func initCpuProfiling(cpuProfile *string) func() { } } -func initMemoryProfiling(memProfile *string) func() { +func InitMemoryProfiling(memProfile *string) func() { log.Print("initializing memory profiling") return func() { @@ -79,14 +79,14 @@ func initMemoryProfiling(memProfile *string) func() { } } -func initLogging(logTime *bool) { +func InitLogging(logTime *bool) { log.SetFlags(0) if *logTime { log.SetFlags(log.Ldate | log.Ltime) } } -func loadConfig(configFilePath string) (*config, error) { +func LoadConfig(configFilePath string) (*config, error) { log.Printf("loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { @@ -99,7 +99,7 @@ func loadConfig(configFilePath string) (*config, error) { return &config, nil } -func onKill(c chan os.Signal, beforeExit []func()) { +func OnKill(c chan os.Signal, beforeExit []func()) { <-c log.Printf("ending program") for _, function := range beforeExit { @@ -111,27 +111,27 @@ func onKill(c chan os.Signal, beforeExit []func()) { func main() { flag.Parse() - initLogging(logTime) + InitLogging(logTime) beforeExit := []func(){} if *cpuProfile != "" { - beforeExit = append(beforeExit, initCpuProfiling(cpuProfile)) + beforeExit = append(beforeExit, InitCpuProfiling(cpuProfile)) } if *memProfile != "" { - beforeExit = append(beforeExit, initMemoryProfiling(memProfile)) + beforeExit = append(beforeExit, InitMemoryProfiling(memProfile)) } // try to handle os interrupt(signal terminated) c := make(chan os.Signal, 2) signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go onKill(c, beforeExit) + go OnKill(c, beforeExit) var err error - if configInstance, err = loadConfig(*configFilePath); err != nil { + if configInstance, err = LoadConfig(*configFilePath); err != nil { log.Fatalf("failed to load config file: %s", err) } - if err := initMatrix(); err != nil { + if err := InitMatrix(); err != nil { log.Fatalf("failed to init Matrix: %s", err) } } diff --git a/src/matrix.go b/src/matrix.go index d93727b..fbbadfa 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -28,7 +28,7 @@ import ( const localSessionID = "sfu" -func initMatrix() error { +func InitMatrix() error { client, err := mautrix.NewClient(configInstance.HomeserverURL, configInstance.UserID, configInstance.AccessToken) if err != nil { log.Fatal("Failed to create client", err) @@ -44,7 +44,7 @@ func initMatrix() error { log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID - focus := new(focus) + focus := new(Focus) focus.Init(fmt.Sprintf("%s (%s)", configInstance.UserID, client.DeviceID)) syncer := client.Syncer.(*mautrix.DefaultSyncer) @@ -70,15 +70,15 @@ func initMatrix() error { // TODO: E2EE - getExistingCall := func(confID string, callID string) (*call, error) { - var conf *conf - var call *call + getExistingCall := func(confID string, callID string) (*Call, error) { + var conf *Conference + var call *Call - if conf, err = focus.getConf(confID, false); err != nil || conf == nil { + if conf, err = focus.GetConf(confID, false); err != nil || conf == nil { log.Printf("failed to get conf %s: %s", confID, err) return nil, err } - if call, err = conf.getCall(callID, false); err != nil || call == nil { + if call, err = conf.GetCall(callID, false); err != nil || call == nil { log.Printf("failed to get call %s: %s", callID, err) return nil, err } @@ -94,8 +94,8 @@ func initMatrix() error { continue } - var conf *conf - var call *call + var conf *Conference + var call *Call if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { log.Printf("received non-call to-device event %s", evt.Type.Type) @@ -112,43 +112,43 @@ func initMatrix() error { switch evt.Type.Type { case CallInvite.Type: invite := evt.Content.AsCallInvite() - if conf, err = focus.getConf(invite.ConfID, true); err != nil || conf == nil { + if conf, err = focus.GetConf(invite.ConfID, true); err != nil || conf == nil { log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) return true } - if err := conf.removeOldCallsByDeviceAndSessionIds(invite.DeviceID, invite.SenderSessionID); err != nil { + if err := conf.RemoveOldCallsByDeviceAndSessionIDs(invite.DeviceID, invite.SenderSessionID); err != nil { log.Printf("%s | error removing old calls - ignoring call: %+v", evt.Sender.String(), err) return true } - if call, err = conf.getCall(invite.CallID, true); err != nil || call == nil { + if call, err = conf.GetCall(invite.CallID, true); err != nil || call == nil { log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) return true } - call.userID = evt.Sender - call.deviceID = invite.DeviceID + call.UserID = evt.Sender + call.DeviceID = invite.DeviceID // XXX: What if an SFU gets restarted? - call.localSessionID = localSessionID - call.remoteSessionID = invite.SenderSessionID - call.client = client - call.onInvite(invite) + call.LocalSessionID = localSessionID + call.RemoteSessionID = invite.SenderSessionID + call.Client = client + call.OnInvite(invite) case CallCandidates.Type: candidates := evt.Content.AsCallCandidates() if call, err = getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil || call == nil { return true } - call.onCandidates(candidates) + call.OnCandidates(candidates) case CallSelectAnswer.Type: selectAnswer := evt.Content.AsCallSelectAnswer() if call, err = getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil || call == nil { return true } - call.onSelectAnswer(selectAnswer) + call.OnSelectAnswer(selectAnswer) case CallHangup.Type: hangup := evt.Content.AsCallHangup() if call, err = getExistingCall(hangup.ConfID, hangup.CallID); err != nil || call == nil { return true } - call.onHangup(hangup) + call.OnHangup(hangup) // Events we don't care about case CallNegotiate.Type: diff --git a/src/utils.go b/src/utils.go index 698bc9b..fec0bde 100644 --- a/src/utils.go +++ b/src/utils.go @@ -22,7 +22,7 @@ import ( "github.com/pion/webrtc/v3" ) -func copyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { +func CopyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.TrackLocalStaticRTP) { buff := make([]byte, 1500) for { i, _, err := trackRemote.Read(buff) From 052aaccd3456e4c4c04168b2434707ec7b6220e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:34:46 +0200 Subject: [PATCH 093/124] Add `WriteRTCP()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 16 +--------------- src/utils.go | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/call.go b/src/call.go index bf51807..f9d0c6d 100644 --- a/src/call.go +++ b/src/call.go @@ -19,14 +19,12 @@ package main import ( "encoding/json" "log" - "strings" "time" "maunium.net/go/mautrix" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "github.com/pion/rtcp" "github.com/pion/webrtc/v3" ) @@ -224,19 +222,7 @@ func (c *Call) IceCandidateHandler(candidate *webrtc.ICECandidate) { } func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { - // FIXME: This is a potential performance killer - if strings.Contains(trackRemote.Codec().MimeType, "video") { - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval - go func() { - ticker := time.NewTicker(time.Millisecond * 200) - for range ticker.C { - if err := c.PeerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { - log.Printf("%s | failed to write RTCP on trackID %s: %s", c.UserID, trackRemote.ID(), err) - break - } - } - }() - } + go WriteRTCP(trackRemote, c.PeerConnection) trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) if err != nil { diff --git a/src/utils.go b/src/utils.go index fec0bde..d68b871 100644 --- a/src/utils.go +++ b/src/utils.go @@ -18,7 +18,10 @@ package main import ( "log" + "strings" + "time" + "github.com/pion/rtcp" "github.com/pion/webrtc/v3" ) @@ -27,12 +30,28 @@ func CopyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.Track for { i, _, err := trackRemote.Read(buff) if err != nil || buff == nil { - log.Printf("ending read on track with StreamID %s: %s", trackRemote.StreamID(), err) + log.Printf("ending read on TrackID %s: %s", trackRemote.ID(), err) break } if _, err = trackLocal.Write(buff[:i]); err != nil { - log.Printf("ending write on track with StreamID %s: %s", trackLocal.StreamID(), err) + log.Printf("ending write on TrackID %s: %s", trackLocal.ID(), err) + break + } + } +} + +func WriteRTCP(trackRemote *webrtc.TrackRemote, peerConnection *webrtc.PeerConnection) { + if !strings.Contains(trackRemote.Codec().MimeType, "video") { + return + } + + // FIXME: This is a potential performance killer + // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + ticker := time.NewTicker(time.Millisecond * 200) + for range ticker.C { + if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + log.Printf("ending RTCP write on TrackID %s: %s", trackRemote.ID(), err) break } } From 8d730e02e673fabf9f6ca0f01a916002513e831b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 15:40:46 +0200 Subject: [PATCH 094/124] Rename `config` to `Config` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- src/main.go | 22 +++++++++++----------- src/matrix.go | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/call.go b/src/call.go index f9d0c6d..ba27212 100644 --- a/src/call.go +++ b/src/call.go @@ -431,7 +431,7 @@ func (c *Call) SendDataChannelMessage(msg dataChannelMessage) { } func (c *Call) CheckKeepAliveTimestamp() { - timeout := time.Second * time.Duration(configInstance.Timeout) + timeout := time.Second * time.Duration(config.Timeout) for range time.Tick(timeout) { if c.LastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { log.Printf("%s | did not get keep-alive message in the last %s:", c.UserID, timeout) diff --git a/src/main.go b/src/main.go index 439c5c0..0b9024d 100644 --- a/src/main.go +++ b/src/main.go @@ -35,7 +35,14 @@ import ( _ "net/http/pprof" ) -var configInstance *config +type Config struct { + UserID id.UserID + HomeserverURL string + AccessToken string + Timeout int +} + +var config *Config var logTime = flag.Bool("logTime", false, "whether or not to print time and date in logs") var configFilePath = flag.String("config", "config.yaml", "configuration file path") @@ -86,13 +93,13 @@ func InitLogging(logTime *bool) { } } -func LoadConfig(configFilePath string) (*config, error) { +func LoadConfig(configFilePath string) (*Config, error) { log.Printf("loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { log.Fatalf("failed to read config: %s", err) } - var config config + var config Config if err := yaml.Unmarshal(file, &config); err != nil { return nil, fmt.Errorf("failed to unmarshal YAML: %s", err) } @@ -127,7 +134,7 @@ func main() { go OnKill(c, beforeExit) var err error - if configInstance, err = LoadConfig(*configFilePath); err != nil { + if config, err = LoadConfig(*configFilePath); err != nil { log.Fatalf("failed to load config file: %s", err) } @@ -136,13 +143,6 @@ func main() { } } -type config struct { - UserID id.UserID - HomeserverURL string - AccessToken string - Timeout int -} - type trackDesc struct { StreamID string `json:"stream_id"` TrackID string `json:"track_id"` diff --git a/src/matrix.go b/src/matrix.go index fbbadfa..5a9485c 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -29,7 +29,7 @@ import ( const localSessionID = "sfu" func InitMatrix() error { - client, err := mautrix.NewClient(configInstance.HomeserverURL, configInstance.UserID, configInstance.AccessToken) + client, err := mautrix.NewClient(config.HomeserverURL, config.UserID, config.AccessToken) if err != nil { log.Fatal("Failed to create client", err) } @@ -38,14 +38,14 @@ func InitMatrix() error { if err != nil { log.Fatal("Failed to identify SFU user", err) } - if configInstance.UserID != whoami.UserID { - log.Fatalf("Access token is for the wrong user: %s", configInstance.UserID) + if config.UserID != whoami.UserID { + log.Fatalf("Access token is for the wrong user: %s", config.UserID) } log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID focus := new(Focus) - focus.Init(fmt.Sprintf("%s (%s)", configInstance.UserID, client.DeviceID)) + focus.Init(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true From cc822e28d27e5e5bdb6d8bdb7d43bfcd1cc8f003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:17:14 +0200 Subject: [PATCH 095/124] Remove unused field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 3 --- src/main.go | 7 ++----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/call.go b/src/call.go index ba27212..116cb44 100644 --- a/src/call.go +++ b/src/call.go @@ -409,10 +409,7 @@ func (c *Call) SendDataChannelMessage(msg dataChannelMessage) { return } - msg.ConfID = c.Conf.ConfID msg.Metadata = c.Conf.GetRemoteMetadataForDevice(c.DeviceID) - // TODO: Set ID - if msg.Op == "metadata" && len(msg.Metadata) == 0 { return } diff --git a/src/main.go b/src/main.go index 0b9024d..5b34c6b 100644 --- a/src/main.go +++ b/src/main.go @@ -149,11 +149,8 @@ type trackDesc struct { } type dataChannelMessage struct { - Op string `json:"op"` - ID string `json:"id"` - Message string `json:"message,omitempty"` - // XXX: is this even needed? we know which conf a given call is for... - ConfID string `json:"conf_id,omitempty"` + Op string `json:"op"` + Message string `json:"message,omitempty"` Start []trackDesc `json:"start,omitempty"` Stop []trackDesc `json:"stop,omitempty"` SDP string `json:"sdp,omitempty"` From 8e1c60856625df17c76fb29cb1a593c40a7e8c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:39:59 +0200 Subject: [PATCH 096/124] Improve typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 30 +++++++++++++++--------------- src/conference.go | 2 +- src/main.go | 23 +++++++++++++++++------ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/call.go b/src/call.go index 116cb44..60b78e3 100644 --- a/src/call.go +++ b/src/call.go @@ -46,7 +46,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { peerConnection := c.PeerConnection d.OnOpen(func() { - c.SendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + c.SendDataChannelMessage(SFUMessage{Op: SFUOperationMetadata}) }) d.OnError(func(err error) { @@ -58,7 +58,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { log.Fatal("Inbound message is not string") } - msg := &dataChannelMessage{} + msg := &SFUMessage{} if err := json.Unmarshal(m.Data, msg); err != nil { log.Fatalf("%s | failed to unmarshal: %s", c.CallID, err) } @@ -74,7 +74,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { } switch msg.Op { - case "select": + case SFUOperationSelect: if len(msg.Start) == 0 { return } @@ -97,7 +97,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { } } - case "publish": + case SFUOperationPublish: log.Printf("%s | received DC publish", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -114,12 +114,12 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { panic(err) } - c.SendDataChannelMessage(dataChannelMessage{ - Op: "answer", + c.SendDataChannelMessage(SFUMessage{ + Op: SFUOperationAnswer, SDP: offer.SDP, }) - case "unpublish": + case SFUOperationUnpublish: for _, trackDesc := range msg.Stop { log.Printf("%s | unpublishing StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) if removedTracksCount := c.Conf.RemoveTracksFromPeerConnectionsByInfo(LocalTrackInfo{ @@ -145,12 +145,12 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { panic(err) } - c.SendDataChannelMessage(dataChannelMessage{ - Op: "answer", + c.SendDataChannelMessage(SFUMessage{ + Op: SFUOperationAnswer, SDP: offer.SDP, }) - case "answer": + case SFUOperationAnswer: log.Printf("%s | received DC answer", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -158,10 +158,10 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { SDP: msg.SDP, }) - case "alive": + case SFUOperationAlive: c.LastKeepAliveTimestamp = time.Now() - case "metadata": + case SFUOperationMetadata: log.Printf("%s | received DC metadata", c.UserID) c.Conf.SendUpdatedMetadataFromCall(c.CallID) @@ -183,8 +183,8 @@ func (c *Call) NegotiationNeededHandler() { panic(err) } - c.SendDataChannelMessage(dataChannelMessage{ - Op: "offer", + c.SendDataChannelMessage(SFUMessage{ + Op: SFUOperationOffer, SDP: offer.SDP, }) } @@ -404,7 +404,7 @@ func (c *Call) SendToDevice(callType event.Type, content *event.Content) { c.Client.SendToDevice(callType, toDevice) } -func (c *Call) SendDataChannelMessage(msg dataChannelMessage) { +func (c *Call) SendDataChannelMessage(msg SFUMessage) { if c.DataChannel == nil { return } diff --git a/src/conference.go b/src/conference.go index 854dab9..7ffc725 100644 --- a/src/conference.go +++ b/src/conference.go @@ -220,7 +220,7 @@ func (c *Conference) RemoveMetadataByDeviceID(deviceID id.DeviceID) { func (c *Conference) SendUpdatedMetadataFromCall(callID string) { for _, call := range c.Calls.Calls { if call.CallID != callID { - call.SendDataChannelMessage(dataChannelMessage{Op: "metadata"}) + call.SendDataChannelMessage(SFUMessage{Op: SFUOperationMetadata}) } } } diff --git a/src/main.go b/src/main.go index 5b34c6b..21c6493 100644 --- a/src/main.go +++ b/src/main.go @@ -143,16 +143,27 @@ func main() { } } -type trackDesc struct { +type TrackDescription struct { StreamID string `json:"stream_id"` TrackID string `json:"track_id"` } -type dataChannelMessage struct { - Op string `json:"op"` - Message string `json:"message,omitempty"` - Start []trackDesc `json:"start,omitempty"` - Stop []trackDesc `json:"stop,omitempty"` +type SFUMessageOperation string + +const ( + SFUOperationSelect SFUMessageOperation = "select" + SFUOperationMetadata SFUMessageOperation = "metadata" + SFUOperationPublish SFUMessageOperation = "publish" + SFUOperationUnpublish SFUMessageOperation = "unpublish" + SFUOperationAlive SFUMessageOperation = "alive" + SFUOperationOffer SFUMessageOperation = "offer" + SFUOperationAnswer SFUMessageOperation = "answer" +) + +type SFUMessage struct { + Op SFUMessageOperation `json:"op"` + Start []TrackDescription `json:"start,omitempty"` + Stop []TrackDescription `json:"stop,omitempty"` SDP string `json:"sdp,omitempty"` Metadata event.CallSDPStreamMetadata `json:"metadata,omitempty"` } From 59907f5a556ac3f6e7ef232d54144de58edb07dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:40:42 +0200 Subject: [PATCH 097/124] Rename type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.go b/src/main.go index 21c6493..e5e9a60 100644 --- a/src/main.go +++ b/src/main.go @@ -143,7 +143,7 @@ func main() { } } -type TrackDescription struct { +type SFUTrackDescription struct { StreamID string `json:"stream_id"` TrackID string `json:"track_id"` } @@ -162,8 +162,8 @@ const ( type SFUMessage struct { Op SFUMessageOperation `json:"op"` - Start []TrackDescription `json:"start,omitempty"` - Stop []TrackDescription `json:"stop,omitempty"` + Start []SFUTrackDescription `json:"start,omitempty"` + Stop []SFUTrackDescription `json:"stop,omitempty"` SDP string `json:"sdp,omitempty"` Metadata event.CallSDPStreamMetadata `json:"metadata,omitempty"` } From 22b3d81b74402614aee6b337a738b802e82eec49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:42:55 +0200 Subject: [PATCH 098/124] Move types into mautrix-go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 30 +++++++++++++++--------------- src/conference.go | 2 +- src/main.go | 26 -------------------------- 3 files changed, 16 insertions(+), 42 deletions(-) diff --git a/src/call.go b/src/call.go index 60b78e3..10a5101 100644 --- a/src/call.go +++ b/src/call.go @@ -46,7 +46,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { peerConnection := c.PeerConnection d.OnOpen(func() { - c.SendDataChannelMessage(SFUMessage{Op: SFUOperationMetadata}) + c.SendDataChannelMessage(event.SFUMessage{Op: event.SFUOperationMetadata}) }) d.OnError(func(err error) { @@ -58,7 +58,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { log.Fatal("Inbound message is not string") } - msg := &SFUMessage{} + msg := &event.SFUMessage{} if err := json.Unmarshal(m.Data, msg); err != nil { log.Fatalf("%s | failed to unmarshal: %s", c.CallID, err) } @@ -74,7 +74,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { } switch msg.Op { - case SFUOperationSelect: + case event.SFUOperationSelect: if len(msg.Start) == 0 { return } @@ -97,7 +97,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { } } - case SFUOperationPublish: + case event.SFUOperationPublish: log.Printf("%s | received DC publish", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -114,12 +114,12 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { panic(err) } - c.SendDataChannelMessage(SFUMessage{ - Op: SFUOperationAnswer, + c.SendDataChannelMessage(event.SFUMessage{ + Op: event.SFUOperationAnswer, SDP: offer.SDP, }) - case SFUOperationUnpublish: + case event.SFUOperationUnpublish: for _, trackDesc := range msg.Stop { log.Printf("%s | unpublishing StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) if removedTracksCount := c.Conf.RemoveTracksFromPeerConnectionsByInfo(LocalTrackInfo{ @@ -145,12 +145,12 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { panic(err) } - c.SendDataChannelMessage(SFUMessage{ - Op: SFUOperationAnswer, + c.SendDataChannelMessage(event.SFUMessage{ + Op: event.SFUOperationAnswer, SDP: offer.SDP, }) - case SFUOperationAnswer: + case event.SFUOperationAnswer: log.Printf("%s | received DC answer", c.UserID) peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -158,10 +158,10 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { SDP: msg.SDP, }) - case SFUOperationAlive: + case event.SFUOperationAlive: c.LastKeepAliveTimestamp = time.Now() - case SFUOperationMetadata: + case event.SFUOperationMetadata: log.Printf("%s | received DC metadata", c.UserID) c.Conf.SendUpdatedMetadataFromCall(c.CallID) @@ -183,8 +183,8 @@ func (c *Call) NegotiationNeededHandler() { panic(err) } - c.SendDataChannelMessage(SFUMessage{ - Op: SFUOperationOffer, + c.SendDataChannelMessage(event.SFUMessage{ + Op: event.SFUOperationOffer, SDP: offer.SDP, }) } @@ -404,7 +404,7 @@ func (c *Call) SendToDevice(callType event.Type, content *event.Content) { c.Client.SendToDevice(callType, toDevice) } -func (c *Call) SendDataChannelMessage(msg SFUMessage) { +func (c *Call) SendDataChannelMessage(msg event.SFUMessage) { if c.DataChannel == nil { return } diff --git a/src/conference.go b/src/conference.go index 7ffc725..40c3925 100644 --- a/src/conference.go +++ b/src/conference.go @@ -220,7 +220,7 @@ func (c *Conference) RemoveMetadataByDeviceID(deviceID id.DeviceID) { func (c *Conference) SendUpdatedMetadataFromCall(callID string) { for _, call := range c.Calls.Calls { if call.CallID != callID { - call.SendDataChannelMessage(SFUMessage{Op: SFUOperationMetadata}) + call.SendDataChannelMessage(event.SFUMessage{Op: event.SFUOperationMetadata}) } } } diff --git a/src/main.go b/src/main.go index e5e9a60..191775d 100644 --- a/src/main.go +++ b/src/main.go @@ -29,7 +29,6 @@ import ( yaml "gopkg.in/yaml.v3" - "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" _ "net/http/pprof" @@ -142,28 +141,3 @@ func main() { log.Fatalf("failed to init Matrix: %s", err) } } - -type SFUTrackDescription struct { - StreamID string `json:"stream_id"` - TrackID string `json:"track_id"` -} - -type SFUMessageOperation string - -const ( - SFUOperationSelect SFUMessageOperation = "select" - SFUOperationMetadata SFUMessageOperation = "metadata" - SFUOperationPublish SFUMessageOperation = "publish" - SFUOperationUnpublish SFUMessageOperation = "unpublish" - SFUOperationAlive SFUMessageOperation = "alive" - SFUOperationOffer SFUMessageOperation = "offer" - SFUOperationAnswer SFUMessageOperation = "answer" -) - -type SFUMessage struct { - Op SFUMessageOperation `json:"op"` - Start []SFUTrackDescription `json:"start,omitempty"` - Stop []SFUTrackDescription `json:"stop,omitempty"` - SDP string `json:"sdp,omitempty"` - Metadata event.CallSDPStreamMetadata `json:"metadata,omitempty"` -} From ccf28d5a75c4294a7bdea782e26eee8b9d739238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:47:46 +0200 Subject: [PATCH 099/124] Use `ToDevice` prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/matrix.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/matrix.go b/src/matrix.go index 5a9485c..cfdf32b 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -52,21 +52,21 @@ func InitMatrix() error { // add to-device flavours of the call events to mautrix for MSC3401 var ( - CallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} - CallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} - CallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} - CallReject = event.Type{"m.call.reject", event.ToDeviceEventType} - CallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} - CallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} - CallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} + ToDeviceCallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} + ToDeviceCallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} + ToDeviceCallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} + ToDeviceCallReject = event.Type{"m.call.reject", event.ToDeviceEventType} + ToDeviceCallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} + ToDeviceCallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} + ToDeviceCallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} ) - event.TypeMap[CallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) - event.TypeMap[CallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) - event.TypeMap[CallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) - event.TypeMap[CallReject] = reflect.TypeOf(event.CallRejectEventContent{}) - event.TypeMap[CallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) - event.TypeMap[CallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) - event.TypeMap[CallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) + event.TypeMap[ToDeviceCallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) + event.TypeMap[ToDeviceCallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) + event.TypeMap[ToDeviceCallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) + event.TypeMap[ToDeviceCallReject] = reflect.TypeOf(event.CallRejectEventContent{}) + event.TypeMap[ToDeviceCallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) + event.TypeMap[ToDeviceCallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) + event.TypeMap[ToDeviceCallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) // TODO: E2EE @@ -100,7 +100,7 @@ func InitMatrix() error { if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { log.Printf("received non-call to-device event %s", evt.Type.Type) continue - } else if evt.Type.Type != CallCandidates.Type && evt.Type.Type != CallSelectAnswer.Type { + } else if evt.Type.Type != ToDeviceCallCandidates.Type && evt.Type.Type != ToDeviceCallSelectAnswer.Type { log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) } @@ -110,7 +110,7 @@ func InitMatrix() error { } switch evt.Type.Type { - case CallInvite.Type: + case ToDeviceCallInvite.Type: invite := evt.Content.AsCallInvite() if conf, err = focus.GetConf(invite.ConfID, true); err != nil || conf == nil { log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) @@ -131,19 +131,19 @@ func InitMatrix() error { call.RemoteSessionID = invite.SenderSessionID call.Client = client call.OnInvite(invite) - case CallCandidates.Type: + case ToDeviceCallCandidates.Type: candidates := evt.Content.AsCallCandidates() if call, err = getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil || call == nil { return true } call.OnCandidates(candidates) - case CallSelectAnswer.Type: + case ToDeviceCallSelectAnswer.Type: selectAnswer := evt.Content.AsCallSelectAnswer() if call, err = getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil || call == nil { return true } call.OnSelectAnswer(selectAnswer) - case CallHangup.Type: + case ToDeviceCallHangup.Type: hangup := evt.Content.AsCallHangup() if call, err = getExistingCall(hangup.ConfID, hangup.CallID); err != nil || call == nil { return true @@ -151,10 +151,10 @@ func InitMatrix() error { call.OnHangup(hangup) // Events we don't care about - case CallNegotiate.Type: + case ToDeviceCallNegotiate.Type: log.Printf("%s | ignoring event %s as should be handled over DC", evt.Sender.String(), evt.Type.Type) - case CallReject.Type: - case CallAnswer.Type: + case ToDeviceCallReject.Type: + case ToDeviceCallAnswer.Type: log.Printf("%s | ignoring event %s as we are always the ones answering", evt.Sender.String(), evt.Type.Type) default: log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Sender.String(), evt.Type.Type) From b528c037399ce8151b69e9b2b6ce77fd717fa8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 17:51:04 +0200 Subject: [PATCH 100/124] Move types to mautrix-go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/matrix.go | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/src/matrix.go b/src/matrix.go index cfdf32b..01417ab 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -19,7 +19,6 @@ package main import ( "fmt" "log" - "reflect" "strings" "maunium.net/go/mautrix" @@ -50,24 +49,6 @@ func InitMatrix() error { syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true - // add to-device flavours of the call events to mautrix for MSC3401 - var ( - ToDeviceCallInvite = event.Type{"m.call.invite", event.ToDeviceEventType} - ToDeviceCallCandidates = event.Type{"m.call.candidates", event.ToDeviceEventType} - ToDeviceCallAnswer = event.Type{"m.call.answer", event.ToDeviceEventType} - ToDeviceCallReject = event.Type{"m.call.reject", event.ToDeviceEventType} - ToDeviceCallSelectAnswer = event.Type{"m.call.select_answer", event.ToDeviceEventType} - ToDeviceCallNegotiate = event.Type{"m.call.negotiate", event.ToDeviceEventType} - ToDeviceCallHangup = event.Type{"m.call.hangup", event.ToDeviceEventType} - ) - event.TypeMap[ToDeviceCallInvite] = reflect.TypeOf(event.CallInviteEventContent{}) - event.TypeMap[ToDeviceCallCandidates] = reflect.TypeOf(event.CallCandidatesEventContent{}) - event.TypeMap[ToDeviceCallAnswer] = reflect.TypeOf(event.CallAnswerEventContent{}) - event.TypeMap[ToDeviceCallReject] = reflect.TypeOf(event.CallRejectEventContent{}) - event.TypeMap[ToDeviceCallSelectAnswer] = reflect.TypeOf(event.CallSelectAnswerEventContent{}) - event.TypeMap[ToDeviceCallNegotiate] = reflect.TypeOf(event.CallNegotiateEventContent{}) - event.TypeMap[ToDeviceCallHangup] = reflect.TypeOf(event.CallHangupEventContent{}) - // TODO: E2EE getExistingCall := func(confID string, callID string) (*Call, error) { @@ -100,7 +81,7 @@ func InitMatrix() error { if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { log.Printf("received non-call to-device event %s", evt.Type.Type) continue - } else if evt.Type.Type != ToDeviceCallCandidates.Type && evt.Type.Type != ToDeviceCallSelectAnswer.Type { + } else if evt.Type.Type != event.ToDeviceCallCandidates.Type && evt.Type.Type != event.ToDeviceCallSelectAnswer.Type { log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) } @@ -110,7 +91,7 @@ func InitMatrix() error { } switch evt.Type.Type { - case ToDeviceCallInvite.Type: + case event.ToDeviceCallInvite.Type: invite := evt.Content.AsCallInvite() if conf, err = focus.GetConf(invite.ConfID, true); err != nil || conf == nil { log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) @@ -131,19 +112,19 @@ func InitMatrix() error { call.RemoteSessionID = invite.SenderSessionID call.Client = client call.OnInvite(invite) - case ToDeviceCallCandidates.Type: + case event.ToDeviceCallCandidates.Type: candidates := evt.Content.AsCallCandidates() if call, err = getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil || call == nil { return true } call.OnCandidates(candidates) - case ToDeviceCallSelectAnswer.Type: + case event.ToDeviceCallSelectAnswer.Type: selectAnswer := evt.Content.AsCallSelectAnswer() if call, err = getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil || call == nil { return true } call.OnSelectAnswer(selectAnswer) - case ToDeviceCallHangup.Type: + case event.ToDeviceCallHangup.Type: hangup := evt.Content.AsCallHangup() if call, err = getExistingCall(hangup.ConfID, hangup.CallID); err != nil || call == nil { return true @@ -151,10 +132,10 @@ func InitMatrix() error { call.OnHangup(hangup) // Events we don't care about - case ToDeviceCallNegotiate.Type: + case event.ToDeviceCallNegotiate.Type: log.Printf("%s | ignoring event %s as should be handled over DC", evt.Sender.String(), evt.Type.Type) - case ToDeviceCallReject.Type: - case ToDeviceCallAnswer.Type: + case event.ToDeviceCallReject.Type: + case event.ToDeviceCallAnswer.Type: log.Printf("%s | ignoring event %s as we are always the ones answering", evt.Sender.String(), evt.Type.Type) default: log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Sender.String(), evt.Type.Type) From 250e2c913fd0166d30bad43a37c39be3fc9b76da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 18:08:49 +0200 Subject: [PATCH 101/124] Remove comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/call.go b/src/call.go index 10a5101..156f121 100644 --- a/src/call.go +++ b/src/call.go @@ -196,7 +196,6 @@ func (c *Call) IceCandidateHandler(candidate *webrtc.ICECandidate) { ice := candidate.ToJSON() - // TODO: batch these up a bit candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ BaseCallEventContent: event.BaseCallEventContent{ @@ -231,7 +230,8 @@ func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece c.Conf.Tracks.Mutex.Lock() c.Conf.Tracks.Tracks = append(c.Conf.Tracks.Tracks, LocalTrackWithInfo{ - Track: trackLocal, + Track: trackLocal, + TrackRemote: trackRemote, Info: LocalTrackInfo{ TrackID: trackLocal.ID(), StreamID: trackLocal.StreamID(), From 839ae57244afa5c485178c032752698aca043f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 13 Aug 2022 18:11:22 +0200 Subject: [PATCH 102/124] Remove type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/call.go b/src/call.go index 156f121..0f397e4 100644 --- a/src/call.go +++ b/src/call.go @@ -230,8 +230,7 @@ func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece c.Conf.Tracks.Mutex.Lock() c.Conf.Tracks.Tracks = append(c.Conf.Tracks.Tracks, LocalTrackWithInfo{ - Track: trackLocal, - TrackRemote: trackRemote, + Track: trackLocal, Info: LocalTrackInfo{ TrackID: trackLocal.ID(), StreamID: trackLocal.StreamID(), From e82cc39d32a9596d86a6f7392e92766f6bb8f113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 13:26:07 +0200 Subject: [PATCH 103/124] Improve logging in utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/utils.go b/src/utils.go index d68b871..e786ac5 100644 --- a/src/utils.go +++ b/src/utils.go @@ -17,6 +17,8 @@ limitations under the License. package main import ( + "errors" + "io" "log" "strings" "time" @@ -29,13 +31,17 @@ func CopyRemoteToLocal(trackRemote *webrtc.TrackRemote, trackLocal *webrtc.Track buff := make([]byte, 1500) for { i, _, err := trackRemote.Read(buff) - if err != nil || buff == nil { - log.Printf("ending read on TrackID %s: %s", trackRemote.ID(), err) + if err != nil { + if !errors.Is(err, io.EOF) { + log.Printf("failed read on StreamID %s TrackID %s: %s", trackLocal.StreamID(), trackRemote.ID(), err) + } break } if _, err = trackLocal.Write(buff[:i]); err != nil { - log.Printf("ending write on TrackID %s: %s", trackLocal.ID(), err) + if !errors.Is(err, io.ErrClosedPipe) { + log.Printf("failed write on StreamID %s TrackID %s: %s", trackLocal.StreamID(), trackLocal.ID(), err) + } break } } @@ -46,11 +52,15 @@ func WriteRTCP(trackRemote *webrtc.TrackRemote, peerConnection *webrtc.PeerConne return } - // FIXME: This is a potential performance killer - // Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval + // FIXME: This is a potential performance killer. This can be less wasteful + // by processing incoming RTCP events, then we would emit a NACK/PLI when a + // viewer requests it + // Send a PLI on an interval so that the publisher is pushing a keyframe + // every 200ms ticker := time.NewTicker(time.Millisecond * 200) for range ticker.C { - if err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}); err != nil { + err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}) + if err != nil && !errors.Is(err, io.ErrClosedPipe) { log.Printf("ending RTCP write on TrackID %s: %s", trackRemote.ID(), err) break } From 71c2a0f3b72926c6e2cb5b5cfc5b064af290b5ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 13:28:36 +0200 Subject: [PATCH 104/124] Don't timeout already ended calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/call.go b/src/call.go index 0f397e4..3576685 100644 --- a/src/call.go +++ b/src/call.go @@ -430,8 +430,10 @@ func (c *Call) CheckKeepAliveTimestamp() { timeout := time.Second * time.Duration(config.Timeout) for range time.Tick(timeout) { if c.LastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { - log.Printf("%s | did not get keep-alive message in the last %s:", c.UserID, timeout) - c.Hangup(event.CallHangupKeepAliveTimeout) + if c.PeerConnection.ConnectionState() != webrtc.PeerConnectionStateClosed { + log.Printf("%s | did not get keep-alive message in the last %s:", c.UserID, timeout) + c.Hangup(event.CallHangupKeepAliveTimeout) + } break } } From 9abb6d9fe020706d1b53f8b158529bee71a58487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 13:35:32 +0200 Subject: [PATCH 105/124] Break out of `WriteRTCP` loop on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/utils.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils.go b/src/utils.go index e786ac5..b131f98 100644 --- a/src/utils.go +++ b/src/utils.go @@ -60,8 +60,10 @@ func WriteRTCP(trackRemote *webrtc.TrackRemote, peerConnection *webrtc.PeerConne ticker := time.NewTicker(time.Millisecond * 200) for range ticker.C { err := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(trackRemote.SSRC())}}) - if err != nil && !errors.Is(err, io.ErrClosedPipe) { - log.Printf("ending RTCP write on TrackID %s: %s", trackRemote.ID(), err) + if err != nil { + if !errors.Is(err, io.ErrClosedPipe) { + log.Printf("ending RTCP write on TrackID %s: %s", trackRemote.ID(), err) + } break } } From 0bb1871de382cec67ecec3a2291885fe56bcf9f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 14:39:16 +0200 Subject: [PATCH 106/124] Add somu mutex locks and unlocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/conference.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/conference.go b/src/conference.go index 40c3925..d18cf5d 100644 --- a/src/conference.go +++ b/src/conference.go @@ -78,6 +78,9 @@ func (c *Conference) GetCall(callID string, create bool) (*Call, error) { } func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (tracks []int) { + c.Tracks.Mutex.Lock() + defer c.Tracks.Mutex.Unlock() + foundIndices := []int{} for index, track := range c.Tracks.Tracks { info := track.Info @@ -97,6 +100,9 @@ func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (trac } func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []webrtc.TrackLocal) { + c.Tracks.Mutex.Lock() + defer c.Tracks.Mutex.Unlock() + indices := c.GetLocalTrackIndicesByInfo(selectInfo) foundTracks := []webrtc.TrackLocal{} for _, index := range indices { @@ -107,6 +113,9 @@ func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []we } func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrackInfo) int { + c.Tracks.Mutex.Lock() + defer c.Tracks.Mutex.Unlock() + indices := c.GetLocalTrackIndicesByInfo(removeInfo) // FIXME: the big O of this must be awful... @@ -169,6 +178,8 @@ func (c *Conference) RemoveOldCallsByDeviceAndSessionIDs(deviceID id.DeviceID, s func (c *Conference) UpdateSDPStreamMetadata(deviceID id.DeviceID, metadata event.CallSDPStreamMetadata) { c.Metadata.Mutex.Lock() + defer c.Metadata.Mutex.Unlock() + // Update existing and add new for streamID, info := range metadata { c.Metadata.Metadata[streamID] = info @@ -180,7 +191,6 @@ func (c *Conference) UpdateSDPStreamMetadata(deviceID id.DeviceID, metadata even delete(c.Metadata.Metadata, streamID) } } - c.Metadata.Mutex.Unlock() } func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.CallSDPStreamMetadata { @@ -210,6 +220,9 @@ func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.Call } func (c *Conference) RemoveMetadataByDeviceID(deviceID id.DeviceID) { + c.Metadata.Mutex.Lock() + defer c.Metadata.Mutex.Unlock() + for streamID, info := range c.Metadata.Metadata { if info.DeviceID == deviceID { delete(c.Metadata.Metadata, streamID) From 5c45ebfbb394357e23a22a7f93fdd6c667edb1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 14:53:38 +0200 Subject: [PATCH 107/124] Fix incorrect mutex usege MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/conference.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/conference.go b/src/conference.go index d18cf5d..d0a96e2 100644 --- a/src/conference.go +++ b/src/conference.go @@ -100,10 +100,11 @@ func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (trac } func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []webrtc.TrackLocal) { + indices := c.GetLocalTrackIndicesByInfo(selectInfo) + c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() - indices := c.GetLocalTrackIndicesByInfo(selectInfo) foundTracks := []webrtc.TrackLocal{} for _, index := range indices { foundTracks = append(foundTracks, c.Tracks.Tracks[index].Track) @@ -113,11 +114,11 @@ func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []we } func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrackInfo) int { + indices := c.GetLocalTrackIndicesByInfo(removeInfo) + c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() - indices := c.GetLocalTrackIndicesByInfo(removeInfo) - // FIXME: the big O of this must be awful... for _, index := range indices { info := c.Tracks.Tracks[index].Info @@ -141,11 +142,11 @@ func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrack } func (c *Conference) RemoveTracksFromConfByInfo(removeInfo LocalTrackInfo) { + indicesToRemove := c.GetLocalTrackIndicesByInfo(removeInfo) + c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() - indicesToRemove := c.GetLocalTrackIndicesByInfo(removeInfo) - newTracks := []LocalTrackWithInfo{} for index, track := range c.Tracks.Tracks { keep := true From 8d4558153aa9f7ca54e6950a8e4bc43c3ceade2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 14 Aug 2022 14:59:00 +0200 Subject: [PATCH 108/124] Fixup same small issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/call.go b/src/call.go index 3576685..f496f72 100644 --- a/src/call.go +++ b/src/call.go @@ -100,10 +100,13 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { case event.SFUOperationPublish: log.Printf("%s | received DC publish", c.UserID) - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: msg.SDP, }) + if err != nil { + panic(err) + } offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { @@ -131,10 +134,13 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { } - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: msg.SDP, }) + if err != nil { + panic(err) + } offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { @@ -153,10 +159,13 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { case event.SFUOperationAnswer: log.Printf("%s | received DC answer", c.UserID) - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeAnswer, SDP: msg.SDP, }) + if err != nil { + panic(err) + } case event.SFUOperationAlive: c.LastKeepAliveTimestamp = time.Now() @@ -252,13 +261,13 @@ func (c *Call) IceConnectionStateHandler(state webrtc.ICEConnectionState) { } } -func (c *Call) OnInvite(content *event.CallInviteEventContent) error { +func (c *Call) OnInvite(content *event.CallInviteEventContent) { c.Conf.UpdateSDPStreamMetadata(c.DeviceID, content.SDPStreamMetadata) offer := content.Offer peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) if err != nil { - return err + panic(err) } c.PeerConnection = peerConnection @@ -278,25 +287,27 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) error { c.IceConnectionStateHandler(state) }) - peerConnection.SetRemoteDescription(webrtc.SessionDescription{ + err = peerConnection.SetRemoteDescription(webrtc.SessionDescription{ Type: webrtc.SDPTypeOffer, SDP: offer.SDP, }) + if err != nil { + panic(err) + + } answer, err := peerConnection.CreateAnswer(nil) if err != nil { - return err + panic(err) } // TODO: trickle ICE for fast conn setup, rather than block here gatherComplete := webrtc.GatheringCompletePromise(peerConnection) if err = peerConnection.SetLocalDescription(answer); err != nil { - return err + panic(err) } <-gatherComplete - answerSdp := peerConnection.LocalDescription().SDP - answerEvtContent := &event.Content{ Parsed: event.CallAnswerEventContent{ BaseCallEventContent: event.BaseCallEventContent{ @@ -310,21 +321,19 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) error { }, Answer: event.CallData{ Type: "answer", - SDP: answerSdp, + SDP: peerConnection.LocalDescription().SDP, }, SDPStreamMetadata: c.Conf.GetRemoteMetadataForDevice(c.DeviceID), }, } c.SendToDevice(event.CallAnswer, answerEvtContent) - - return err } func (c *Call) OnSelectAnswer(content *event.CallSelectAnswerEventContent) { selectedPartyID := content.SelectedPartyID if selectedPartyID != string(c.Client.DeviceID) { c.Terminate() - log.Printf("%s | Call was answered on a different device: %s", c.UserID, selectedPartyID) + log.Printf("%s | call was answered on a different device: %s", c.UserID, selectedPartyID) } } @@ -332,7 +341,7 @@ func (c *Call) OnHangup(content *event.CallHangupEventContent) { c.Terminate() } -func (c *Call) OnCandidates(content *event.CallCandidatesEventContent) error { +func (c *Call) OnCandidates(content *event.CallCandidatesEventContent) { for _, candidate := range content.Candidates { sdpMLineIndex := uint16(candidate.SDPMLineIndex) ice := webrtc.ICECandidateInit{ @@ -343,10 +352,8 @@ func (c *Call) OnCandidates(content *event.CallCandidatesEventContent) error { } if err := c.PeerConnection.AddICECandidate(ice); err != nil { log.Print("Failed to add ICE candidate", content) - return err } } - return nil } func (c *Call) Terminate() { From f5282257e98d80a24f42a671376b6c5936f52744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 13:49:05 +0200 Subject: [PATCH 109/124] =?UTF-8?q?Don't=20assume=20everyone=20is=20=C5=A0?= =?UTF-8?q?imon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- scripts/profile.sh | 2 +- scripts/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/profile.sh b/scripts/profile.sh index bb106cf..071103e 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof --logTime +go run ./src/*.go --cpuProfile cpuProfile.pprof --memProfile memProfile.pprof --logTime diff --git a/scripts/run.sh b/scripts/run.sh index 5c721ee..bec8362 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -clear && go run ./src/*.go --logTime +go run ./src/*.go --logTime From 4bf1c556f293969960ca7e6f50f3ac37fbbcac1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 13:54:28 +0200 Subject: [PATCH 110/124] Don't kill the SFU when receiving invalid messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/call.go b/src/call.go index f496f72..1669cca 100644 --- a/src/call.go +++ b/src/call.go @@ -55,12 +55,14 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { d.OnMessage(func(m webrtc.DataChannelMessage) { if !m.IsString { - log.Fatal("Inbound message is not string") + log.Printf("%s | inbound message is not string - ignoring: %+v", c.UserID, m) + return } msg := &event.SFUMessage{} if err := json.Unmarshal(m.Data, msg); err != nil { - log.Fatalf("%s | failed to unmarshal: %s", c.CallID, err) + log.Printf("%s | failed to unmarshal %+v - ignoring: %s", c.CallID, msg, err) + return } // TODO: hook cascade back up. @@ -176,7 +178,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { c.Conf.SendUpdatedMetadataFromCall(c.CallID) default: - log.Fatalf("Unknown operation %s", msg.Op) + log.Printf("Unknown operation - ignoring: %s", msg.Op) // TODO: hook up msg.Stop to unsubscribe from tracks } }) From f0cfb9eb0b192099edad3d2d639b5a86d733e99a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 13:57:59 +0200 Subject: [PATCH 111/124] Improve log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/call.go b/src/call.go index 1669cca..814efd1 100644 --- a/src/call.go +++ b/src/call.go @@ -353,7 +353,7 @@ func (c *Call) OnCandidates(content *event.CallCandidatesEventContent) { UsernameFragment: new(string), } if err := c.PeerConnection.AddICECandidate(ice); err != nil { - log.Print("Failed to add ICE candidate", content) + log.Printf("%s | failed to add ICE candidate %+v: %s", c.UserID, content, err) } } } From 1e127f04a7b9f31a9c728190d144b08521336271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 14:13:53 +0200 Subject: [PATCH 112/124] Separate `DataChannelHandler()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 220 ++++++++++++++++++++++++++++------------------------ 1 file changed, 119 insertions(+), 101 deletions(-) diff --git a/src/call.go b/src/call.go index 814efd1..24875db 100644 --- a/src/call.go +++ b/src/call.go @@ -41,9 +41,116 @@ type Call struct { LastKeepAliveTimestamp time.Time } +func (c *Call) onDCSelect(start []event.SFUTrackDescription) { + if len(start) == 0 { + return + } + + for _, trackDesc := range start { + log.Printf("%s | selecting StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) + foundTracks := c.Conf.GetLocalTrackByInfo(LocalTrackInfo{ + StreamID: trackDesc.StreamID, + TrackID: trackDesc.TrackID, + }) + if len(foundTracks) == 0 { + log.Printf("%s | no track found StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) + continue + } + for _, track := range foundTracks { + log.Printf("%s | adding %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) + if _, err := c.PeerConnection.AddTrack(track); err != nil { + panic(err) + } + } + } +} + +func (c *Call) onDCPublish(sdp string) { + log.Printf("%s | received DC publish", c.UserID) + + err := c.PeerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: sdp, + }) + if err != nil { + panic(err) + } + + offer, err := c.PeerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + err = c.PeerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + c.SendDataChannelMessage(event.SFUMessage{ + Op: event.SFUOperationAnswer, + SDP: offer.SDP, + }) +} + +func (c *Call) onDCUnpublish(stop []event.SFUTrackDescription, sdp string) { + for _, trackDesc := range stop { + log.Printf("%s | unpublishing StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) + if removedTracksCount := c.Conf.RemoveTracksFromPeerConnectionsByInfo(LocalTrackInfo{ + StreamID: trackDesc.StreamID, + TrackID: trackDesc.TrackID, + }); removedTracksCount == 0 { + log.Printf("%s | no tracks to remove for: %+v", c.UserID, stop) + } + + } + + err := c.PeerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: sdp, + }) + if err != nil { + panic(err) + } + + offer, err := c.PeerConnection.CreateAnswer(nil) + if err != nil { + panic(err) + } + err = c.PeerConnection.SetLocalDescription(offer) + if err != nil { + panic(err) + } + + c.SendDataChannelMessage(event.SFUMessage{ + Op: event.SFUOperationAnswer, + SDP: offer.SDP, + }) +} + +func (c *Call) onDCAnswer(sdp string) { + log.Printf("%s | received DC answer", c.UserID) + + err := c.PeerConnection.SetRemoteDescription(webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: sdp, + }) + if err != nil { + panic(err) + } +} + +func (c *Call) onDCAlive() { + c.LastKeepAliveTimestamp = time.Now() + +} + +func (c *Call) onDCMetadata(metadata event.CallSDPStreamMetadata) { + log.Printf("%s | received DC metadata", c.UserID) + + c.Conf.SendUpdatedMetadataFromCall(c.CallID) +} + func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { c.DataChannel = d - peerConnection := c.PeerConnection d.OnOpen(func() { c.SendDataChannelMessage(event.SFUMessage{Op: event.SFUOperationMetadata}) @@ -65,121 +172,32 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { return } - // TODO: hook cascade back up. - // As we're not an AS, we'd rely on the client - // to send us a "connect" op to tell us how to - // connect to another focus in order to select - // its streams. - if msg.Metadata != nil { c.Conf.UpdateSDPStreamMetadata(c.DeviceID, msg.Metadata) } switch msg.Op { case event.SFUOperationSelect: - if len(msg.Start) == 0 { - return - } - - for _, trackDesc := range msg.Start { - log.Printf("%s | selecting StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) - foundTracks := c.Conf.GetLocalTrackByInfo(LocalTrackInfo{ - StreamID: trackDesc.StreamID, - TrackID: trackDesc.TrackID, - }) - if len(foundTracks) == 0 { - log.Printf("%s | no track found StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) - continue - } - for _, track := range foundTracks { - log.Printf("%s | adding %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) - if _, err := c.PeerConnection.AddTrack(track); err != nil { - panic(err) - } - } - } - + c.onDCSelect(msg.Start) case event.SFUOperationPublish: - log.Printf("%s | received DC publish", c.UserID) - - err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }) - if err != nil { - panic(err) - } - - offer, err := c.PeerConnection.CreateAnswer(nil) - if err != nil { - panic(err) - } - err = c.PeerConnection.SetLocalDescription(offer) - if err != nil { - panic(err) - } - - c.SendDataChannelMessage(event.SFUMessage{ - Op: event.SFUOperationAnswer, - SDP: offer.SDP, - }) - + c.onDCPublish(msg.SDP) case event.SFUOperationUnpublish: - for _, trackDesc := range msg.Stop { - log.Printf("%s | unpublishing StreamID %s TrackID %s", c.UserID, trackDesc.StreamID, trackDesc.TrackID) - if removedTracksCount := c.Conf.RemoveTracksFromPeerConnectionsByInfo(LocalTrackInfo{ - StreamID: trackDesc.StreamID, - TrackID: trackDesc.TrackID, - }); removedTracksCount == 0 { - log.Printf("%s | no tracks to remove for: %+v", c.UserID, msg.Stop) - } - - } - - err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeOffer, - SDP: msg.SDP, - }) - if err != nil { - panic(err) - } - - offer, err := c.PeerConnection.CreateAnswer(nil) - if err != nil { - panic(err) - } - err = c.PeerConnection.SetLocalDescription(offer) - if err != nil { - panic(err) - } - - c.SendDataChannelMessage(event.SFUMessage{ - Op: event.SFUOperationAnswer, - SDP: offer.SDP, - }) - + c.onDCUnpublish(msg.Stop, msg.SDP) case event.SFUOperationAnswer: - log.Printf("%s | received DC answer", c.UserID) - - err := peerConnection.SetRemoteDescription(webrtc.SessionDescription{ - Type: webrtc.SDPTypeAnswer, - SDP: msg.SDP, - }) - if err != nil { - panic(err) - } - + c.onDCAnswer(msg.SDP) case event.SFUOperationAlive: - c.LastKeepAliveTimestamp = time.Now() - + c.onDCAlive() case event.SFUOperationMetadata: - log.Printf("%s | received DC metadata", c.UserID) - - c.Conf.SendUpdatedMetadataFromCall(c.CallID) + c.onDCMetadata(msg.Metadata) default: log.Printf("Unknown operation - ignoring: %s", msg.Op) // TODO: hook up msg.Stop to unsubscribe from tracks + // TODO: hook cascade back up. + // As we're not an AS, we'd rely on the client + // to send us a "connect" op to tell us how to + // connect to another focus in order to select + // its streams. } }) } From 80be219d1f37c0ba36b14a521c6b9b1b6cb1aad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 14:25:03 +0200 Subject: [PATCH 113/124] Don't panic where we don't need to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 52 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/call.go b/src/call.go index 24875db..87ffeca 100644 --- a/src/call.go +++ b/src/call.go @@ -57,9 +57,10 @@ func (c *Call) onDCSelect(start []event.SFUTrackDescription) { continue } for _, track := range foundTracks { - log.Printf("%s | adding %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) - if _, err := c.PeerConnection.AddTrack(track); err != nil { - panic(err) + if _, err := c.PeerConnection.AddTrack(track); err == nil { + log.Printf("%s | added %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) + } else { + log.Printf("%s | failed to add %s StreamID %s TrackID %s", c.UserID, track.Kind(), track.StreamID(), track.ID()) } } } @@ -73,16 +74,19 @@ func (c *Call) onDCPublish(sdp string) { SDP: sdp, }) if err != nil { - panic(err) + log.Printf("%s | failed to set remote description %+v - ignoring: %s", c.UserID, sdp, err) + return } offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { - panic(err) + log.Printf("%s | failed to create answer - ignoring: %s", c.UserID, err) + return } err = c.PeerConnection.SetLocalDescription(offer) if err != nil { - panic(err) + log.Printf("%s | failed to set local description %+v - ignoring: %s", c.UserID, offer.SDP, err) + return } c.SendDataChannelMessage(event.SFUMessage{ @@ -108,16 +112,19 @@ func (c *Call) onDCUnpublish(stop []event.SFUTrackDescription, sdp string) { SDP: sdp, }) if err != nil { - panic(err) + log.Printf("%s | failed to set remote description %+v - ignoring: %s", c.UserID, sdp, err) + return } offer, err := c.PeerConnection.CreateAnswer(nil) if err != nil { - panic(err) + log.Printf("%s | failed to create answer - ignoring: %s", c.UserID, err) + return } err = c.PeerConnection.SetLocalDescription(offer) if err != nil { - panic(err) + log.Printf("%s | failed to set local description %+v - ignoring: %s", c.UserID, offer.SDP, err) + return } c.SendDataChannelMessage(event.SFUMessage{ @@ -134,7 +141,8 @@ func (c *Call) onDCAnswer(sdp string) { SDP: sdp, }) if err != nil { - panic(err) + log.Printf("%s | failed to set remote description %+v - ignoring: %s", c.UserID, sdp, err) + return } } @@ -205,11 +213,13 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { func (c *Call) NegotiationNeededHandler() { offer, err := c.PeerConnection.CreateOffer(nil) if err != nil { - panic(err) + log.Printf("%s | failed to create offer - ignoring: %s", c.UserID, err) + return } err = c.PeerConnection.SetLocalDescription(offer) if err != nil { - panic(err) + log.Printf("%s | failed to set local description %+v - ignoring: %s", c.UserID, offer.SDP, err) + return } c.SendDataChannelMessage(event.SFUMessage{ @@ -254,7 +264,8 @@ func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) if err != nil { - panic(err) + log.Printf("%s | failed to create new track local static RTP %+v - ignoring: %s", c.UserID, trackRemote.Codec().RTPCodecCapability, err) + return } c.Conf.Tracks.Mutex.Lock() @@ -287,7 +298,7 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) { peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) if err != nil { - panic(err) + log.Panicf("%s | failed to create new peer connection: %s", c.UserID, err) } c.PeerConnection = peerConnection @@ -312,19 +323,21 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) { SDP: offer.SDP, }) if err != nil { - panic(err) - + log.Printf("%s | failed to set remote description %+v - ignoring: %s", c.UserID, offer.SDP, err) + return } answer, err := peerConnection.CreateAnswer(nil) if err != nil { - panic(err) + log.Printf("%s | failed to create answer - ignoring: %s", c.UserID, err) + return } // TODO: trickle ICE for fast conn setup, rather than block here gatherComplete := webrtc.GatheringCompletePromise(peerConnection) if err = peerConnection.SetLocalDescription(answer); err != nil { - panic(err) + log.Printf("%s | failed to set local description %+v - ignoring: %s", c.UserID, offer.SDP, err) + return } <-gatherComplete @@ -442,7 +455,8 @@ func (c *Call) SendDataChannelMessage(msg event.SFUMessage) { marshaled, err := json.Marshal(msg) if err != nil { - panic(err) + log.Printf("%s | failed to marshal %+v - ignoring: %s", c.UserID, msg, err) + return } err = c.DataChannel.SendText(string(marshaled)) From 6a2bf28252515d0b45ab6596e069ea5c67a6389d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 16:04:24 +0200 Subject: [PATCH 114/124] Comment `GetRemoteMetadataForDevice()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/conference.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/conference.go b/src/conference.go index d0a96e2..9360217 100644 --- a/src/conference.go +++ b/src/conference.go @@ -194,19 +194,27 @@ func (c *Conference) UpdateSDPStreamMetadata(deviceID id.DeviceID, metadata even } } +// Get metadata to send to deviceID. This will not include the device's own +// metadata and metadata which includes tracks which we have not received yet func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.CallSDPStreamMetadata { + // First we copy the metadata metadata := make(event.CallSDPStreamMetadata) c.Metadata.Mutex.Lock() for streamID, info := range c.Metadata.Metadata { metadata[streamID] = info } c.Metadata.Mutex.Unlock() + // Loop over the copied metadata for streamID, info := range metadata { + // Delete metadata received from the device that we're sending metadata to if info.DeviceID == deviceID { delete(metadata, streamID) continue } + // Loop over the tracks in the copied metadata for trackID := range info.Tracks { + // Delete metadata, if we're the client hasn't published a track that is + // included in the metadata yet if len(c.GetLocalTrackIndicesByInfo(LocalTrackInfo{ StreamID: streamID, TrackID: trackID, @@ -216,7 +224,6 @@ func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.Call } } } - return metadata } From a0504d02fbb81f54fe38d3c7ad31bdf848660246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 16:06:19 +0200 Subject: [PATCH 115/124] Rename function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.go b/src/main.go index 191775d..453a800 100644 --- a/src/main.go +++ b/src/main.go @@ -105,7 +105,7 @@ func LoadConfig(configFilePath string) (*Config, error) { return &config, nil } -func OnKill(c chan os.Signal, beforeExit []func()) { +func killListener(c chan os.Signal, beforeExit []func()) { <-c log.Printf("ending program") for _, function := range beforeExit { @@ -130,7 +130,7 @@ func main() { // try to handle os interrupt(signal terminated) c := make(chan os.Signal, 2) signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go OnKill(c, beforeExit) + go killListener(c, beforeExit) var err error if config, err = LoadConfig(*configFilePath); err != nil { From 02bcae817670186a7596956d91e3419b552abffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 16:19:54 +0200 Subject: [PATCH 116/124] Add `NewFocus` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/focus.go | 6 +++++- src/matrix.go | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/focus.go b/src/focus.go index 64ac40d..21816cc 100644 --- a/src/focus.go +++ b/src/focus.go @@ -33,9 +33,13 @@ type Focus struct { confs Confs } -func (f *Focus) Init(name string) { +func NewFocus(name string) *Focus { + f := new(Focus) + f.name = name f.confs.confs = make(map[string]*Conference) + + return f } func (f *Focus) GetConf(confID string, create bool) (*Conference, error) { diff --git a/src/matrix.go b/src/matrix.go index 01417ab..cfb4e57 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -43,8 +43,7 @@ func InitMatrix() error { log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID - focus := new(Focus) - focus.Init(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) + focus := NewFocus(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true From 42ebd11b3bb32692e92ffa4893376543c603bf09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 16:27:24 +0200 Subject: [PATCH 117/124] Fix public/private MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 44 ++++++++++++++++++++++---------------------- src/conference.go | 10 +++++----- src/main.go | 16 ++++++++-------- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/call.go b/src/call.go index 87ffeca..23650a2 100644 --- a/src/call.go +++ b/src/call.go @@ -37,8 +37,8 @@ type Call struct { Client *mautrix.Client PeerConnection *webrtc.PeerConnection Conf *Conference - DataChannel *webrtc.DataChannel - LastKeepAliveTimestamp time.Time + dataChannel *webrtc.DataChannel + lastKeepAliveTimestamp time.Time } func (c *Call) onDCSelect(start []event.SFUTrackDescription) { @@ -147,7 +147,7 @@ func (c *Call) onDCAnswer(sdp string) { } func (c *Call) onDCAlive() { - c.LastKeepAliveTimestamp = time.Now() + c.lastKeepAliveTimestamp = time.Now() } @@ -157,8 +157,8 @@ func (c *Call) onDCMetadata(metadata event.CallSDPStreamMetadata) { c.Conf.SendUpdatedMetadataFromCall(c.CallID) } -func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { - c.DataChannel = d +func (c *Call) dataChannelHandler(d *webrtc.DataChannel) { + c.dataChannel = d d.OnOpen(func() { c.SendDataChannelMessage(event.SFUMessage{Op: event.SFUOperationMetadata}) @@ -210,7 +210,7 @@ func (c *Call) DataChannelHandler(d *webrtc.DataChannel) { }) } -func (c *Call) NegotiationNeededHandler() { +func (c *Call) negotiationNeededHandler() { offer, err := c.PeerConnection.CreateOffer(nil) if err != nil { log.Printf("%s | failed to create offer - ignoring: %s", c.UserID, err) @@ -228,7 +228,7 @@ func (c *Call) NegotiationNeededHandler() { }) } -func (c *Call) IceCandidateHandler(candidate *webrtc.ICECandidate) { +func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { if candidate == nil { return } @@ -256,10 +256,10 @@ func (c *Call) IceCandidateHandler(candidate *webrtc.ICECandidate) { }, }, } - c.SendToDevice(event.CallCandidates, candidateEvtContent) + c.sendToDevice(event.CallCandidates, candidateEvtContent) } -func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { +func (c *Call) trackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPReceiver) { go WriteRTCP(trackRemote, c.PeerConnection) trackLocal, err := webrtc.NewTrackLocalStaticRTP(trackRemote.Codec().RTPCodecCapability, trackRemote.ID(), trackRemote.StreamID()) @@ -285,9 +285,9 @@ func (c *Call) TrackHandler(trackRemote *webrtc.TrackRemote, rec *webrtc.RTPRece go CopyRemoteToLocal(trackRemote, trackLocal) } -func (c *Call) IceConnectionStateHandler(state webrtc.ICEConnectionState) { +func (c *Call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { if state == webrtc.ICEConnectionStateCompleted || state == webrtc.ICEConnectionStateConnected { - c.LastKeepAliveTimestamp = time.Now() + c.lastKeepAliveTimestamp = time.Now() go c.CheckKeepAliveTimestamp() } } @@ -303,19 +303,19 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) { c.PeerConnection = peerConnection peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { - c.TrackHandler(track, receiver) + c.trackHandler(track, receiver) }) peerConnection.OnDataChannel(func(d *webrtc.DataChannel) { - c.DataChannelHandler(d) + c.dataChannelHandler(d) }) peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) { - c.IceCandidateHandler(candidate) + c.iceCandidateHandler(candidate) }) peerConnection.OnNegotiationNeeded(func() { - c.NegotiationNeededHandler() + c.negotiationNeededHandler() }) peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) { - c.IceConnectionStateHandler(state) + c.iceConnectionStateHandler(state) }) err = peerConnection.SetRemoteDescription(webrtc.SessionDescription{ @@ -359,7 +359,7 @@ func (c *Call) OnInvite(content *event.CallInviteEventContent) { SDPStreamMetadata: c.Conf.GetRemoteMetadataForDevice(c.DeviceID), }, } - c.SendToDevice(event.CallAnswer, answerEvtContent) + c.sendToDevice(event.CallAnswer, answerEvtContent) } func (c *Call) OnSelectAnswer(content *event.CallSelectAnswerEventContent) { @@ -422,11 +422,11 @@ func (c *Call) Hangup(reason event.CallHangupReason) { Reason: reason, }, } - c.SendToDevice(event.CallHangup, hangupEvtContent) + c.sendToDevice(event.CallHangup, hangupEvtContent) c.Terminate() } -func (c *Call) SendToDevice(callType event.Type, content *event.Content) { +func (c *Call) sendToDevice(callType event.Type, content *event.Content) { if callType.Type != event.CallCandidates.Type { log.Printf("%s | sending to device %s", c.UserID, callType.Type) } @@ -444,7 +444,7 @@ func (c *Call) SendToDevice(callType event.Type, content *event.Content) { } func (c *Call) SendDataChannelMessage(msg event.SFUMessage) { - if c.DataChannel == nil { + if c.dataChannel == nil { return } @@ -459,7 +459,7 @@ func (c *Call) SendDataChannelMessage(msg event.SFUMessage) { return } - err = c.DataChannel.SendText(string(marshaled)) + err = c.dataChannel.SendText(string(marshaled)) if err != nil { log.Printf("%s | failed to send %s over DC: %s", c.UserID, msg.Op, err) } @@ -470,7 +470,7 @@ func (c *Call) SendDataChannelMessage(msg event.SFUMessage) { func (c *Call) CheckKeepAliveTimestamp() { timeout := time.Second * time.Duration(config.Timeout) for range time.Tick(timeout) { - if c.LastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { + if c.lastKeepAliveTimestamp.Add(timeout).Before(time.Now()) { if c.PeerConnection.ConnectionState() != webrtc.PeerConnectionStateClosed { log.Printf("%s | did not get keep-alive message in the last %s:", c.UserID, timeout) c.Hangup(event.CallHangupKeepAliveTimeout) diff --git a/src/conference.go b/src/conference.go index 9360217..26f7621 100644 --- a/src/conference.go +++ b/src/conference.go @@ -77,7 +77,7 @@ func (c *Conference) GetCall(callID string, create bool) (*Call, error) { return ca, nil } -func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (tracks []int) { +func (c *Conference) getLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (tracks []int) { c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() @@ -100,7 +100,7 @@ func (c *Conference) GetLocalTrackIndicesByInfo(selectInfo LocalTrackInfo) (trac } func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []webrtc.TrackLocal) { - indices := c.GetLocalTrackIndicesByInfo(selectInfo) + indices := c.getLocalTrackIndicesByInfo(selectInfo) c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() @@ -114,7 +114,7 @@ func (c *Conference) GetLocalTrackByInfo(selectInfo LocalTrackInfo) (tracks []we } func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrackInfo) int { - indices := c.GetLocalTrackIndicesByInfo(removeInfo) + indices := c.getLocalTrackIndicesByInfo(removeInfo) c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() @@ -142,7 +142,7 @@ func (c *Conference) RemoveTracksFromPeerConnectionsByInfo(removeInfo LocalTrack } func (c *Conference) RemoveTracksFromConfByInfo(removeInfo LocalTrackInfo) { - indicesToRemove := c.GetLocalTrackIndicesByInfo(removeInfo) + indicesToRemove := c.getLocalTrackIndicesByInfo(removeInfo) c.Tracks.Mutex.Lock() defer c.Tracks.Mutex.Unlock() @@ -215,7 +215,7 @@ func (c *Conference) GetRemoteMetadataForDevice(deviceID id.DeviceID) event.Call for trackID := range info.Tracks { // Delete metadata, if we're the client hasn't published a track that is // included in the metadata yet - if len(c.GetLocalTrackIndicesByInfo(LocalTrackInfo{ + if len(c.getLocalTrackIndicesByInfo(LocalTrackInfo{ StreamID: streamID, TrackID: trackID, })) == 0 { diff --git a/src/main.go b/src/main.go index 453a800..264177a 100644 --- a/src/main.go +++ b/src/main.go @@ -48,7 +48,7 @@ var configFilePath = flag.String("config", "config.yaml", "configuration file pa var cpuProfile = flag.String("cpuProfile", "", "write CPU profile to `file`") var memProfile = flag.String("memProfile", "", "write memory profile to `file`") -func InitCpuProfiling(cpuProfile *string) func() { +func initCpuProfiling(cpuProfile *string) func() { log.Print("initializing CPU profiling") f, err := os.Create(*cpuProfile) @@ -67,7 +67,7 @@ func InitCpuProfiling(cpuProfile *string) func() { } } -func InitMemoryProfiling(memProfile *string) func() { +func initMemoryProfiling(memProfile *string) func() { log.Print("initializing memory profiling") return func() { @@ -85,14 +85,14 @@ func InitMemoryProfiling(memProfile *string) func() { } } -func InitLogging(logTime *bool) { +func initLogging(logTime *bool) { log.SetFlags(0) if *logTime { log.SetFlags(log.Ldate | log.Ltime) } } -func LoadConfig(configFilePath string) (*Config, error) { +func loadConfig(configFilePath string) (*Config, error) { log.Printf("loading %s", configFilePath) file, err := ioutil.ReadFile(configFilePath) if err != nil { @@ -117,14 +117,14 @@ func killListener(c chan os.Signal, beforeExit []func()) { func main() { flag.Parse() - InitLogging(logTime) + initLogging(logTime) beforeExit := []func(){} if *cpuProfile != "" { - beforeExit = append(beforeExit, InitCpuProfiling(cpuProfile)) + beforeExit = append(beforeExit, initCpuProfiling(cpuProfile)) } if *memProfile != "" { - beforeExit = append(beforeExit, InitMemoryProfiling(memProfile)) + beforeExit = append(beforeExit, initMemoryProfiling(memProfile)) } // try to handle os interrupt(signal terminated) @@ -133,7 +133,7 @@ func main() { go killListener(c, beforeExit) var err error - if config, err = LoadConfig(*configFilePath); err != nil { + if config, err = loadConfig(*configFilePath); err != nil { log.Fatalf("failed to load config file: %s", err) } From 8698c088a051ae768c55d0b318007f04746dfa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 16:28:48 +0200 Subject: [PATCH 118/124] Rename var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/call.go b/src/call.go index 23650a2..185cac9 100644 --- a/src/call.go +++ b/src/call.go @@ -233,7 +233,7 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { return } - ice := candidate.ToJSON() + jsonCandidate := candidate.ToJSON() candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ @@ -248,9 +248,9 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { }, Candidates: []event.CallCandidate{ { - Candidate: ice.Candidate, - SDPMLineIndex: int(*ice.SDPMLineIndex), - SDPMID: *ice.SDPMid, + Candidate: jsonCandidate.Candidate, + SDPMLineIndex: int(*jsonCandidate.SDPMLineIndex), + SDPMID: *jsonCandidate.SDPMid, // XXX: what about ice.UsernameFragment? }, }, From 7a47a55b47195495b9ec2550620f3ba4019aac51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 17:05:35 +0200 Subject: [PATCH 119/124] Send `UsernameFragment` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/call.go b/src/call.go index 185cac9..9bd583a 100644 --- a/src/call.go +++ b/src/call.go @@ -235,6 +235,15 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { jsonCandidate := candidate.ToJSON() + callCandidate := event.CallCandidate{ + Candidate: jsonCandidate.Candidate, + SDPMLineIndex: int(*jsonCandidate.SDPMLineIndex), + SDPMID: *jsonCandidate.SDPMid, + } + if jsonCandidate.UsernameFragment != nil { + callCandidate.UsernameFragment = *jsonCandidate.UsernameFragment + } + candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ BaseCallEventContent: event.BaseCallEventContent{ @@ -246,14 +255,7 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { PartyID: string(c.Client.DeviceID), Version: event.CallVersion("1"), }, - Candidates: []event.CallCandidate{ - { - Candidate: jsonCandidate.Candidate, - SDPMLineIndex: int(*jsonCandidate.SDPMLineIndex), - SDPMID: *jsonCandidate.SDPMid, - // XXX: what about ice.UsernameFragment? - }, - }, + Candidates: []event.CallCandidate{callCandidate}, }, } c.sendToDevice(event.CallCandidates, candidateEvtContent) From 223fbd7b2f5e1840767b19016f01d3508ddae8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 18:12:01 +0200 Subject: [PATCH 120/124] Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- go.mod | 4 ++++ go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b61baaa..cbbf85b 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,10 @@ require ( github.com/pion/transport v0.13.0 // indirect github.com/pion/turn/v2 v2.0.8 // indirect github.com/pion/udp v0.1.1 // indirect + github.com/tidwall/gjson v1.14.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/sjson v1.2.4 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect diff --git a/go.sum b/go.sum index 254fb11..e95d047 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= -github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -21,13 +17,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ= -github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2 h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -91,6 +83,15 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= +github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -129,7 +130,6 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From e0255e412cf59b63cbb51457da559db949dc83bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 18:12:35 +0200 Subject: [PATCH 121/124] Restructure to use `OnEvent` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/focus.go | 101 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/matrix.go | 99 +------------------------------------------------ 2 files changed, 100 insertions(+), 100 deletions(-) diff --git a/src/focus.go b/src/focus.go index 21816cc..6b5ec40 100644 --- a/src/focus.go +++ b/src/focus.go @@ -18,8 +18,11 @@ package main import ( "errors" + "log" + "strings" "sync" + "maunium.net/go/mautrix" "maunium.net/go/mautrix/event" ) @@ -29,14 +32,16 @@ type Confs struct { } type Focus struct { - name string - confs Confs + name string + client *mautrix.Client + confs Confs } -func NewFocus(name string) *Focus { +func NewFocus(name string, client *mautrix.Client) *Focus { f := new(Focus) f.name = name + f.client = client f.confs.confs = make(map[string]*Conference) return f @@ -61,3 +66,93 @@ func (f *Focus) GetConf(confID string, create bool) (*Conference, error) { } return co, nil } + +func (f *Focus) getExistingCall(confID string, callID string) (*Call, error) { + var conf *Conference + var call *Call + var err error + + if conf, err = f.GetConf(confID, false); err != nil || conf == nil { + log.Printf("failed to get conf %s: %s", confID, err) + return nil, err + } + if call, err = conf.GetCall(callID, false); err != nil || call == nil { + log.Printf("failed to get call %s: %s", callID, err) + return nil, err + } + return call, nil +} + +func (f *Focus) onEvent(_ mautrix.EventSource, evt *event.Event) { + // We only care about to-device events + if evt.Type.Class != event.ToDeviceEventType { + return + } + + if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { + log.Printf("received non-call to-device event %s", evt.Type.Type) + return + + } else if evt.Type.Type != event.ToDeviceCallCandidates.Type && evt.Type.Type != event.ToDeviceCallSelectAnswer.Type { + log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) + } + + if evt.Content.Raw["dest_session_id"] != localSessionID { + log.Printf("%s | SessionID %s does not match our SessionID - ignoring", evt.Content.Raw["dest_session_id"], localSessionID) + return + } + + var conf *Conference + var call *Call + var err error + + switch evt.Type.Type { + case event.ToDeviceCallInvite.Type: + invite := evt.Content.AsCallInvite() + if conf, err = f.GetConf(invite.ConfID, true); err != nil { + log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) + return + } + if err := conf.RemoveOldCallsByDeviceAndSessionIDs(invite.DeviceID, invite.SenderSessionID); err != nil { + log.Printf("%s | error removing old calls - ignoring call: %+v", evt.Sender.String(), err) + return + } + if call, err = conf.GetCall(invite.CallID, true); err != nil || call == nil { + log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) + return + } + call.UserID = evt.Sender + call.DeviceID = invite.DeviceID + // XXX: What if an SFU gets restarted? + call.LocalSessionID = localSessionID + call.RemoteSessionID = invite.SenderSessionID + call.Client = f.client + call.OnInvite(invite) + case event.ToDeviceCallCandidates.Type: + candidates := evt.Content.AsCallCandidates() + if call, err = f.getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil { + return + } + call.OnCandidates(candidates) + case event.ToDeviceCallSelectAnswer.Type: + selectAnswer := evt.Content.AsCallSelectAnswer() + if call, err = f.getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil { + return + } + call.OnSelectAnswer(selectAnswer) + case event.ToDeviceCallHangup.Type: + hangup := evt.Content.AsCallHangup() + if call, err = f.getExistingCall(hangup.ConfID, hangup.CallID); err != nil { + return + } + call.OnHangup(hangup) + // Events we don't care about + case event.ToDeviceCallNegotiate.Type: + log.Printf("%s | ignoring event %s as should be handled over DC", evt.Sender.String(), evt.Type.Type) + case event.ToDeviceCallReject.Type: + case event.ToDeviceCallAnswer.Type: + log.Printf("%s | ignoring event %s as we are always the ones answering", evt.Sender.String(), evt.Type.Type) + default: + log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Sender.String(), evt.Type.Type) + } +} diff --git a/src/matrix.go b/src/matrix.go index cfb4e57..e99b7cc 100644 --- a/src/matrix.go +++ b/src/matrix.go @@ -19,10 +19,8 @@ package main import ( "fmt" "log" - "strings" "maunium.net/go/mautrix" - "maunium.net/go/mautrix/event" ) const localSessionID = "sfu" @@ -43,106 +41,13 @@ func InitMatrix() error { log.Printf("Identified SFU as device %s", whoami.DeviceID) client.DeviceID = whoami.DeviceID - focus := NewFocus(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID)) + focus := NewFocus(fmt.Sprintf("%s (%s)", config.UserID, client.DeviceID), client) syncer := client.Syncer.(*mautrix.DefaultSyncer) syncer.ParseEventContent = true // TODO: E2EE - - getExistingCall := func(confID string, callID string) (*Call, error) { - var conf *Conference - var call *Call - - if conf, err = focus.GetConf(confID, false); err != nil || conf == nil { - log.Printf("failed to get conf %s: %s", confID, err) - return nil, err - } - if call, err = conf.GetCall(callID, false); err != nil || call == nil { - log.Printf("failed to get call %s: %s", callID, err) - return nil, err - } - return call, nil - } - - syncer.OnSync(func(resp *mautrix.RespSync, since string) bool { - for _, evt := range resp.ToDevice.Events { - evt.Type.Class = event.ToDeviceEventType - err := evt.Content.ParseRaw(evt.Type) - if err != nil { - log.Printf("failed to parse to-device event of type %s: %v", evt.Type.Type, err) - continue - } - - var conf *Conference - var call *Call - - if !strings.HasPrefix(evt.Type.Type, "m.call.") && !strings.HasPrefix(evt.Type.Type, "org.matrix.call.") { - log.Printf("received non-call to-device event %s", evt.Type.Type) - continue - } else if evt.Type.Type != event.ToDeviceCallCandidates.Type && evt.Type.Type != event.ToDeviceCallSelectAnswer.Type { - log.Printf("%s | received to-device event %s", evt.Sender.String(), evt.Type.Type) - } - - if evt.Content.Raw["dest_session_id"] != localSessionID { - log.Printf("%s | SessionID %s does not match our SessionID %s - ignoring", evt.Content.Raw["dest_session_id"], localSessionID, err) - continue - } - - switch evt.Type.Type { - case event.ToDeviceCallInvite.Type: - invite := evt.Content.AsCallInvite() - if conf, err = focus.GetConf(invite.ConfID, true); err != nil || conf == nil { - log.Printf("%s | failed to create conf %s: %+v", evt.Sender.String(), invite.ConfID, err) - return true - } - if err := conf.RemoveOldCallsByDeviceAndSessionIDs(invite.DeviceID, invite.SenderSessionID); err != nil { - log.Printf("%s | error removing old calls - ignoring call: %+v", evt.Sender.String(), err) - return true - } - if call, err = conf.GetCall(invite.CallID, true); err != nil || call == nil { - log.Printf("%s | failed to create call: %+v", evt.Sender.String(), err) - return true - } - call.UserID = evt.Sender - call.DeviceID = invite.DeviceID - // XXX: What if an SFU gets restarted? - call.LocalSessionID = localSessionID - call.RemoteSessionID = invite.SenderSessionID - call.Client = client - call.OnInvite(invite) - case event.ToDeviceCallCandidates.Type: - candidates := evt.Content.AsCallCandidates() - if call, err = getExistingCall((*candidates).ConfID, (*candidates).CallID); err != nil || call == nil { - return true - } - call.OnCandidates(candidates) - case event.ToDeviceCallSelectAnswer.Type: - selectAnswer := evt.Content.AsCallSelectAnswer() - if call, err = getExistingCall(selectAnswer.ConfID, selectAnswer.CallID); err != nil || call == nil { - return true - } - call.OnSelectAnswer(selectAnswer) - case event.ToDeviceCallHangup.Type: - hangup := evt.Content.AsCallHangup() - if call, err = getExistingCall(hangup.ConfID, hangup.CallID); err != nil || call == nil { - return true - } - call.OnHangup(hangup) - - // Events we don't care about - case event.ToDeviceCallNegotiate.Type: - log.Printf("%s | ignoring event %s as should be handled over DC", evt.Sender.String(), evt.Type.Type) - case event.ToDeviceCallReject.Type: - case event.ToDeviceCallAnswer.Type: - log.Printf("%s | ignoring event %s as we are always the ones answering", evt.Sender.String(), evt.Type.Type) - default: - log.Printf("%s | ignoring unrecognised to-device event of type %s", evt.Sender.String(), evt.Type.Type) - } - } - - return true - }) + syncer.OnEvent(focus.onEvent) if err = client.Sync(); err != nil { log.Panic("Sync failed", err) From b7e8cbc3dd9f145af2aa8be36b35a7348872935b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 15 Aug 2022 18:20:24 +0200 Subject: [PATCH 122/124] Update `go.mod` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cbbf85b..5d0716a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/Sean-Der/sfu-to-sfu +module github.com/matrix-org/sfu-to-sfu go 1.18 From cc0c2e57ea06126750be24a71121f456ed2a16e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 17 Aug 2022 16:16:04 +0200 Subject: [PATCH 123/124] Send end of ICE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/call.go b/src/call.go index 9bd583a..7c64f54 100644 --- a/src/call.go +++ b/src/call.go @@ -39,6 +39,7 @@ type Call struct { Conf *Conference dataChannel *webrtc.DataChannel lastKeepAliveTimestamp time.Time + sentEndOfCandidates bool } func (c *Call) onDCSelect(start []event.SFUTrackDescription) { @@ -291,6 +292,25 @@ func (c *Call) iceConnectionStateHandler(state webrtc.ICEConnectionState) { if state == webrtc.ICEConnectionStateCompleted || state == webrtc.ICEConnectionStateConnected { c.lastKeepAliveTimestamp = time.Now() go c.CheckKeepAliveTimestamp() + + if !c.sentEndOfCandidates { + candidateEvtContent := &event.Content{ + Parsed: event.CallCandidatesEventContent{ + BaseCallEventContent: event.BaseCallEventContent{ + CallID: c.CallID, + ConfID: c.Conf.ConfID, + DeviceID: c.Client.DeviceID, + SenderSessionID: c.LocalSessionID, + DestSessionID: c.RemoteSessionID, + PartyID: string(c.Client.DeviceID), + Version: event.CallVersion("1"), + }, + Candidates: []event.CallCandidate{{Candidate: ""}}, + }, + } + c.sendToDevice(event.CallCandidates, candidateEvtContent) + c.sentEndOfCandidates = true + } } } From 0f40571edd8e7467cc070c0dbfee58ca8332d2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 17 Aug 2022 16:28:27 +0200 Subject: [PATCH 124/124] Remove UsernameFragment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/call.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/call.go b/src/call.go index 7c64f54..231e22f 100644 --- a/src/call.go +++ b/src/call.go @@ -236,15 +236,6 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { jsonCandidate := candidate.ToJSON() - callCandidate := event.CallCandidate{ - Candidate: jsonCandidate.Candidate, - SDPMLineIndex: int(*jsonCandidate.SDPMLineIndex), - SDPMID: *jsonCandidate.SDPMid, - } - if jsonCandidate.UsernameFragment != nil { - callCandidate.UsernameFragment = *jsonCandidate.UsernameFragment - } - candidateEvtContent := &event.Content{ Parsed: event.CallCandidatesEventContent{ BaseCallEventContent: event.BaseCallEventContent{ @@ -256,7 +247,11 @@ func (c *Call) iceCandidateHandler(candidate *webrtc.ICECandidate) { PartyID: string(c.Client.DeviceID), Version: event.CallVersion("1"), }, - Candidates: []event.CallCandidate{callCandidate}, + Candidates: []event.CallCandidate{{ + Candidate: jsonCandidate.Candidate, + SDPMLineIndex: int(*jsonCandidate.SDPMLineIndex), + SDPMID: *jsonCandidate.SDPMid, + }}, }, } c.sendToDevice(event.CallCandidates, candidateEvtContent)