naive LRU to throw away old rooms

halve forward pagination frequency
don't stack up on forward pagination jobs

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

Author: t3chguy

src/github.com/t3chguy/matrix-static/job-room-aliases.go

@@ -44,4 +44,5 @@ func (job RoomAliasesJob) Work(w *Worker) {
 		job.pageSize,
 		job.page,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/job-room-events.go

@@ -52,4 +52,5 @@ func (job RoomEventsJob) Work(w *Worker) {
 		atBottomEnd,
 		err,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/job-room-forwardpaginate.go

@@ -14,12 +14,33 @@
 
 package main
 
+import (
+	log "github.com/Sirupsen/logrus"
+	"sync"
+	"time"
+)
+
 // This Job has no Resp.
 
-type RoomForwardPaginateJob struct{}
+type RoomForwardPaginateJob struct {
+	wg *sync.WaitGroup
+}
+
+const LastAccessDiscardDuration = 30 * time.Minute
 
 func (job RoomForwardPaginateJob) Work(w *Worker) {
+	// discard old rooms first
+	numRoomsBefore := len(w.rooms)
+	for id, room := range w.rooms {
+		if room.LastAccess.Before(time.Now().Add(-LastAccessDiscardDuration)) {
+			delete(w.rooms, id)
+		}
+	}
+	numRoomsAfter := len(w.rooms)
+	log.WithField("worker", w.ID).WithField("numRooms", numRoomsAfter).Infof("Removed %d rooms", numRoomsBefore-numRoomsAfter)
+
 	for _, room := range w.rooms {
 		room.ForwardPaginateRoom()
 	}
+	job.wg.Done()
 }

src/github.com/t3chguy/matrix-static/job-room-memberinfo.go

@@ -59,4 +59,5 @@ func (job RoomMemberInfoJob) Work(w *Worker) {
 		memberInfo,
 		err,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/job-room-members.go

@@ -50,4 +50,5 @@ func (job RoomMembersJob) Work(w *Worker) {
 		job.pageSize,
 		job.page,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/job-room-powerlevels.go

@@ -35,4 +35,5 @@ func (job RoomPowerLevelsJob) Work(w *Worker) {
 		room.RoomInfo(),
 		powerLevels,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/job-room-servers.go

@@ -44,4 +44,5 @@ func (job RoomServersJob) Work(w *Worker) {
 		job.pageSize,
 		job.page,
 	}
+	room.Access()
 }

src/github.com/t3chguy/matrix-static/matrix-static.go

@@ -34,6 +34,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"sync"
 	"time"
 	"unicode"
 	"unicode/utf8"
@@ -362,13 +363,17 @@ func startPublicRoomListTimer(worldReadableRooms *mxclient.WorldReadableRooms) {
 	}
 }
 
-const LazyForwardPaginateRooms = time.Minute
+const LazyForwardPaginateRooms = 2 * time.Minute
 
 func startForwardPaginator(workers *Workers) {
-	t := time.NewTicker(LazyForwardPaginateRooms)
+	//t := time.NewTicker(LazyForwardPaginateRooms)
+	wg := sync.WaitGroup{}
 	for {
-		<-t.C
+		//<-t.C
+		time.Sleep(LazyForwardPaginateRooms)
+		wg.Add(int(workers.numWorkers))
 		log.Info("Forward paginating all loaded rooms")
-		workers.JobForAllWorkers(RoomForwardPaginateJob{})
+		workers.JobForAllWorkers(RoomForwardPaginateJob{&wg})
+		wg.Wait()
 	}
 }

src/github.com/t3chguy/matrix-static/mxclient/room.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"github.com/matrix-org/gomatrix"
 	"github.com/t3chguy/matrix-static/utils"
+	"time"
 )
 
 type RoomInfo struct {
@@ -46,6 +47,12 @@ type Room struct {
 	latestRoomState RoomState
 
 	HasReachedHistoricEndOfTimeline bool
+
+	LastAccess time.Time
+}
+
+func (r *Room) Access() {
+	r.LastAccess = time.Now()
 }
 
 // ForwardPaginateRoom queries the API for any events newer than the latest one currently in the timeline and appends them.
@@ -197,6 +204,7 @@ func (m *Client) NewRoom(roomID string) (*Room, error) {
 		backPaginationToken:    resp.Messages.Start,
 		eventList:              filteredEventList,
 		latestRoomState:        *NewRoomState(m),
+		LastAccess:             time.Now(),
 	}
 
 	for _, event := range resp.State {