Added extended logging (#240), fixed bug that prevented setting Manage_Users API permission on new API key, added Manage_Logs API permission

* Added Manage_Logs API permission, added API endpoint to delete logs, added more logging, added filtering and deletion of logs in UI, fixed bug that prevented setting Manage_Users API permission on new API key

Author: Forceu

build/go-generate/minifyStaticContent.go

@@ -137,6 +137,6 @@ func fileExists(filename string) bool {
 // Auto-generated content below, do not modify
 // Version codes can be changed in updateVersionNumbers.go
 
-const jsAdminVersion = 8
+const jsAdminVersion = 9
 const jsE2EVersion = 5
 const cssMainVersion = 4

build/go-generate/updateVersionNumbers.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 )
 
-const versionJsAdmin = 8
+const versionJsAdmin = 9
 const versionJsDropzone = 5
 const versionJsE2EAdmin = 5
 const versionCssMain = 4

cmd/gokapi/Main.go

@@ -72,7 +72,7 @@ func main() {
 	createSsl(passedFlags)
 	initCloudConfig(passedFlags)
 	go storage.CleanUp(true)
-	logging.AddString("Gokapi started")
+	logging.LogStartup()
 	go webserver.Start()
 
 	c := make(chan os.Signal)
@@ -85,6 +85,7 @@ func main() {
 func shutdown() {
 	fmt.Println("Shutting down...")
 	webserver.Shutdown()
+	logging.LogShutdown()
 	database.Close()
 }
 
@@ -164,6 +165,7 @@ func initCloudConfig(passedFlags flagparser.MainFlags) {
 // Checks for command line arguments that have to be parsed after loading the configuration
 func reconfigureServer(passedFlags flagparser.MainFlags) bool {
 	if passedFlags.Reconfigure {
+		logging.LogSetup()
 		setup.RunConfigModification()
 		return true
 	}
@@ -210,7 +212,7 @@ func setDeploymentPassword(passedFlags flagparser.MainFlags) {
 	if passedFlags.DeploymentPassword == "" {
 		return
 	}
-	logging.AddString("Password has been changed for deployment")
+	logging.LogDeploymentPassword()
 	configuration.SetDeploymentPassword(passedFlags.DeploymentPassword)
 }
 

internal/configuration/Configuration.go

@@ -16,7 +16,7 @@ import (
 	"github.com/forceu/gokapi/internal/configuration/database"
 	"github.com/forceu/gokapi/internal/environment"
 	"github.com/forceu/gokapi/internal/helper"
-	log "github.com/forceu/gokapi/internal/logging"
+	"github.com/forceu/gokapi/internal/logging"
 	"github.com/forceu/gokapi/internal/models"
 	"github.com/forceu/gokapi/internal/storage/filesystem"
 	"io"
@@ -93,7 +93,7 @@ func Load() {
 	}
 	helper.CreateDir(serverSettings.DataDir)
 	filesystem.Init(serverSettings.DataDir)
-	log.Init(Environment.DataDir)
+	logging.Init(Environment.DataDir)
 }
 
 // ConnectDatabase loads the database that is defined in the configuration
@@ -152,6 +152,7 @@ func MigrateToV2(authPassword string, allowedUsers []string) {
 		database.SaveMetaData(file)
 	}
 	database.DeleteAllSessions()
+	logging.UpgradeToV2()
 	fmt.Println("Migration complete")
 }
 

internal/configuration/database/provider/redis/Redis.go

@@ -18,7 +18,7 @@ type DatabaseProvider struct {
 }
 
 // DatabaseSchemeVersion contains the version number to be expected from the current database. If lower, an upgrade will be performed
-const DatabaseSchemeVersion = 4
+const DatabaseSchemeVersion = 5
 
 // New returns an instance
 func New(dbConfig models.DbConnection) (DatabaseProvider, error) {
@@ -96,7 +96,7 @@ func (p DatabaseProvider) Upgrade(currentDbVersion int) {
 	if currentDbVersion < 3 {
 		fmt.Println("Please update to v1.9.6 before upgrading to 2.0.0")
 	}
-	// < v2.0.0-beta
+	// < v2.0.0-beta1
 	if currentDbVersion < 4 {
 		p.DeleteAllSessions()
 		apiKeys := p.GetAllApiKeys()
@@ -109,6 +109,15 @@ func (p DatabaseProvider) Upgrade(currentDbVersion int) {
 		p.SaveEnd2EndInfo(legacyE2e, 0)
 		p.deleteKey("e2einfo")
 	}
+	// < v2.0.0-beta2
+	if currentDbVersion < 5 {
+		keys := p.GetAllApiKeys()
+		for _, key := range keys {
+			if key.IsSystemKey {
+				p.DeleteApiKey(key.Id)
+			}
+		}
+	}
 }
 
 const keyDbVersion = "dbversion"

internal/configuration/database/provider/sqlite/Sqlite.go

@@ -20,7 +20,7 @@ type DatabaseProvider struct {
 }
 
 // DatabaseSchemeVersion contains the version number to be expected from the current database. If lower, an upgrade will be performed
-const DatabaseSchemeVersion = 7
+const DatabaseSchemeVersion = 8
 
 // New returns an instance
 func New(dbConfig models.DbConnection) (DatabaseProvider, error) {
@@ -81,6 +81,15 @@ func (p DatabaseProvider) Upgrade(currentDbVersion int) {
 			p.SaveEnd2EndInfo(legacyE2E, 0)
 		}
 	}
+	// < v2.0.0-beta2
+	if currentDbVersion < 8 {
+		keys := p.GetAllApiKeys()
+		for _, key := range keys {
+			if key.IsSystemKey {
+				p.DeleteApiKey(key.Id)
+			}
+		}
+	}
 }
 
 func getLegacyE2EConfig(p DatabaseProvider) models.E2EInfoEncrypted {

internal/logging/Logging.go

@@ -1,6 +1,7 @@
 package logging
 
 import (
+	"bufio"
 	"fmt"
 	"github.com/forceu/gokapi/internal/environment"
 	"github.com/forceu/gokapi/internal/helper"
@@ -16,6 +17,13 @@ import (
 var logPath = "config/log.txt"
 var mutex sync.Mutex
 
+const categoryInfo = "info"
+const categoryDownload = "download"
+const categoryUpload = "upload"
+const categoryEdit = "edit"
+const categoryAuth = "auth"
+const categoryWarning = "warning"
+
 var outputToStdout = false
 
 // Init sets the path where to write the log file to
@@ -25,27 +33,176 @@ func Init(filePath string) {
 	outputToStdout = env.LogToStdout
 }
 
-// AddString adds a line to the logfile including the current date. Non-Blocking
-func AddString(text string) {
-	output := formatDate(text)
+// GetAll returns all log entries as a single string and if the log file exists
+func GetAll() (string, bool) {
+	if helper.FileExists(logPath) {
+		content, err := os.ReadFile(logPath)
+		helper.Check(err)
+		return string(content), true
+	} else {
+		return fmt.Sprintf("[%s] No log file found!", categoryWarning), false
+	}
+}
+
+// createLogEntry adds a line to the logfile including the current date. Also outputs to Stdout if set.
+func createLogEntry(category, text string, blocking bool) {
+	output := createLogFormat(category, text)
 	if outputToStdout {
 		fmt.Println(output)
 	}
-	go writeToFile(output)
+	if blocking {
+		writeToFile(output)
+	} else {
+		go writeToFile(output)
+	}
 }
 
-// GetLogPath returns the relative path to the log file
-func GetLogPath() string {
-	return logPath
+func createLogFormat(category, text string) string {
+	return createLogFormatCustomTimestamp(category, text, time.Now())
+}
+func createLogFormatCustomTimestamp(category, text string, timestamp time.Time) string {
+	return fmt.Sprintf("%s   [%s] %s", getDate(timestamp), category, text)
 }
 
-// AddDownload adds a line to the logfile when a download was requested. Non-Blocking
-func AddDownload(file *models.File, r *http.Request, saveIp bool) {
+// LogStartup adds a log entry to indicate that Gokapi has started. Non-blocking
+func LogStartup() {
+	createLogEntry(categoryInfo, "Gokapi started", false)
+}
+
+// LogShutdown adds a log entry to indicate that Gokapi is shutting down. Blocking call
+func LogShutdown() {
+	createLogEntry(categoryInfo, "Gokapi shutting down", true)
+}
+
+// LogSetup adds a log entry to indicate that the setup was run. Non-blocking
+func LogSetup() {
+	createLogEntry(categoryAuth, "Re-running Gokapi setup", false)
+}
+
+// LogDeploymentPassword adds a log entry to indicate that a deployment password was set. Non-blocking
+func LogDeploymentPassword() {
+	createLogEntry(categoryAuth, "Setting new admin password", false)
+}
+
+// LogUserDeletion adds a log entry to indicate that a user was deleted. Non-blocking
+func LogUserDeletion(modifiedUser, userEditor models.User) {
+	createLogEntry(categoryAuth, fmt.Sprintf("%s (#%d) was deleted by %s (user #%d)",
+		modifiedUser.Name, modifiedUser.Id, userEditor.Name, userEditor.Id), false)
+}
+
+// LogUserEdit adds a log entry to indicate that a user was modified. Non-blocking
+func LogUserEdit(modifiedUser, userEditor models.User) {
+	createLogEntry(categoryAuth, fmt.Sprintf("%s (#%d) was modified by %s (user #%d)",
+		modifiedUser.Name, modifiedUser.Id, userEditor.Name, userEditor.Id), false)
+}
+
+// LogUserCreation adds a log entry to indicate that a user was created. Non-blocking
+func LogUserCreation(modifiedUser, userEditor models.User) {
+	createLogEntry(categoryAuth, fmt.Sprintf("%s (#%d) was created by %s (user #%d)",
+		modifiedUser.Name, modifiedUser.Id, userEditor.Name, userEditor.Id), false)
+}
+
+// LogDownload adds a log entry when a download was requested. Non-Blocking
+func LogDownload(file models.File, r *http.Request, saveIp bool) {
 	if saveIp {
-		AddString(fmt.Sprintf("Download: Filename %s, IP %s, ID %s, Useragent %s", file.Name, getIpAddress(r), file.Id, r.UserAgent()))
+		createLogEntry(categoryDownload, fmt.Sprintf("%s, IP %s, ID %s, Useragent %s", file.Name, getIpAddress(r), file.Id, r.UserAgent()), false)
 	} else {
-		AddString(fmt.Sprintf("Download: Filename %s, ID %s, Useragent %s", file.Name, file.Id, r.UserAgent()))
+		createLogEntry(categoryDownload, fmt.Sprintf("%s, ID %s, Useragent %s", file.Name, file.Id, r.UserAgent()), false)
+	}
+}
+
+// LogUpload adds a log entry when an upload was created. Non-Blocking
+func LogUpload(file models.File, user models.User) {
+	createLogEntry(categoryUpload, fmt.Sprintf("%s, ID %s, uploaded by %s (user #%d)", file.Name, file.Id, user.Name, user.Id), false)
+}
+
+// LogEdit adds a log entry when an upload was edited. Non-Blocking
+func LogEdit(file models.File, user models.User) {
+	createLogEntry(categoryEdit, fmt.Sprintf("%s, ID %s, edited by %s (user #%d)", file.Name, file.Id, user.Name, user.Id), false)
+}
+
+// LogReplace adds a log entry when an upload was replaced. Non-Blocking
+func LogReplace(originalFile, newContent models.File, user models.User) {
+	createLogEntry(categoryEdit, fmt.Sprintf("%s, ID %s had content replaced with %s (ID %s) by %s (user #%d)",
+		originalFile.Name, originalFile.Id, newContent.Name, newContent.Id, user.Name, user.Id), false)
+}
+
+// LogDelete adds a log entry when an upload was deleted. Non-Blocking
+func LogDelete(file models.File, user models.User) {
+	createLogEntry(categoryEdit, fmt.Sprintf("%s, ID %s, deleted by %s (user #%d)", file.Name, file.Id, user.Name, user.Id), false)
+}
+
+// UpgradeToV2 adds tags to existing logs
+// deprecated
+func UpgradeToV2() {
+	content, exists := GetAll()
+	mutex.Lock()
+	if !exists {
+		return
+	}
+	var newLogs strings.Builder
+	scanner := bufio.NewScanner(strings.NewReader(content))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "Gokapi started") {
+			line = strings.Replace(line, "Gokapi started", "["+categoryInfo+"] Gokapi started", 1)
+		}
+		if strings.Contains(line, "Download: Filename") {
+			line = strings.Replace(line, "Download: Filename", "["+categoryDownload+"] Filename", 1)
+		}
+		newLogs.WriteString(line)
+		newLogs.WriteString("\n")
 	}
+	helper.Check(scanner.Err())
+	err := os.WriteFile(logPath, []byte(newLogs.String()), 0600)
+	helper.Check(err)
+	defer mutex.Unlock()
+}
+
+func DeleteLogs(userName string, userId int, cutoff int64, r *http.Request) {
+	if cutoff == 0 {
+		deleteAllLogs(userName, userId, r)
+		return
+	}
+	mutex.Lock()
+	logFile, err := os.ReadFile(logPath)
+	helper.Check(err)
+	var newFile strings.Builder
+	scanner := bufio.NewScanner(strings.NewReader(string(logFile)))
+	newFile.WriteString(getLogDeletionMessage(userName, userId, r, time.Unix(cutoff, 0)))
+	for scanner.Scan() {
+		line := scanner.Text()
+		timeEntry, err := parseTimeLogEntry(line)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		if timeEntry.Unix() > cutoff {
+			newFile.WriteString(line + "\n")
+		}
+	}
+	err = os.WriteFile(logPath, []byte(newFile.String()), 0600)
+	helper.Check(err)
+	defer mutex.Unlock()
+}
+
+func parseTimeLogEntry(input string) (time.Time, error) {
+	const layout = "Mon, 02 Jan 2006 15:04:05 MST"
+	lineContent := strings.Split(input, "   [")
+	return time.Parse(layout, lineContent[0])
+}
+
+func getLogDeletionMessage(userName string, userId int, r *http.Request, timestamp time.Time) string {
+	return createLogFormatCustomTimestamp(categoryWarning, fmt.Sprintf("Previous logs deleted by %s (user #%d) on %s. IP: %s\n",
+		userName, userId, getDate(time.Now()), getIpAddress(r)), timestamp)
+}
+
+func deleteAllLogs(userName string, userId int, r *http.Request) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	message := getLogDeletionMessage(userName, userId, r, time.Now())
+	err := os.WriteFile(logPath, []byte(message), 0600)
+	helper.Check(err)
 }
 
 func writeToFile(text string) {
@@ -58,8 +215,8 @@ func writeToFile(text string) {
 	helper.Check(err)
 }
 
-func formatDate(input string) string {
-	return time.Now().UTC().Format(time.RFC1123) + "   " + input
+func getDate(timestamp time.Time) string {
+	return timestamp.UTC().Format(time.RFC1123)
 }
 
 func getIpAddress(r *http.Request) string {

internal/logging/Logging_test.go

@@ -39,12 +39,10 @@ func TestInit(t *testing.T) {
 
 func TestAddString(t *testing.T) {
 	test.FileDoesNotExist(t, "test/log.txt")
-	AddString("Hello")
-	// Need sleep, as AddString() is non-blocking
-	time.Sleep(500 * time.Millisecond)
+	createLogEntry(categoryInfo, "Hello", true)
 	test.FileExists(t, "test/log.txt")
 	content, _ := os.ReadFile("test/log.txt")
-	test.IsEqualBool(t, strings.Contains(string(content), "UTC   Hello"), true)
+	test.IsEqualBool(t, strings.Contains(string(content), "UTC   [info] Hello"), true)
 }
 
 func TestAddDownload(t *testing.T) {
@@ -55,19 +53,15 @@ func TestAddDownload(t *testing.T) {
 	r := httptest.NewRequest("GET", "/test", nil)
 	r.Header.Set("User-Agent", "testAgent")
 	r.Header.Add("X-REAL-IP", "1.1.1.1")
-	AddDownload(&file, r, true)
-	// Need sleep, as AddDownload() is non-blocking
+	LogDownload(file, r, true)
+	// Need sleep, as LogDownload() is non-blocking
 	time.Sleep(500 * time.Millisecond)
 	content, _ := os.ReadFile("test/log.txt")
-	test.IsEqualBool(t, strings.Contains(string(content), "UTC   Download: Filename testName, IP 1.1.1.1, ID testId, Useragent testAgent"), true)
+	test.IsEqualBool(t, strings.Contains(string(content), "UTC   [download] testName, IP 1.1.1.1, ID testId, Useragent testAgent"), true)
 	r.Header.Add("X-REAL-IP", "2.2.2.2")
-	AddDownload(&file, r, false)
-	// Need sleep, as AddDownload() is non-blocking
+	LogDownload(file, r, false)
+	// Need sleep, as LogDownload() is non-blocking
 	time.Sleep(500 * time.Millisecond)
 	content, _ = os.ReadFile("test/log.txt")
 	test.IsEqualBool(t, strings.Contains(string(content), "2.2.2.2"), false)
 }
-
-func TestGetLogPath(t *testing.T) {
-	test.IsEqualString(t, GetLogPath(), "test/log.txt")
-}