worker|cmd: Use XDG for Cache and data (#86)

Author: xescugc

cmd/root.go

@@ -4,9 +4,13 @@ import (
 	"github.com/urfave/cli/v3"
 )
 
+var (
+	AppName = "qid"
+)
+
 var (
 	Cmd = &cli.Command{
-		Name:  "qid",
+		Name:  AppName,
 		Usage: "QID is a small CI/CD build on top of a Queue(Pub/Sub) system",
 		Commands: []*cli.Command{
 			serverCmd,

cmd/server.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 
@@ -27,6 +28,8 @@ import (
 
 	"gocloud.dev/pubsub/mempubsub"
 	"gocloud.dev/pubsub/natspubsub"
+
+	"github.com/adrg/xdg"
 )
 
 var (
@@ -45,7 +48,6 @@ var (
 			&cli.StringFlag{Name: "db-password", Usage: "Database Password"},
 			&cli.StringFlag{Name: "db-name", Usage: "Database Name"},
 			&cli.BoolFlag{Name: "run-migrations", Value: true, Usage: "Flag to know if migrations should be ran"},
-			&cli.StringFlag{Name: "db-file", Usage: "Flag to know where the DB File is"},
 
 			&cli.BoolFlag{Name: "run-worker", Value: true, Usage: "Runs a worker with QID server"},
 			&cli.IntFlag{Name: "concurrency", Value: 1, Usage: "Number of workers to start in one instance"},
@@ -102,14 +104,17 @@ var (
 				return fmt.Errorf("failed to open: %v", err)
 			}
 			defer topic.Shutdown(ctx)
-
+			dbFile, err := xdg.DataFile(filepath.Join(AppName, AppName+".db"))
+			if err != nil {
+				return fmt.Errorf("failed to create dbFile: %v", err)
+			}
 			logger.Log("msg", "DB connection starting ...", "db-system", cfg.DBSystem)
 			db, err := mysql.New(cfg.DBHost, cfg.DBPort, cfg.DBUser, cfg.DBPassword, mysql.Options{
 				DBName:          cfg.DBName,
 				MultiStatements: true,
 				ClientFoundRows: true,
 				System:          cfg.DBSystem,
-				DBFile:          cfg.DBFile,
+				DBFile:          dbFile,
 			})
 			if err != nil {
 				panic(err)

docs/documentation.md

@@ -16,7 +16,7 @@ Job that depends on that Resource that changed. If there is then a new job/s wil
 Then when it finishes it checks if another job depends on it and queues for that job/s to be triggered.
 
 Workers execute everything on the host machine for now (either on the server if local or on the worker host), each time something is executed
-it's done on a unique `tmp/` folder so it never collisions with a previous run, will potentially [change](https://github.com/xescugc/qid/issues/57) on the future
+it's done on a unique `$XDG_CACHE_HOME/qid/{UUID}/` folder so it never collisions with a previous run, will potentially [change](https://github.com/xescugc/qid/issues/57) on the future
 
 ## Server Configuration
 
@@ -28,8 +28,7 @@ it's done on a unique `tmp/` folder so it never collisions with a previous run,
     * `db-user`: Database User
     * `db-password`: Database Password
     * `db-name`: Database Name
-  * `sqlite`: Uses SQLite
-    * `db-file`: The file in which to store the DB (does not need to exist as it'll be created)
+  * `sqlite`: Uses SQLite. The file will be at `$XDG_DATA_HOME/qid/qid.db`
   * `mem`: Run the DB in memory, which basically uses the SQLite memory option
 * `run-worker`: Specifies if you want to run the workers on the same server or not
 * `concurrency`: How many parallel instances has the worker

go.mod

@@ -4,12 +4,14 @@ go 1.25.1
 
 require (
 	github.com/VividCortex/mysqlerr v1.0.0
+	github.com/adrg/xdg v0.5.3
 	github.com/awalterschulze/gographviz v2.0.3+incompatible
 	github.com/cycloidio/sqlr v1.0.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-kit/kit v0.13.0
 	github.com/go-kit/log v0.2.0
 	github.com/go-sql-driver/mysql v1.9.3
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/handlers v1.5.2
 	github.com/gorilla/mux v1.8.1
 	github.com/gosimple/slug v1.15.0
@@ -44,7 +46,6 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect

go.sum

@@ -13,8 +13,16 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/VividCortex/mysqlerr v1.0.0 h1:5pZ2TZA+YnzPgzBfiUWGqWmKDVNBdrkf9g+DNe1Tiq8=
 github.com/VividCortex/mysqlerr v1.0.0/go.mod h1:xERx8E4tBhLvpjzdUyQiSfUxeMcATEQrflDAfXsqcAE=
+github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
+github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
 github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk=
+github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
@@ -111,6 +119,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lopezator/migrator v0.3.1 h1:ZFPT6aC7+nGWkqhleynABZ6ftycSf6hmHHLOaryq1Og=
 github.com/lopezator/migrator v0.3.1/go.mod h1:X+lHDMZ9Ci3/KdbypJcQYFFwipVrJsX4fRCQ4QLauYk=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
@@ -137,6 +147,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
+github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=

qid/config/config.go

@@ -12,9 +12,6 @@ type Config struct {
 	DBPassword string `koanf:"db-password"`
 	DBName     string `koanf:"db-name"`
 
-	// SQLite
-	DBFile string `koanf:"db-file"`
-
 	RunWorker   bool `koanf:"run-worker"`
 	Concurrency int  `koanf:"concurrency"`
 

qid/transport/http/templates/views/layouts/index.tmpl

@@ -767,22 +767,36 @@
         template: _.template($('#resource-versions-view').html()),
         initialize: function() {
           this.listenTo(this.collection, "add", this.addVersion)
+          this.listenTo(this.model, "change", this.render)
 
           // TODO: most likely add the {reset: true} as the
           // order is really important
           this.collection.fetch()
+
+          var that = this
+          this.intervalID = window.setInterval(function() {
+            that.model.fetch()
+          }, 1000);
         },
         render: function () {
           this.$el.html(this.template({
             pipeline: this.collection.resource.collection.pipeline.toJSON(),
             resource: this.collection.resource.toJSON(),
           }));
+          var that = this;
+          this.collection.each(function(m) {
+            that.addVersion(m)
+          })
           return this; // enable chained calls
         },
         addVersion: function(m) {
           var ver = new app.ResourceVersionView({model: m})
           $('#resource-versions').append(ver.render().el);
-        }
+        },
+        remove: function() {
+          clearInterval(this.intervalID)
+          Backbone.View.prototype.remove.call(this);
+        },
       })
       app.ResourceVersionView = Backbone.View.extend({
         // TODO: Check to set the attributes
@@ -899,6 +913,7 @@
           var that = this
           $.when.apply($, complete).done(function() {
             that.contentView = new app.ResourceVersionsView({
+              model: rs,
               collection: versions,
             });
             $('#main').html(that.contentView.render().el)

worker/service.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"slices"
 	"strings"
 
-	"github.com/davecgh/go-spew/spew"
+	"github.com/adrg/xdg"
+	"github.com/google/uuid"
 	"github.com/xescugc/qid/qid"
 	"github.com/xescugc/qid/qid/build"
 	"github.com/xescugc/qid/qid/queue"
@@ -55,15 +57,17 @@ func (w *Worker) Run(ctx context.Context) error {
 			continue
 		}
 
-		cwd, err := os.MkdirTemp("", "qid")
+		uuiddir, err := uuid.NewV7()
+		if err != nil {
+			return fmt.Errorf("failed to creat UUID %w", err)
+		}
+		// We append a file "qid" just so the CacheFile creates the full dir,
+		// afterward we just get the Dir of the cwd
+		cwd, err := xdg.CacheFile(filepath.Join("qid", uuiddir.String(), "qid"))
 		if err != nil {
 			return fmt.Errorf("failed to creat Temp Dir: %w", err)
 		}
-
-		//err = os.Mkdir(cwd, 0755)
-		//if err != nil {
-		//return fmt.Errorf("failed to create dir %q: %w", cwd, err)
-		//}
+		cwd = filepath.Dir(cwd)
 
 		pp, err := w.qid.GetPipeline(ctx, m.PipelineName)
 		if err != nil {
@@ -127,7 +131,7 @@ func (w *Worker) Run(ctx context.Context) error {
 							if err != nil {
 								b.Get = append(b.Get, build.Step{
 									Name: g.Name,
-									Logs: err.Error(),
+									Logs: string(stdouterr) + "\n" + err.Error(),
 								})
 								w.failBuild(ctx, m, b, nil)
 								w.logger.Log("error", fmt.Errorf("failed to run command %q with args %q (%s): %w", rt.Pull.Path, rt.Pull.Args, stdouterr, err))
@@ -156,7 +160,7 @@ func (w *Worker) Run(ctx context.Context) error {
 				if err != nil {
 					b.Task = append(b.Task, build.Step{
 						Name: t.Name,
-						Logs: err.Error(),
+						Logs: string(stdouterr) + "\n" + err.Error(),
 					})
 					w.failBuild(ctx, m, b, nil)
 					w.logger.Log("error", fmt.Errorf("failed to run command %q with args %q: %w", t.Run.Path, t.Run.Args, err))
@@ -173,7 +177,6 @@ func (w *Worker) Run(ctx context.Context) error {
 					w.logger.Log("error", ferr)
 					continue
 				}
-				spew.Dump(string(stdouterr))
 				for _, nj := range pp.Jobs {
 					for _, g := range nj.Get {
 						if slices.Contains(g.Passed, j.Name) && g.Trigger {
@@ -228,10 +231,10 @@ func (w *Worker) Run(ctx context.Context) error {
 							}
 							stdouterr, err := cmd.CombinedOutput()
 							if err != nil {
-								r.Logs = string(stdouterr)
-								err = w.qid.UpdatePipelineResource(ctx, m.PipelineName, r.Canonical, r)
-								if err != nil {
-									w.logger.Log("error", fmt.Errorf("failed update Resource %q.%q from Pipeline %q: %w", r.Type, r.Name, m.PipelineName, err))
+								r.Logs = string(stdouterr) + "\n" + err.Error()
+								nerr := w.qid.UpdatePipelineResource(ctx, m.PipelineName, r.Canonical, r)
+								if nerr != nil {
+									w.logger.Log("error", fmt.Errorf("failed update Resource %q.%q from Pipeline %q: %w", r.Type, r.Name, m.PipelineName, nerr))
 								}
 								w.logger.Log("error", fmt.Errorf("failed to run command %q with args %q (%s): %w", rt.Check.Path, rt.Check.Args, stdouterr, err))
 								goto END