Merge pull request #496 from rusq/conn-pool

Preps for v3.1.0-alpha

Author: rusq

TODO.org

@@ -1,22 +1,24 @@
 * V3.1
-** TODO Database backend
+** DONE Database backend
+   CLOSED: [2025-04-08 Tue 21:54]
 *** DONE Add database backend support to Dump converter, it relies on the files
 *** DONE Check if the thread export works as intended.
 *** DONE Conversion to and from.
    CLOSED: [2025-03-11 Tue 20:53]
 *** DONE Files and avatars support during conversion.
    CLOSED: [2025-03-14 Fri 21:33]
-*** TODO Tests for dbproc and related packages.
+*** DONE Tests for dbproc and related packages.
+    CLOSED: [2025-04-08 Tue 21:54]
 **** DONE repository
      CLOSED: [2025-03-23 Sun 21:40]
 **** DONE control
      CLOSED: [2025-03-23 Sun 21:40]
-**** TODO dbproc
+**** DONE dbproc
+     CLOSED: [2025-04-08 Tue 21:53]
 *** DONE Resume will choke on export->database converted workspaces due to
     CLOSED: [2025-03-26 Wed 19:35]
     missing "workspace" information. Find another way to verify the
     workspace name, possibly by team ID on the messages/users.
-*** TODO Info command, prints out the database sessions, workspaces and message stats.
 ** DONE Source detection
    CLOSED: [2025-03-14 Fri 21:34]
 *** DONE detect standard file storage
@@ -35,24 +37,48 @@
 ** DONE Verify that consequent calls of conversation.replies don't have the first thread message as the item with 0-index.
    CLOSED: [2025-03-16 Sun 22:01]
    Yes it does: https://github.com/rusq/slackdump/issues/128#issuecomment-1234773510
-** DONE Only participating users #287.
+** DONE Only participating users #287, #125.
    CLOSED: [2025-03-21 Fri 22:05]
-** TODO Attachment filter, use the same terms as channel filter, -filter-files=[public,private,dm] (#233)
-** TODO Channel filter (private/public/DMs) -filter-channel=[public,private,dm] (controller already supports this)
-** Resume
+   Also solves #108 - dumping users with guest role.
+** DONE Resume
+   CLOSED: [2025-04-08 Tue 21:54]
 *** DONE user_not_found error when running with -channel-users
     CLOSED: [2025-03-21 Fri 22:32]
 *** DONE when resuming with -channel-users, and there are no new messages, the resume crashes with "no users collected"
     CLOSED: [2025-03-21 Fri 22:32]
+** DONE Formatter for all type of source data
+   CLOSED: [2025-04-08 Tue 21:55]
+** DONE Optional emoji download (-files=false) for emoji command
+** DONE Log colorisation (and output to stderr) (#467)
+   CLOSED: [2025-04-08 Tue 22:03]
+** DONE Converter from early v1.0.x format (as a tool subcommand)
+   CLOSED: [2025-04-08 Tue 22:02]
+
+** Fixes
+*** DONE Some attachment IDs can be strings, not just integers (#491, #492)
+    CLOSED: [2025-04-08 Tue 21:58]
+*** DONE Some bot messages caused panic (#482)
+    CLOSED: [2025-04-08 Tue 21:59]
+*** DONE Lack of synchronisation between Generator goroutine and controller, causing panic on premature termination (#479)
+    CLOSED: [2025-04-08 Tue 21:59]
+*** DONE Properly handle UnexpectedEOF error (#468)
+    CLOSED: [2025-04-08 Tue 22:03]
+*** DONE Parsing of the enterprise URLs (#461)
+    CLOSED: [2025-04-08 Tue 22:06]
 * V3.2
-** TODO Starred items and bookmarks.
+** Filters
+*** TODO Channel filter (private/public/DMs) -filter-channel=[public,private,dm] (controller already supports this)
+*** TODO Attachment filter, use the same terms as channel filter, -filter-files=[public,private,dm] (#233)
+** TODO bookmarks support
+** TODO Saved items
 ** TODO Connection pooling
 ** TODO load emojis alongside the messages during archive.
 ** TODO Static viewer output #416
 ** TODO Thumbnail download (#111)
 ** TODO Custom fields for users (#62)
+** TODO Info command, prints out the database sessions, workspaces and message stats.
+** TODO Aliasing users in the viewer, click on the username, assign a name, save (for archive source only).
 * Future
-** TODO Aliasing users in the viewer, click on the username, assign a name, save.
 ** TODO detect external users and fetch them individually
    even if running without -channel-users.
 ** TODO export compatibility (?)

cmd/slackdump/internal/archive/archive.go

@@ -13,7 +13,6 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/rusq/fsadapter"
 
-	"github.com/rusq/slackdump/v3"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
@@ -22,6 +21,7 @@ import (
 	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase/repository"
 	"github.com/rusq/slackdump/v3/internal/chunk/backend/directory"
 	"github.com/rusq/slackdump/v3/internal/chunk/control"
+	"github.com/rusq/slackdump/v3/internal/client"
 	"github.com/rusq/slackdump/v3/internal/convert/transform/fileproc"
 	"github.com/rusq/slackdump/v3/internal/structures"
 	"github.com/rusq/slackdump/v3/stream"
@@ -61,7 +61,7 @@ func runChunkArchive(ctx context.Context, _ *base.Command, args []string) error
 		base.SetExitStatus(base.SUserError)
 		return err
 	}
-	sess, err := bootstrap.SlackdumpSession(ctx)
+	client, err := bootstrap.Slack(ctx)
 	if err != nil {
 		base.SetExitStatus(base.SInitializationError)
 		return err
@@ -73,7 +73,7 @@ func runChunkArchive(ctx context.Context, _ *base.Command, args []string) error
 	}
 	defer cd.Close()
 
-	ctrl, err := ArchiveController(ctx, cd, sess)
+	ctrl, err := ArchiveController(ctx, cd, client)
 	if err != nil {
 		return err
 	}
@@ -93,7 +93,7 @@ func runDBArchive(ctx context.Context, cmd *base.Command, args []string) error {
 		base.SetExitStatus(base.SUserError)
 		return err
 	}
-	sess, err := bootstrap.SlackdumpSession(ctx)
+	client, err := bootstrap.Slack(ctx)
 	if err != nil {
 		base.SetExitStatus(base.SInitializationError)
 		return err
@@ -112,7 +112,7 @@ func runDBArchive(ctx context.Context, cmd *base.Command, args []string) error {
 
 	flags := control.Flags{MemberOnly: cfg.MemberOnly, RecordFiles: cfg.RecordFiles, ChannelUsers: cfg.OnlyChannelUsers}
 
-	ctrl, err := DBController(ctx, cmd, conn, sess, dirname, flags)
+	ctrl, err := DBController(ctx, cmd, conn, client, dirname, flags)
 	if err != nil {
 		return err
 	}
@@ -150,7 +150,7 @@ func NewDirectory(name string) (*chunk.Directory, error) {
 // parameters.
 //
 // Obscene, just obscene amount of arguments.
-func DBController(ctx context.Context, cmd *base.Command, conn *sqlx.DB, sess *slackdump.Session, dirname string, flags control.Flags, opts ...stream.Option) (RunCloser, error) {
+func DBController(ctx context.Context, cmd *base.Command, conn *sqlx.DB, client client.Slack, dirname string, flags control.Flags, opts ...stream.Option) (RunCloser, error) {
 	lg := cfg.Log
 	dbp, err := dbase.New(ctx, conn, bootstrap.SessionInfo(cmd.Name()))
 	if err != nil {
@@ -166,22 +166,22 @@ func DBController(ctx context.Context, cmd *base.Command, conn *sqlx.DB, sess *s
 	dl := fileproc.NewDownloader(
 		ctx,
 		cfg.WithFiles,
-		sess.Client(),
+		client,
 		fsadapter.NewDirectory(dirname),
 		lg,
 	)
 	// start avatar downloader
 	avdl := fileproc.NewDownloader(
 		ctx,
 		cfg.WithAvatars,
-		sess.Client(),
+		client,
 		fsadapter.NewDirectory(dirname),
 		lg,
 	)
 
 	ctrl, err := control.New(
 		ctx,
-		sess.Stream(sopts...),
+		stream.New(client, cfg.Limits, sopts...),
 		dbp,
 		control.WithFiler(fileproc.New(dl)),
 		control.WithAvatarProcessor(fileproc.NewAvatarProc(avdl)),
@@ -200,7 +200,7 @@ type RunCloser interface {
 
 // ArchiveController returns the default archive controller initialised based
 // on global configuration parameters.
-func ArchiveController(ctx context.Context, cd *chunk.Directory, sess *slackdump.Session, opts ...stream.Option) (*control.Controller, error) {
+func ArchiveController(ctx context.Context, cd *chunk.Directory, client client.Slack, opts ...stream.Option) (*control.Controller, error) {
 	lg := cfg.Log
 
 	sopts := []stream.Option{
@@ -214,15 +214,15 @@ func ArchiveController(ctx context.Context, cd *chunk.Directory, sess *slackdump
 	dl := fileproc.NewDownloader(
 		ctx,
 		cfg.WithFiles,
-		sess.Client(),
+		client,
 		fsadapter.NewDirectory(cd.Name()),
 		lg,
 	)
 	// start avatar downloader
 	avdl := fileproc.NewDownloader(
 		ctx,
 		cfg.WithAvatars,
-		sess.Client(),
+		client,
 		fsadapter.NewDirectory(cd.Name()),
 		lg,
 	)
@@ -231,7 +231,7 @@ func ArchiveController(ctx context.Context, cd *chunk.Directory, sess *slackdump
 
 	ctrl, err := control.New(
 		ctx,
-		sess.Stream(sopts...),
+		stream.New(client, cfg.Limits, sopts...),
 		erc,
 		control.WithLogger(lg),
 		control.WithFlags(control.Flags{MemberOnly: cfg.MemberOnly, RecordFiles: cfg.RecordFiles, ChannelUsers: cfg.OnlyChannelUsers}),
@@ -242,14 +242,6 @@ func ArchiveController(ctx context.Context, cd *chunk.Directory, sess *slackdump
 		return nil, err
 	}
 
-	// ctrl := control.NewDir(
-	// 	cd,
-	// 	sess.Stream(sopts...),
-	// 	control.WithLogger(lg),
-	// 	control.WithFlags(control.Flags{MemberOnly: cfg.MemberOnly, RecordFiles: cfg.RecordFiles}),
-	// 	control.WithFiler(fileproc.New(dl)),
-	// 	control.WithAvatarProcessor(fileproc.NewAvatarProc(avdl)),
-	// )
 	return ctrl, nil
 }
 

cmd/slackdump/internal/archive/search.go

@@ -7,18 +7,16 @@ import (
 	"strings"
 	"sync"
 
-	fileproc2 "github.com/rusq/slackdump/v3/internal/convert/transform/fileproc"
-
-	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
-
 	"github.com/rusq/fsadapter"
 	"github.com/schollz/progressbar/v3"
 
-	"github.com/rusq/slackdump/v3"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
+	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
 	"github.com/rusq/slackdump/v3/internal/chunk/control"
+	"github.com/rusq/slackdump/v3/internal/client"
+	"github.com/rusq/slackdump/v3/internal/convert/transform/fileproc"
 	"github.com/rusq/slackdump/v3/stream"
 )
 
@@ -100,13 +98,13 @@ func runSearch(ctx context.Context, cmd *base.Command, args []string, typ contro
 
 	cfg.Log.Info("running command", "cmd", cmd.Name())
 
-	sess, err := bootstrap.SlackdumpSession(ctx)
+	client, err := bootstrap.Slack(ctx)
 	if err != nil {
 		base.SetExitStatus(base.SInitializationError)
 		return err
 	}
 
-	ctrl, stop, err := searchControllerv31(ctx, cfg.Output, sess, args)
+	ctrl, stop, err := searchControllerv31(ctx, cfg.Output, client, args)
 	if err != nil {
 		base.SetExitStatus(base.SApplicationError)
 		return err
@@ -138,7 +136,7 @@ func (s stopFn) Stop() error {
 	return err
 }
 
-func searchControllerv31(ctx context.Context, dir string, sess *slackdump.Session, terms []string) (*control.Controller, stopFn, error) {
+func searchControllerv31(ctx context.Context, dir string, client client.Slack, terms []string) (*control.Controller, stopFn, error) {
 	var stop stopFn
 	if len(terms) == 0 {
 		base.SetExitStatus(base.SInvalidParameters)
@@ -158,10 +156,10 @@ func searchControllerv31(ctx context.Context, dir string, sess *slackdump.Sessio
 
 	lg := cfg.Log
 
-	dl := fileproc2.NewDownloader(
+	dl := fileproc.NewDownloader(
 		ctx,
 		cfg.WithFiles,
-		sess.Client(),
+		client,
 		fsadapter.NewDirectory(cd.Name()),
 		lg,
 	)
@@ -189,10 +187,10 @@ func searchControllerv31(ctx context.Context, dir string, sess *slackdump.Sessio
 
 	ctrl, err := control.New(
 		ctx,
-		sess.Stream(sopts...),
+		stream.New(client, cfg.Limits, sopts...),
 		erc,
 		control.WithLogger(lg),
-		control.WithFiler(fileproc2.New(dl)),
+		control.WithFiler(fileproc.New(dl)),
 		control.WithFlags(control.Flags{RecordFiles: cfg.RecordFiles}),
 	)
 	if err != nil {

cmd/slackdump/internal/bootstrap/slackdump.go

@@ -6,6 +6,7 @@ import (
 	"github.com/rusq/slackdump/v3"
 	"github.com/rusq/slackdump/v3/auth"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
+	"github.com/rusq/slackdump/v3/internal/client"
 )
 
 // SlackdumpSession returns the Slackdump Session initialised with the provider
@@ -18,7 +19,7 @@ func SlackdumpSession(ctx context.Context, opts ...slackdump.Option) (*slackdump
 		return nil, err
 	}
 
-	var stdOpts = []slackdump.Option{
+	stdOpts := []slackdump.Option{
 		slackdump.WithLogger(cfg.Log),
 		slackdump.WithForceEnterprise(cfg.ForceEnterprise),
 		slackdump.WithLimits(cfg.Limits),
@@ -31,3 +32,18 @@ func SlackdumpSession(ctx context.Context, opts ...slackdump.Option) (*slackdump
 		stdOpts...,
 	)
 }
+
+// Slack returns the Slack client initialised with the provider from context
+// and a standard set of options initialised from the configuration.
+func Slack(ctx context.Context, opts ...client.Option) (client.Slack, error) {
+	prov, err := auth.FromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, client.WithEnterprise(cfg.ForceEnterprise))
+	client, err := client.New(ctx, prov, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}

cmd/slackdump/internal/convertcmd/convert.go

@@ -8,9 +8,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/rusq/slackdump/v3/source"
+
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/golang/base"
-	"github.com/rusq/slackdump/v3/internal/source"
 )
 
 //go:embed assets/convert.md

cmd/slackdump/internal/convertcmd/to_chunk.go

@@ -11,14 +11,12 @@ import (
 	"text/tabwriter"
 	"time"
 
-	"github.com/rusq/slackdump/v3/internal/chunk/backend/directory"
-
-	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
-	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase/repository"
-
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/internal/chunk"
-	"github.com/rusq/slackdump/v3/internal/source"
+	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
+	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase/repository"
+	"github.com/rusq/slackdump/v3/internal/chunk/backend/directory"
+	"github.com/rusq/slackdump/v3/source"
 )
 
 func toChunk(ctx context.Context, src, trg string, cflg convertflags) error {

cmd/slackdump/internal/convertcmd/to_database.go

@@ -6,17 +6,15 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/rusq/fsadapter"
-
-	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
-	"github.com/rusq/slackdump/v3/internal/convert"
-
 	"github.com/jmoiron/sqlx"
+	"github.com/rusq/fsadapter"
 
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/bootstrap"
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/internal/chunk"
-	"github.com/rusq/slackdump/v3/internal/source"
+	"github.com/rusq/slackdump/v3/internal/chunk/backend/dbase"
+	"github.com/rusq/slackdump/v3/internal/convert"
+	"github.com/rusq/slackdump/v3/source"
 )
 
 // toDatabase converts the source to the database format.

cmd/slackdump/internal/convertcmd/to_dump.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/rusq/slackdump/v3/cmd/slackdump/internal/cfg"
 	"github.com/rusq/slackdump/v3/internal/convert"
-	"github.com/rusq/slackdump/v3/internal/source"
+	"github.com/rusq/slackdump/v3/source"
 )
 
 var ErrMeaningless = errors.New("meaningless conversion")