Merge pull request #593 from rusq/i62

rusq · web-flow · commit ec463873b6e5 · 2025-12-31T10:58:04.000+10:00
Custom User Profile Fields
diff --git a/cmd/slackdump/internal/archive/archive.go b/cmd/slackdump/internal/archive/archive.go
@@ -121,7 +121,13 @@ func runDBArchive(ctx context.Context, cmd *base.Command, args []string) error {
 	}
 	defer conn.Close()
 
-	flags := control.Flags{MemberOnly: cfg.MemberOnly, RecordFiles: cfg.RecordFiles, ChannelUsers: cfg.OnlyChannelUsers, ChannelTypes: cfg.ChannelTypes}
+	flags := control.Flags{
+		MemberOnly:    cfg.MemberOnly,
+		RecordFiles:   cfg.RecordFiles,
+		ChannelUsers:  cfg.OnlyChannelUsers,
+		IncludeLabels: cfg.IncludeCustomLabels,
+		ChannelTypes:  cfg.ChannelTypes,
+	}
 
 	ctrl, err := DBController(ctx, cmd.Name(), conn, client, dirname, flags, []stream.Option{})
 	if err != nil {
@@ -246,10 +252,11 @@ func ArchiveController(ctx context.Context, cd *chunk.Directory, client client.S
 	erc := directory.NewERC(cd, lg)
 
 	flags := control.Flags{
-		MemberOnly:   cfg.MemberOnly,
-		RecordFiles:  cfg.RecordFiles,
-		ChannelUsers: cfg.OnlyChannelUsers,
-		ChannelTypes: cfg.ChannelTypes,
+		MemberOnly:    cfg.MemberOnly,
+		RecordFiles:   cfg.RecordFiles,
+		ChannelUsers:  cfg.OnlyChannelUsers,
+		IncludeLabels: cfg.IncludeCustomLabels,
+		ChannelTypes:  cfg.ChannelTypes,
 	}
 
 	ctrl, err := control.New(
diff --git a/cmd/slackdump/internal/archive/archive_wizard.go b/cmd/slackdump/internal/archive/archive_wizard.go
@@ -35,6 +35,7 @@ func configuration() cfgui.Configuration {
 				cfgui.ChannelIDs(&entryList, false),
 				cfgui.MemberOnly(),
 				cfgui.OnlyChannelUsers(),
+				cfgui.IncludeCustomLabels(),
 				cfgui.RecordFiles(),
 				cfgui.Avatars(),
 				cfgui.ChannelTypes(),
diff --git a/cmd/slackdump/internal/cfg/cfg.go b/cmd/slackdump/internal/cfg/cfg.go
@@ -36,8 +36,9 @@ var (
 	MachineIDOvr    string // Machine ID override
 	NoEncryption    bool   // disable encryption
 
-	MemberOnly       bool
-	OnlyChannelUsers bool
+	MemberOnly          bool
+	OnlyChannelUsers    bool
+	IncludeCustomLabels bool // Requests labels for user custom fields, use with caution due to request throttling.
 	// ChannelTypes lists channel types to fetch.
 	ChannelTypes = slackChanTypes(slackdump.AllChanTypes)
 
@@ -198,6 +199,7 @@ func SetBaseFlags(fs *flag.FlagSet, mask FlagMask) {
 	if mask&OmitCustomUserFlags == 0 {
 		fs.BoolVar(&MemberOnly, "member-only", false, "export only channels, which the current user belongs to (if no channels are specified)")
 		fs.BoolVar(&OnlyChannelUsers, "channel-users", false, "export only users involved in the channel, and skip fetching of all users")
+		fs.BoolVar(&IncludeCustomLabels, "custom-labels", false, "request user's custom fields labels, may result in requests being throttled hard")
 	}
 	if mask&OmitChunkFileMode == 0 {
 		fs.BoolVar(&UseChunkFiles, "legacy", false, "use chunk files for data storage instead of sqlite database (incompatible with resuming)")
diff --git a/cmd/slackdump/internal/export/v3.go b/cmd/slackdump/internal/export/v3.go
@@ -79,10 +79,11 @@ func exportv31(ctx context.Context, sess client.Slack, fsa fsadapter.FS, list *s
 	)
 
 	flags := control.Flags{
-		MemberOnly:   cfg.MemberOnly,
-		RecordFiles:  false, // archive format is transitory, don't need extra info.
-		ChannelUsers: cfg.OnlyChannelUsers,
-		ChannelTypes: cfg.ChannelTypes,
+		MemberOnly:    cfg.MemberOnly,
+		RecordFiles:   false, // archive format is transitory, don't need extra info.
+		ChannelUsers:  cfg.OnlyChannelUsers,
+		ChannelTypes:  cfg.ChannelTypes,
+		IncludeLabels: cfg.IncludeCustomLabels,
 	}
 	ctr, err := control.New(
 		ctx,
@@ -167,10 +168,11 @@ func export(ctx context.Context, sess client.Slack, fsa fsadapter.FS, list *stru
 	)
 
 	flags := control.Flags{
-		MemberOnly:   cfg.MemberOnly,
-		RecordFiles:  false, // archive format is transitory, don't need extra info.
-		ChannelUsers: cfg.OnlyChannelUsers,
-		ChannelTypes: cfg.ChannelTypes,
+		MemberOnly:    cfg.MemberOnly,
+		RecordFiles:   false, // archive format is transitory, don't need extra info.
+		ChannelUsers:  cfg.OnlyChannelUsers,
+		IncludeLabels: cfg.IncludeCustomLabels,
+		ChannelTypes:  cfg.ChannelTypes,
 	}
 	ctr := control.NewDir(
 		chunkdir,
diff --git a/cmd/slackdump/internal/export/wizard.go b/cmd/slackdump/internal/export/wizard.go
@@ -52,6 +52,7 @@ func (fl *exportFlags) configuration() cfgui.Configuration {
 				},
 				cfgui.MemberOnly(),
 				cfgui.OnlyChannelUsers(),
+				cfgui.IncludeCustomLabels(),
 				cfgui.Avatars(),
 				{
 					Name:        "Export Token",
diff --git a/cmd/slackdump/internal/resume/resume.go b/cmd/slackdump/internal/resume/resume.go
@@ -125,9 +125,11 @@ func runResume(ctx context.Context, cmd *base.Command, args []string) error {
 	defer wconn.Close()
 
 	cf := control.Flags{
-		Refresh:      resumeFlags.Refresh,
-		ChannelUsers: cfg.OnlyChannelUsers,
-		ChannelTypes: cfg.ChannelTypes,
+		Refresh:       resumeFlags.Refresh,
+		ChannelUsers:  cfg.OnlyChannelUsers,
+		ChannelTypes:  cfg.ChannelTypes,
+		IncludeLabels: cfg.IncludeCustomLabels,
+		MemberOnly:    cfg.MemberOnly,
 	}
 	// inclusive is false, because we don't want to include the latest message
 	// which is already in the database.
@@ -146,10 +148,6 @@ func runResume(ctx context.Context, cmd *base.Command, args []string) error {
 	return nil
 }
 
-// oldestAdjustment is one second adjustment for Oldest timestamp to allow for
-// fetching thread messages (see #584)
-const oldestAdjustment = -1 * time.Hour
-
 func latest(ctx context.Context, src source.Resumer, includeThreads bool, lookBack time.Duration) (*structures.EntityList, error) {
 	if lookBack > 0 {
 		lookBack = -lookBack
diff --git a/cmd/slackdump/internal/resume/wizard.go b/cmd/slackdump/internal/resume/wizard.go
@@ -54,6 +54,7 @@ func configuration() cfgui.Configuration {
 			Name: "Optional parameters",
 			Params: []cfgui.Parameter{
 				cfgui.OnlyChannelUsers(),
+				cfgui.IncludeCustomLabels(),
 				cfgui.Avatars(),
 			},
 		},
diff --git a/cmd/slackdump/internal/ui/cfgui/common_params.go b/cmd/slackdump/internal/ui/cfgui/common_params.go
@@ -98,3 +98,12 @@ func ChannelTypes() Parameter {
 			Options(options...)),
 	}
 }
+
+func IncludeCustomLabels() Parameter {
+	return Parameter{
+		Name:        "Include Custom Field Labels",
+		Value:       Checkbox(cfg.IncludeCustomLabels),
+		Description: "Channel users custom user profile fields labels (may result in request throttling).",
+		Updater:     updaters.NewBool(&cfg.IncludeCustomLabels),
+	}
+}
diff --git a/internal/chunk/control/control.go b/internal/chunk/control/control.go
@@ -93,8 +93,9 @@ func (c *Controller) mkSuperprocessor(ctx context.Context, rec *chunk.Recorder,
 		ucoll := newMsgUserIDsCollector()
 		conv = processor.PrependMessenger(conv, ucoll)
 		streamer = &userCollectingStreamer{
-			Streamer: streamer,
-			userIDC:  ucoll.C(),
+			Streamer:      streamer,
+			userIDC:       ucoll.C(),
+			includeLabels: c.flags.IncludeLabels,
 		}
 	}
 
diff --git a/internal/chunk/control/interfaces.go b/internal/chunk/control/interfaces.go
@@ -23,6 +23,7 @@ type Streamer interface {
 	SearchMessages(ctx context.Context, proc processor.MessageSearcher, query string) error
 	SearchFiles(ctx context.Context, proc processor.FileSearcher, query string) error
 	UsersBulk(ctx context.Context, proc processor.Users, ids ...string) error
+	UsersBulkWithCustom(ctx context.Context, proc processor.Users, includeLabels bool, ids ...string) error
 }
 
 type TransformStarter interface {
diff --git a/internal/chunk/control/mock_control/mock_interfaces.go b/internal/chunk/control/mock_control/mock_interfaces.go
diff --git a/internal/chunk/control/runner.go b/internal/chunk/control/runner.go
@@ -34,6 +34,9 @@ type Flags struct {
 	// ChannelTypes is the list of channel types to fetch.  If empty, all
 	// channel types are fetched.
 	ChannelTypes []string
+	// IncludeLabels requests API to include the labels for the custom fields.
+	// works only with ChannelUsers. Server may throttle requests hard.
+	IncludeLabels bool
 }
 
 // Error is a controller error.
diff --git a/internal/chunk/control/stream.go b/internal/chunk/control/stream.go
@@ -15,7 +15,8 @@ import (
 // the userIDC channel and calls the Users processor method.
 type userCollectingStreamer struct {
 	Streamer
-	userIDC <-chan []string
+	userIDC       <-chan []string
+	includeLabels bool
 }
 
 // Users is the override for the Streamer.Users method.
@@ -28,7 +29,7 @@ func (u *userCollectingStreamer) Users(ctx context.Context, proc processor.Users
 			if !ok {
 				return nil
 			}
-			if err := u.UsersBulk(ctx, proc, ids...); err != nil {
+			if err := u.UsersBulkWithCustom(ctx, proc, u.includeLabels, ids...); err != nil {
 				return err
 			}
 		}
diff --git a/internal/chunk/control/stream_test.go b/internal/chunk/control/stream_test.go
@@ -18,7 +18,8 @@ func Test_userCollectingStreamer_Users(t *testing.T) {
 
 	type fields struct {
 		// Streamer Streamer
-		userIDC <-chan []string
+		userIDC       <-chan []string
+		includeLabels bool
 	}
 	type args struct {
 		ctx context.Context
@@ -52,7 +53,26 @@ func Test_userCollectingStreamer_Users(t *testing.T) {
 				userIDC <- []string{"U12345678", "U87654321"}
 			},
 			expectFn: func(ms *mock_control.MockStreamer, mup *mock_processor.MockUsers) {
-				ms.EXPECT().UsersBulk(gomock.Any(), mup, "U12345678", "U87654321").Return(nil)
+				ms.EXPECT().UsersBulkWithCustom(gomock.Any(), mup, false, "U12345678", "U87654321").Return(nil)
+			},
+			wantErr: false,
+		},
+		{
+			name: "propagates include labels",
+			fields: fields{
+				includeLabels: true,
+			},
+			args: args{
+				ctx: t.Context(),
+			},
+			prepFn: func(f *fields) {
+				userIDC := make(chan []string, 1)
+				defer close(userIDC)
+				f.userIDC = userIDC
+				userIDC <- []string{"U12345678", "U87654321"}
+			},
+			expectFn: func(ms *mock_control.MockStreamer, mup *mock_processor.MockUsers) {
+				ms.EXPECT().UsersBulkWithCustom(gomock.Any(), mup, true, "U12345678", "U87654321").Return(nil)
 			},
 			wantErr: false,
 		},
@@ -68,7 +88,7 @@ func Test_userCollectingStreamer_Users(t *testing.T) {
 				userIDC <- []string{"U12345678", "U87654321"}
 			},
 			expectFn: func(ms *mock_control.MockStreamer, mup *mock_processor.MockUsers) {
-				ms.EXPECT().UsersBulk(gomock.Any(), mup, "U12345678", "U87654321").Return(assert.AnError)
+				ms.EXPECT().UsersBulkWithCustom(gomock.Any(), mup, false, "U12345678", "U87654321").Return(assert.AnError)
 			},
 			wantErr: true,
 		},
@@ -85,8 +105,9 @@ func Test_userCollectingStreamer_Users(t *testing.T) {
 				tt.prepFn(&tt.fields)
 			}
 			u := &userCollectingStreamer{
-				Streamer: ms,
-				userIDC:  tt.fields.userIDC,
+				Streamer:      ms,
+				userIDC:       tt.fields.userIDC,
+				includeLabels: tt.fields.includeLabels,
 			}
 			if err := u.Users(tt.args.ctx, mup, tt.args.opt...); (err != nil) != tt.wantErr {
 				t.Errorf("userCollectingStreamer.Users() error = %v, wantErr %v", err, tt.wantErr)
diff --git a/internal/client/client.go b/internal/client/client.go
@@ -30,6 +30,7 @@ type Slack interface {
 	ListBookmarks(channelID string) ([]slack.Bookmark, error)
 	SearchFilesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchFiles, error)
 	SearchMessagesContext(ctx context.Context, query string, params slack.SearchParameters) (*slack.SearchMessages, error)
+	GetUserProfileContext(ctx context.Context, params *slack.GetUserProfileParameters) (*slack.UserProfile, error)
 }
 
 // SlackClienter is an extended interface that includes Client method that
diff --git a/internal/client/mock_client/mock_client.go b/internal/client/mock_client/mock_client.go
diff --git a/internal/client/pool.go b/internal/client/pool.go
@@ -101,3 +101,7 @@ func (p *Pool) GetFileInfoContext(ctx context.Context, fileID string, count int,
 func (p *Pool) GetUserInfoContext(ctx context.Context, user string) (*slack.User, error) {
 	return p.next().GetUserInfoContext(ctx, user)
 }
+
+func (w *Pool) GetUserProfileContext(ctx context.Context, params *slack.GetUserProfileParameters) (*slack.UserProfile, error) {
+	return w.next().GetUserProfileContext(ctx, params)
+}
diff --git a/internal/edge/wrapper.go b/internal/edge/wrapper.go
@@ -89,3 +89,7 @@ func (w *Wrapper) GetFileInfoContext(ctx context.Context, fileID string, count i
 func (w *Wrapper) GetUserInfoContext(ctx context.Context, user string) (*slack.User, error) {
 	return w.cl.GetUserInfoContext(ctx, user)
 }
+
+func (w *Wrapper) GetUserProfileContext(ctx context.Context, params *slack.GetUserProfileParameters) (*slack.UserProfile, error) {
+	return w.cl.GetUserProfileContext(ctx, params)
+}
diff --git a/stream/stream.go b/stream/stream.go
diff --git a/stream/stream_test.go b/stream/stream_test.go

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,7 @@ func (fl *exportFlags) configuration() cfgui.Configuration {`
`52`	`52`	`},`
`53`	`53`	`cfgui.MemberOnly(),`
`54`	`54`	`cfgui.OnlyChannelUsers(),`
	`55`	`+ cfgui.IncludeCustomLabels(),`
`55`	`56`	`cfgui.Avatars(),`
`56`	`57`	`{`
`57`	`58`	`Name: "Export Token",`
Original file line number	Diff line number	Diff line change
`@@ -98,3 +98,12 @@ func ChannelTypes() Parameter {`
`98`	`98`	`Options(options...)),`
`99`	`99`	`}`
`100`	`100`	`}`
	`101`	`+`
	`102`	`+func IncludeCustomLabels() Parameter {`
	`103`	`+ return Parameter{`
	`104`	`+ Name: "Include Custom Field Labels",`
	`105`	`+ Value: Checkbox(cfg.IncludeCustomLabels),`
	`106`	`+ Description: "Channel users custom user profile fields labels (may result in request throttling).",`
	`107`	`+ Updater: updaters.NewBool(&cfg.IncludeCustomLabels),`
	`108`	`+ }`
	`109`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -93,8 +93,9 @@ func (c Controller) mkSuperprocessor(ctx context.Context, rec chunk.Recorder,`
`93`	`93`	`ucoll := newMsgUserIDsCollector()`
`94`	`94`	`conv = processor.PrependMessenger(conv, ucoll)`
`95`	`95`	`streamer = &userCollectingStreamer{`
`96`		`- Streamer: streamer,`
`97`		`- userIDC: ucoll.C(),`
	`96`	`+ Streamer: streamer,`
	`97`	`+ userIDC: ucoll.C(),`
	`98`	`+ includeLabels: c.flags.IncludeLabels,`
`98`	`99`	`}`
`99`	`100`	`}`
`100`	`101`
Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ type Streamer interface {`
`23`	`23`	`SearchMessages(ctx context.Context, proc processor.MessageSearcher, query string) error`
`24`	`24`	`SearchFiles(ctx context.Context, proc processor.FileSearcher, query string) error`
`25`	`25`	`UsersBulk(ctx context.Context, proc processor.Users, ids ...string) error`
	`26`	`+ UsersBulkWithCustom(ctx context.Context, proc processor.Users, includeLabels bool, ids ...string) error`
`26`	`27`	`}`
`27`	`28`
`28`	`29`	`type TransformStarter interface {`