Merge pull request #513 from rusq/fixes

Brush up Convert and Transform APIs + doc

Author: rusq

cmd/slackdump/internal/apiconfig/assets/config_new.md

@@ -0,0 +1,84 @@
+# "New" Command
+
+Creates a new API configuration file containing default values. You will need
+to specify the filename, for example:
+
+    slackdump config new myconfig.toml
+
+If the extension is omitted, ".toml" is automatically appended to the name of
+the file.
+
+Configuration file contains the following groups of settings:
+- File download concurrency and retries;
+- Rate limits;
+- Batch sizes per request;
+
+### Slack Rate Limits
+Slack imposes rate limits on API calls. The default values are set to the
+maximum allowed by Slack. If you want to change the rate limits, you can do so
+in the configuration file.
+
+Slack API has four tiers of [rate limits][1], with Tier 1 being the most
+restrictive and Tier 4 being the least restrictive. The rate limits are
+measured in requests per minute and enforced on a (token,method) pair.
+
+Let's look at the example configuration file:
+
+```toml
+# File download settings
+workers = 4
+download_retries = 3
+
+# Rate limits
+[tier_2]
+  boost = 20
+  burst = 3
+  retries = 20
+
+[tier_3]
+  boost = 60
+  burst = 5
+  retries = 3
+
+[tier_4]
+  boost = 10
+  burst = 7
+  retries = 3
+
+# Batch size settings
+[per_request]
+  conversations = 100
+  channels = 100
+  replies = 200
+```
+
+The base Tier values are hardcoded in the application, but the configuration
+file allows to tweak the "boost" and "burst" values for each tier.
+
+The "boost" value is the number of requests that slackdump will make *on top*
+of the base rate limit. **For example**: "Slack Web API Tier 2" has base limit
+of 20+ requests per minute, but if "boost" is set to 20, Slackdump will make 40
+requests per minute.
+
+The "burst" value is the number of requests that slackdump will make *in
+addition* to the base rate limit and "boost".  It is passed directly as an
+argument to the [rate limiting library][2].
+
+"Retries" is the number of time Slackdump will retry the request if it fails
+with a **recoverable** error.  Recoverable errors are:
+- Rate limit exceeded;
+- Unexpected network disconnect;
+- Network error (timeout, connection refused, etc.);
+- HTTP errors:  408, 500, 502 - 599.
+
+If Slackdump receives a recoverable error, it will do one of the following:
+- If it is a rate limit error, it will wait for the specified amount of time
+  and retry the request;
+- For network related errors, it will use the exponential backoff algorithm to
+  wait and retry the request up to a limit of 5 minutes.
+- For other types of recoverable errors, it will use the cubic backoff, capped
+  at 5 minutes as well.
+
+
+[1]: https://api.slack.com/apis/rate-limits
+[2]: https://pkg.go.dev/golang.org/x/time/rate#NewLimiter

cmd/slackdump/internal/apiconfig/new.go

@@ -2,6 +2,7 @@ package apiconfig
 
 import (
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"os"
@@ -14,20 +15,13 @@ import (
 	"github.com/rusq/slackdump/v3/internal/network"
 )
 
-var CmdConfigNew = &base.Command{
-	UsageLine: "slackdump config new",
-	Short:     "creates a new API config with the default values",
-	Long: `
-# "New" Command
-
-Creates a new API configuration file containing default values. You will need
-to specify the filename, for example:
+//go:embed assets/config_new.md
+var configNewMD string
 
-    slackdump config new myconfig.toml
-
-If the extension is omitted, ".toml" is automatically appended to the name of
-the file.
-`,
+var CmdConfigNew = &base.Command{
+	UsageLine:  "slackdump config new",
+	Short:      "creates a new API config with the default values",
+	Long:       configNewMD,
 	FlagMask:   cfg.OmitAll,
 	PrintFlags: true,
 }

cmd/slackdump/internal/archive/assets/archive.md

@@ -1,35 +1,71 @@
 # Archive Command
 
-The `archive` command saves your Slack workspace as a directory of files. By
+The `archive` command saves your Slack workspace as a SQLite database. By
 default, it archives the entire workspace that your user can access. You can
 customize the archive to include specific channels, groups, or direct messages
-by providing their URLs or IDs.
+by providing their URLs or IDs on the command line or in the Wizard.
+
+The database is located in `slackdump.sqlite` in the output directory.
+
+Alternatively, you can use `-legacy` flag to archive into chunk file format, if
+you experience problems with the database.  Note, that `-legacy` flag is
+temporary for the transition period, and will be removed in v3.2.0.
+<!-- TODO: remove above paragraph once -legacy is deleted -->
 
 The benefits of using "Archive" over "Export" and "Dump" are:
-- it is well documented (see `slackdump help chunk`);
+- it is faster, because it does not need to convert the data to export or dump
+  formats;
+- the database can be easily queried using SQL using SQLite CLI or SQLite
+  Browser;
+- the archiving can be "resumed" by `resume` command, meaning that you can
+  continue where you left off in case of previous failure or incremental
+  backups.
 - it can be converted to other formats, including the native Slack Export
   format (see `slackdump help convert`);
-- it is easier to parse with tools like `jq` or `grep`;
 - it is more convenient to build your own tools around it;
 - it is used internally by Slackdump to generate Slack Export and dump files,
   so it can be seen as "master" format for the data;
-- It is natively supported by `slackdump view` command, everything else uses
-  an adapter.
 
 ## Features
 
-### Default Behaviour
-- Archives the full workspace accessible to your user.
+## Database Archive Contents
+
+The archive contains the following files:
+
+- **`slackdump.sqlite`**: The SQLite database file containing all the data
+  from the workspace.
+- **`__uploads`**: A directory containing files attached to messages that were
+  downloaded, if the file download is enabled.
+- **`__avatars`**: A directory containing user avatars that were downloaded,
+  if the avatar download is enabled.
+
+Sometimes you might see `slackdump.sqlite-shm` and `slackdump.sqlite-wal` files
+in the output directory. These are temporary files created by SQLite for
+performance reasons. They are not necessary for the archive, unless Slackdump
+was interrupted or crashed. You can safely delete them if you are sure that the
+archive is complete.
 
-### Optional Customization
-- Specify channels, groups, or DMs to archive by providing their URLs or IDs.
+### Database Structure
 
-### Output Format
-- The archive uses the **"Chunk" format**, which can be:
-  - Viewed using the `view` command.
-  - Converted to other formats, including the native Slack Export format.
+The database contains the following tables:
+- **CHANNEL**:  Contains all channels in the workspace.
+- **CHANNEL_USER**:  Contains the mapping between channels and users.
+- **CHUNK**:  Contains the "chunk" metadata, including the chunk type, number
+  of records retrieved and the SESSION ID.
+- **FILE**:  Contains all discovered file metadata from messages.
+- **MESSAGE**:  Contains all messages and thread messages from the workspace.
+- **SEARCH_FILE**:  Contains search results for files.
+- **SEARCH_MESSAGE**:  Contains search results for messages.
+- **SESSION**:  Contains the session information, including the start and end
+  time of the period.
+- **S_USER**:  Contains all users in the workspace.
+- **WORKSPACE**: Contains the workspace information, including the workspace ID
+  and name.
 
-## Archive Contents
+There are also additional views, starting with `V_`, which are used by Slackdump
+during the archiving process, they should not be removed or modified.
+
+## Legacy Archive Contents
 
 The archive behaves like the Slackdump export feature. A successful run
 output includes:
@@ -49,8 +85,17 @@ output includes:
 - Note: The `archive` command does not create ZIP files, but you can manually
   compress the output directory into a ZIP file if needed.
 
-## What is the Chunk Format?
+For details on this format, run:  `slackdump help chunk`
+
+## Migrating from v3.x
+If you're using Chunk files in your tooling, you can convert the database to the
+chunk format using the `convert` command. For example:
+```bash
+slackdump convert -f chunk ./slackdump_20211231_123456
+```
 
-The Chunk format is a specific structure used for archiving data. For details
-on this format, run:  `slackdump help chunk`
+or
 
+```bash
+slackdump convert -f chunk ./slackdump_20211231_123456/slackdump.sqlite
+```

cmd/slackdump/internal/archive/assets/search.md

@@ -5,15 +5,15 @@ Slack workspace based on specified query terms. This command supports searching
 for messages, files, or both, and outputs the results in a directory.
 
 ### Subcommands
-- **`slackdump search messages`**: Searches and records messages matching the
-  given query.
-- **`slackdump search files`**: Searches and records files matching the given
+- **`slackdump search messages <query>`**: Searches and records messages
+  matching the given query.
+- **`slackdump search files <query>`**: Searches and records files matching the given
   query.
-- **`slackdump search all`**: Searches and records both messages and files
+- **`slackdump search all <query>`**: Searches and records both messages and files
   matching the query.
 
 ### Flags
-- **`--no-channel-users`**: Skips retrieving user data for channels, making the
+- **`-no-channel-users`**: Skips retrieving user data for channels, making the
   process approximately 2.5x faster.
 
 ### Requirements
@@ -41,16 +41,14 @@ slackdump search all "project updates"
 ```
 
 ### Faster Searches
-To speed up searches, add the `--no-channel-users` flag:
+To speed up searches, add the `-no-channel-users` flag:
 
 ```bash
 slackdump search messages -no-channel-users "status update"
 ```
 
 
-## Output Directory
-The search command outputs results to the specified directory. The directory
-contains:
-
-- **`search.jsonl.gz`**: A list of messages matching the query.
-- directory with saved files (if files are included in the search).
+## Output
+The search command outputs results to the database in the output directory.
+See `slackdump help archive` for details on the database structure and
+contents.

cmd/slackdump/internal/convertcmd/assets/convert.md

@@ -1,10 +1,39 @@
 # Convert Command
 
-Convert Slackdump Archive format to other supported formats.
+Converts between different Slackdump supported formats.
 
-By default it converts a directory with Slackdump archive to a ZIP-archive or
-directory in Slack Export format.
+## Usage
+```bash
+slackdump convert [-f format] [-o output] <input>
+```
 
-Reference:
-- Slackdump archive "chunk" format: `slackdump help chunk`
+Where format is one of the following:
+- `chunk` **Chunk format**: JSON.GZ files with metadata, output is a directory.
+- `database` **SQLite database**: SQLite database format used by Slackdump, output is a directory.
+- `dump` **Dump**: JSON files where each channel is a large JSON object. Output is a directory or a zip file.
+- `export`: **Slack Export**: The native Slack export format. Output is a directory or a zip file.
+
+By default Slackdump converts to Slack Export format and writes to a ZIP file
+output.
+
+If any files were saved in the source location, they will be copied to the target directory or ZIP file, unless
+`-files=false` is specified.
+
+To copy avatars, use `-avatars` flag.  By default, avatars are not copied.
+
+## Example
+
+Convert Slack Export to database format:
+```bash
+slackdump convert -f database -o MyArchive/ slack_export.zip
+```
+
+Converting from database format to Slack Export format:
+```bash
+slackdump convert -f export -o my_archive.zip slackdump_20211231_150405/
+```
+Note, that there's no necessity to specify the "slackdump.sqlite" file, Slackdump
+will automatically find it in the directory.
+
+See also:
 - `slackdump help archive`

cmd/slackdump/internal/dump/dump.go

@@ -204,7 +204,7 @@ func dumpv3(ctx context.Context, sess client.Slack, fsa fsadapter.FS, p dumppara
 		opts = append(opts, transform.DumpWithPipeline(subproc.PathUpdateFunc))
 	}
 
-	tf, err := transform.NewDumpConverter(fsa, src, opts...)
+	tf, err := transform.NewDump(fsa, src, opts...)
 	if err != nil {
 		return fmt.Errorf("failed to create transform: %w", err)
 	}
@@ -301,7 +301,7 @@ func dumpv31(ctx context.Context, client client.Slack, fsa fsadapter.FS, p dumpp
 		opts = append(opts, transform.DumpWithPipeline(subproc.PathUpdateFunc))
 	}
 
-	tf, err := transform.NewDumpConverter(fsa, src, opts...)
+	tf, err := transform.NewDump(fsa, src, opts...)
 	if err != nil {
 		return fmt.Errorf("failed to create transform: %w", err)
 	}

cmd/slackdump/internal/format/format.go

@@ -59,7 +59,7 @@ func runFormat(ctx context.Context, cmd *base.Command, args []string) error {
 		return err
 	} else {
 		var ok bool
-		formatterInit, ok := format.Converters[convType]
+		formatterInit, ok := convType.FormatFunc()
 		if !ok {
 			base.SetExitStatus(base.SInvalidParameters)
 			return errors.New("unknown converter type")

cmd/slackdump/internal/list/common.go

@@ -149,7 +149,7 @@ func saveData(ctx context.Context, data any, filename string, typ format.Type, u
 // users by ID.
 func fmtPrint(ctx context.Context, w io.Writer, a any, typ format.Type, u []slack.User, bare bool) error {
 	// get the converter
-	initFn, ok := format.Converters[typ]
+	initFn, ok := typ.FormatFunc()
 	if !ok {
 		return fmt.Errorf("unknown converter type: %s", typ)
 	}

internal/chunk/backend/directory/conversations.go

@@ -187,7 +187,8 @@ func (cv *Conversations) finalise(ctx context.Context, id chunk.FileID) error {
 		return err
 	}
 	if cv.tf != nil {
-		return cv.tf.Transform(ctx, id)
+		channelID, threadTS := id.Split()
+		return cv.tf.Transform(ctx, channelID, threadTS)
 	}
 	return nil
 }

internal/chunk/backend/directory/conversations_test.go

@@ -490,7 +490,7 @@ func TestConversations_finalise(t *testing.T) {
 			expectFn: func(mt *Mocktracker, mtf *mock_chunk.MockTransformer) {
 				mt.EXPECT().RefCount(chunk.FileID("fileID")).Return(0)
 				mt.EXPECT().Unregister(chunk.FileID("fileID")).Return(nil)
-				mtf.EXPECT().Transform(gomock.Any(), chunk.FileID("fileID")).Return(nil)
+				mtf.EXPECT().Transform(gomock.Any(), "fileID", "").Return(nil)
 			},
 			wantErr: false,
 		},
@@ -538,7 +538,7 @@ func TestConversations_finalise(t *testing.T) {
 			expectFn: func(mt *Mocktracker, mtf *mock_chunk.MockTransformer) {
 				mt.EXPECT().RefCount(chunk.FileID("fileID")).Return(0)
 				mt.EXPECT().Unregister(chunk.FileID("fileID")).Return(nil)
-				mtf.EXPECT().Transform(gomock.Any(), chunk.FileID("fileID")).Return(errors.New("transform error"))
+				mtf.EXPECT().Transform(gomock.Any(), "fileID", "").Return(errors.New("transform error"))
 			},
 			wantErr: true,
 		},