From cdb864a933f4750245980eb1d5ac9793e0f3b6a2 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Tue, 22 Feb 2022 21:00:57 +1000 Subject: [PATCH 1/5] getting conversation name --- clienter_mock.go | 15 ++++++++ messages.go | 27 +++++++++++++- messages_test.go | 97 +++++++++++++++++++++++++++++++++++++++++++++--- slackdump.go | 1 + 4 files changed, 133 insertions(+), 7 deletions(-) diff --git a/clienter_mock.go b/clienter_mock.go index e516da45..6e7d00d7 100644 --- a/clienter_mock.go +++ b/clienter_mock.go @@ -51,6 +51,21 @@ func (mr *mockClienterMockRecorder) GetConversationHistoryContext(ctx, params in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationHistoryContext", reflect.TypeOf((*mockClienter)(nil).GetConversationHistoryContext), ctx, params) } +// GetConversationInfoContext mocks base method. +func (m *mockClienter) GetConversationInfoContext(ctx context.Context, channelID string, includeLocale bool) (*slack.Channel, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConversationInfoContext", ctx, channelID, includeLocale) + ret0, _ := ret[0].(*slack.Channel) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetConversationInfoContext indicates an expected call of GetConversationInfoContext. +func (mr *mockClienterMockRecorder) GetConversationInfoContext(ctx, channelID, includeLocale interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConversationInfoContext", reflect.TypeOf((*mockClienter)(nil).GetConversationInfoContext), ctx, channelID, includeLocale) +} + // GetConversationRepliesContext mocks base method. func (m *mockClienter) GetConversationRepliesContext(ctx context.Context, params *slack.GetConversationRepliesParameters) ([]slack.Message, bool, string, error) { m.ctrl.T.Helper() diff --git a/messages.go b/messages.go index 5b1b5ac0..b79475a2 100644 --- a/messages.go +++ b/messages.go @@ -37,6 +37,7 @@ type Channel = Conversation // Conversation keeps the slice of messages. type Conversation struct { + Name string `json:"name"` Messages []Message `json:"messages"` // ID is the channel ID. ID string `json:"channel_id"` @@ -178,7 +179,25 @@ func (sd *SlackDumper) dumpMessages(ctx context.Context, channelID string, oldes sortMessages(messages) - return &Conversation{Messages: messages, ID: channelID}, nil + name, err := sd.getChannelName(ctx, sd.limiter(tier3), channelID) + if err != nil { + return nil, err + } + + return &Conversation{Name: name, Messages: messages, ID: channelID}, nil +} + +func (sd *SlackDumper) getChannelName(ctx context.Context, l *rate.Limiter, channelID string) (string, error) { + // get channel name + var ci *slack.Channel + if err := withRetry(ctx, l, sd.options.Tier3Retries, func() error { + var err error + ci, err = sd.client.GetConversationInfoContext(ctx, channelID, false) + return err + }); err != nil { + return "", err + } + return ci.NameNormalized, nil } // convHistoryParams returns GetConversationHistoryParameters. @@ -309,7 +328,13 @@ func (sd *SlackDumper) DumpThread(ctx context.Context, channelID, threadTS strin sortMessages(threadMsgs) + name, err := sd.getChannelName(ctx, sd.limiter(tier3), channelID) + if err != nil { + return nil, err + } + return &Conversation{ + Name: name, Messages: threadMsgs, ID: channelID, ThreadTS: threadTS, diff --git a/messages_test.go b/messages_test.go index dc33d69e..43ab046e 100644 --- a/messages_test.go +++ b/messages_test.go @@ -172,9 +172,11 @@ func TestSlackDumper_DumpMessages(t *testing.T) { }, }, nil) + mockConvInfo(c, "CHANNEL", "channel_name") }, &Conversation{ - ID: "CHANNEL", + Name: "channel_name", + ID: "CHANNEL", Messages: []Message{ testMsg1, testMsg2, @@ -232,9 +234,11 @@ func TestSlackDumper_DumpMessages(t *testing.T) { nil, ). After(first) + mockConvInfo(c, "CHANNEL", "channel_name") }, &Conversation{ - ID: "CHANNEL", + Name: "channel_name", + ID: "CHANNEL", Messages: []Message{ testMsg1, testMsg2, @@ -593,8 +597,9 @@ func TestSlackDumper_DumpThread(t *testing.T) { nil, ). Times(1) + mockConvInfo(mc, "CHANNEL", "channel_name") }, - &Conversation{ID: "CHANNEL", ThreadTS: "THREAD", Messages: []Message{testMsg1, testMsg2, testMsg3}}, + &Conversation{Name: "channel_name", ID: "CHANNEL", ThreadTS: "THREAD", Messages: []Message{testMsg1, testMsg2, testMsg3}}, false, }, { @@ -626,8 +631,9 @@ func TestSlackDumper_DumpThread(t *testing.T) { nil, ). Times(1) + mockConvInfo(mc, "CHANNEL", "channel_name") }, - &Conversation{ID: "CHANNEL", ThreadTS: "THREAD", Messages: []Message{testMsg1, testMsg2}}, + &Conversation{Name: "channel_name", ID: "CHANNEL", ThreadTS: "THREAD", Messages: []Message{testMsg1, testMsg2}}, false, }, { @@ -707,8 +713,9 @@ func TestSlackDumper_DumpURL(t *testing.T) { }, nil, ) + mockConvInfo(sc, "CHM82GF99", "unittest") }, - want: &Conversation{ID: "CHM82GF99", Messages: []Message{testMsg1}}, + want: &Conversation{Name: "unittest", ID: "CHM82GF99", Messages: []Message{testMsg1}}, wantErr: false, }, { @@ -721,8 +728,9 @@ func TestSlackDumper_DumpURL(t *testing.T) { "", nil, ) + mockConvInfo(sc, "CHM82GF99", "unittest") }, - want: &Conversation{ID: "CHM82GF99", ThreadTS: "1577694990.000400", Messages: []Message{testMsg1}}, + want: &Conversation{Name: "unittest", ID: "CHM82GF99", ThreadTS: "1577694990.000400", Messages: []Message{testMsg1}}, wantErr: false, }, { @@ -751,6 +759,7 @@ func TestSlackDumper_DumpURL(t *testing.T) { t.Errorf("SlackDumper.DumpURL() error = %v, wantErr %v", err, tt.wantErr) return } + assert.Equal(t, tt.want, got) if !reflect.DeepEqual(got, tt.want) { t.Errorf("SlackDumper.DumpURL() = %v, want %v", got, tt.want) } @@ -758,6 +767,10 @@ func TestSlackDumper_DumpURL(t *testing.T) { } } +func mockConvInfo(mc *mockClienter, channelID, wantName string) { + mc.EXPECT().GetConversationInfoContext(gomock.Any(), channelID, false).Return(&slack.Channel{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{NameNormalized: wantName}}}, nil) +} + func TestConversation_String(t *testing.T) { type fields struct { Messages []Message @@ -887,3 +900,75 @@ func TestSlackDumper_convHistoryParams(t *testing.T) { }) } } + +func TestSlackDumper_getChannelName(t *testing.T) { + type fields struct { + Users Users + UserIndex map[string]*slack.User + options Options + } + type args struct { + ctx context.Context + l *rate.Limiter + channelID string + } + tests := []struct { + name string + fields fields + args args + expectFn func(mc *mockClienter) + want string + wantErr bool + }{ + { + name: "ok", + fields: fields{}, + args: args{ + ctx: context.Background(), + l: newLimiter(noTier, 1, 0), + channelID: "TESTCHAN", + }, + expectFn: func(sc *mockClienter) { + sc.EXPECT().GetConversationInfoContext(gomock.Any(), "TESTCHAN", false).Return(&slack.Channel{GroupConversation: slack.GroupConversation{Conversation: slack.Conversation{NameNormalized: "unittest"}}}, nil) + }, + want: "unittest", + wantErr: false, + }, + { + name: "error", + fields: fields{}, + args: args{ + ctx: context.Background(), + l: newLimiter(noTier, 1, 0), + channelID: "TESTCHAN", + }, + expectFn: func(sc *mockClienter) { + sc.EXPECT().GetConversationInfoContext(gomock.Any(), "TESTCHAN", false).Return(nil, errors.New("rekt")) + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mc := newmockClienter(ctrl) + + tt.expectFn(mc) + sd := &SlackDumper{ + client: mc, + Users: tt.fields.Users, + UserIndex: tt.fields.UserIndex, + options: tt.fields.options, + } + got, err := sd.getChannelName(tt.args.ctx, tt.args.l, tt.args.channelID) + if (err != nil) != tt.wantErr { + t.Errorf("SlackDumper.getChannelName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("SlackDumper.getChannelName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/slackdump.go b/slackdump.go index be6a8fba..a2e26f86 100644 --- a/slackdump.go +++ b/slackdump.go @@ -35,6 +35,7 @@ type SlackDumper struct { // clienter is the interface with some functions of slack.Client with the sole // purpose of mocking in tests (see client_mock.go) type clienter interface { + GetConversationInfoContext(ctx context.Context, channelID string, includeLocale bool) (*slack.Channel, error) GetConversationHistoryContext(ctx context.Context, params *slack.GetConversationHistoryParameters) (*slack.GetConversationHistoryResponse, error) GetConversationRepliesContext(ctx context.Context, params *slack.GetConversationRepliesParameters) (msgs []slack.Message, hasMore bool, nextCursor string, err error) GetConversations(params *slack.GetConversationsParameters) (channels []slack.Channel, nextCursor string, err error) From 17d5167f395007d58b8323f8950d260e10597f16 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Tue, 22 Feb 2022 21:51:35 +1000 Subject: [PATCH 2/5] template naming --- cmd/slackdump/main.go | 2 + internal/app/app.go | 11 ++++- internal/app/config.go | 52 ++++++++++++++++++++++ internal/app/config_test.go | 86 +++++++++++++++++++++++++++++++++++++ messages.go | 4 ++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 internal/app/config_test.go diff --git a/cmd/slackdump/main.go b/cmd/slackdump/main.go index a6244c48..bb83d10d 100644 --- a/cmd/slackdump/main.go +++ b/cmd/slackdump/main.go @@ -136,6 +136,8 @@ func parseCmdLine(args []string) (params, error) { fs.StringVar(&p.appCfg.Output.Filename, "o", "-", "Output `filename` for users and channels. Use '-' for standard\nOutput.") fs.StringVar(&p.appCfg.Output.Format, "r", "", "report `format`. One of 'json' or 'text'") + fs.StringVar(&p.appCfg.FilenameTemplate, "ft", "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}", "output file naming template.") + // options fs.BoolVar(&p.appCfg.Options.DumpFiles, "f", slackdump.DefOptions.DumpFiles, "same as -download") fs.BoolVar(&p.appCfg.Options.DumpFiles, "download", slackdump.DefOptions.DumpFiles, "enable files download.") diff --git a/internal/app/app.go b/internal/app/app.go index 2a931703..529ed846 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -105,6 +105,15 @@ func (app *App) newDumpFunc(s string) dumpFunc { } } +func (app *App) renderFilename(c *slackdump.Conversation) string { + var buf strings.Builder + if err := app.cfg.tmpl.ExecuteTemplate(&buf, filenameTmplName, c); err != nil { + // this should nevar happen + panic(err) + } + return buf.String() +} + // dumpOneChannel dumps just one channel having ID = id. If generateText is // true, it will also generate a ID.txt text file. func (app *App) dumpOne(ctx context.Context, s string, fn dumpFunc) error { @@ -113,7 +122,7 @@ func (app *App) dumpOne(ctx context.Context, s string, fn dumpFunc) error { return err } - return app.writeFile(cnv.String(), cnv) + return app.writeFile(app.renderFilename(cnv), cnv) } func (app *App) writeFile(name string, cnv *slackdump.Conversation) error { diff --git a/internal/app/config.go b/internal/app/config.go index c3e38b63..f45979ea 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -4,12 +4,16 @@ import ( "bufio" "errors" "fmt" + "html/template" "io" "strings" "github.com/rusq/slackdump" + "github.com/slack-go/slack" ) +const filenameTmplName = "fnt" + type Config struct { Creds SlackCreds ListFlags ListFlags @@ -20,6 +24,9 @@ type Config struct { Oldest TimeValue // oldest time to dump conversations from Latest TimeValue // latest time to dump conversations to + FilenameTemplate string + tmpl *template.Template + Options slackdump.Options } @@ -110,6 +117,51 @@ func (p *Config) Validate() error { return fmt.Errorf("invalid Output type: %q, must use one of %v", p.Output.Format, []string{OutputTypeJSON, OutputTypeText}) } + // validate file naming template + if err := p.compileValidateTemplate(); err != nil { + return err + } + + return nil +} + +func (p *Config) compileValidateTemplate() error { + var err error + p.tmpl, err = template.New(filenameTmplName).Parse(p.FilenameTemplate) + if err != nil { + return err + } + + // are you ready for some filth? Here we go! + + // let's define some indicators + const ( + NotOK = "$$ERROR$$" // not allowed at all + OK = "$$OK$$" // required + PartialOK = "$$PARTIAL$$" // partial (only goes well with OK) + ) + + // marking all the fields we want with OK, all the rest (the ones we DO NOT + // WANT) with NotOK. + tc := slackdump.Conversation{ + Name: OK, + ID: OK, + Messages: []slackdump.Message{{Message: slack.Message{Msg: slack.Msg{Channel: NotOK}}}}, + ThreadTS: PartialOK, + } + + // now we render the template and check for OK/NotOK values in the output. + var buf strings.Builder + if err := p.tmpl.ExecuteTemplate(&buf, filenameTmplName, tc); err != nil { + return err + } + if strings.Contains(buf.String(), NotOK) || len(buf.String()) == 0 { + return fmt.Errorf("invalid fields in the template: %q", p.FilenameTemplate) + } + if !strings.Contains(buf.String(), OK) { + // must contain at least one OK + return fmt.Errorf("this does not resolve to anything useful: %q", p.FilenameTemplate) + } return nil } diff --git a/internal/app/config_test.go b/internal/app/config_test.go new file mode 100644 index 00000000..3644a8c8 --- /dev/null +++ b/internal/app/config_test.go @@ -0,0 +1,86 @@ +package app + +import ( + "html/template" + "testing" + + "github.com/rusq/slackdump" +) + +func TestConfig_compileValidateTemplate(t *testing.T) { + type fields struct { + Creds SlackCreds + ListFlags ListFlags + Input Input + Output Output + Oldest TimeValue + Latest TimeValue + FilenameTemplate string + tmpl *template.Template + Options slackdump.Options + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + "id is ok", + fields{FilenameTemplate: "{{.ID}}", tmpl: template.New("")}, + false, + }, + { + "name is ok", + fields{FilenameTemplate: "{{.Name}}", tmpl: template.New("")}, + false, + }, + { + "just threadTS is not ok", + fields{FilenameTemplate: "{{.ThreadTS}}", tmpl: template.New("")}, + true, + }, + { + "threadTS and message ID is ok", + fields{FilenameTemplate: "{{.ID}}-{{.ThreadTS}}", tmpl: template.New("")}, + false, + }, + { + "threadTS and message ID is ok (conditional)", + fields{FilenameTemplate: "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}", tmpl: template.New("")}, + false, + }, + { + "message is not ok", + fields{FilenameTemplate: "{{.Message}}", tmpl: template.New("")}, + true, + }, + { + "unknown field is not ok", + fields{FilenameTemplate: "{{.Who_dis}}", tmpl: template.New("")}, + true, + }, + { + "empty not ok", + fields{FilenameTemplate: "", tmpl: template.New("")}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &Config{ + Creds: tt.fields.Creds, + ListFlags: tt.fields.ListFlags, + Input: tt.fields.Input, + Output: tt.fields.Output, + Oldest: tt.fields.Oldest, + Latest: tt.fields.Latest, + FilenameTemplate: tt.fields.FilenameTemplate, + tmpl: tt.fields.tmpl, + Options: tt.fields.Options, + } + if err := p.compileValidateTemplate(); (err != nil) != tt.wantErr { + t.Errorf("Config.compileValidateTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/messages.go b/messages.go index b79475a2..68daf034 100644 --- a/messages.go +++ b/messages.go @@ -53,6 +53,10 @@ func (c Conversation) String() string { return c.ID + "-" + c.ThreadTS } +func (c Conversation) IsThread() bool { + return c.ThreadTS != "" +} + // Message is the internal representation of message with thread. type Message struct { slack.Message From 4d90083c397a66e69b0848c75ff128da19444a68 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Tue, 22 Feb 2022 22:29:43 +1000 Subject: [PATCH 3/5] update readme --- README.rst | 88 +++++++++++++++++++++++++++++++++++++++---- cmd/slackdump/main.go | 2 +- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 9c0d9e9c..a83c8e4f 100644 --- a/README.rst +++ b/README.rst @@ -87,7 +87,7 @@ Setting up the application Dumping conversations -~~~~~~~~~~~~~~~~~~~~~ +--------------------- As it was already mentioned in the introduction, Slackdump supports two ways of providing the conversation IDs that you want to save: @@ -102,7 +102,7 @@ should be placed on a separate line. Slackdump can automatically detect if it's an ID or a URL. Providing the list on the command line -++++++++++++++++++++++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Firstly, dump the channel list to choose what you want to dump:: @@ -132,7 +132,7 @@ you want the convesation to be saved to a text file as well, use the ``-r text`` command line parameter. See example below. Example -^^^^^^^ ++++++++ Say, you want to dump convesations with @alice and @bob to the text files and also want to save all the files that you all shared in those @@ -171,7 +171,7 @@ Run the slackdump and provide the URL link as an input:: Reading data from the file -++++++++++++++++++++++++++ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Slackdump can read the list of the channels and URLs to dump from the file. @@ -188,7 +188,7 @@ file. ╰───────: instructs slackdump to use the file input Dumping users -~~~~~~~~~~~~~ +------------- To view all users, run:: @@ -197,8 +197,8 @@ To view all users, run:: By default, slackdump exports users in text format. If you need to output json, use ``-r json`` flag. -Dumping chanels -~~~~~~~~~~~~~~~ +Dumping channels +---------------- To view channels, that are visible to your account, including group conversations, archived chats and public channels, run:: @@ -208,6 +208,80 @@ conversations, archived chats and public channels, run:: By default, slackdump exports users in text format. If you need to output json, use ``-r json`` flag. +Command line flags reference +============================ + +In this section there will be some explanation provided for the +possible command line flags. + +This doc may be out of date, to get the current command line flags +with a brief description, run:: + + slackdump -h + +Command line flags are described as of version ``v1.3.1``. + +-V print version and exit +-c same as -list-channels +-cookie along with ``-t`` sets the authentication values. Can also be + set using ``COOKIE`` environment variable. Must contain the + value of ``d=`` cookie. +-cpr number of conversation items per request. (default 200). This is + the amount of individual messages that will be fetched from + Slack API per single API request. +-dl-retries rate limit retries for file downloads. (default 3). If + the file download process hits the Slack Rate Limit + reponse (HTTP ERROR 429), slackdump will retry the + download this number of times, for each file. +-download enable files download. If this flag is specified, slackdump + will download all attachements. +-download-workers number of file download worker threads. (default 4). + File download is performed with multiple + goroutines. This is the number of goroutines that + will be downloading files. You generally wouldn't + need to modify this value. +-dump-from timestamp of the oldest message to fetch from + (i.e. 2020-12-31T23:59:59). Allows setting the lower + boundary of the timeframe for conversation dump. This is + useful when you don't need everything from the beginning + of times. +-dump-to timestamp of the latest message to fetch to + (i.e. 2020-12-31T23:59:59). Same as above, but for upper + boundary. +-f same as -download +-ft output file naming template. This parameter allows to define + custom naming for output conversation files. See "Filename + templates" section for explanation and examples. +-i specify the input file with Channel IDs or URLs to be used instead + of giving the list on the command line, one per line. Use "-" + to read input from STDIN. Example: ``-i my_links.txt``. +-limiter-boost same as -t3-boost. (default 120) +-limiter-burst same as -t3-burst. (default 1) +-list-channels list channels (aka conversations) and their IDs for export. +-list-users list users and their IDs. +-o output filename for users and channels. Use '-' for + standard output. (default "-") +-r report (output) format. One of 'json' or 'text'. + For channels and users - will output only in the specified + format. For messages - if 'text' is requested, + the text file will be generated along with json. +-t Specify slack API_token, (environment: SLACK_TOKEN). + This should be used along with ``--cookie`` flag. +-t2-boost Tier-2 limiter boost in events per minute (affects users + and channels). +-t2-burst Tier-2 limiter burst in events (affects users and channels). (default 1) +-t2-retries rate limit retries for channel listing. (default 20) +-t3-boost Tier-3 rate limiter boost in events per minute, will be added to the + base slack tier event per minute value. (default 120) +-t3-burst allow up to N burst events per second. Default value is safe. (default 1) +-t3-retries rate limit retries for conversation. (default 3) +-trace trace file (optional) (default "trace.out") +-u same as -list-users +-user-cache-age user cache lifetime duration. Set this to 0 to disable cache. (default 4h0m0s) +-user-cache-file user cache filename. (default "users.json") +-v verbose messages + + As a library ============ diff --git a/cmd/slackdump/main.go b/cmd/slackdump/main.go index bb83d10d..5abe2d14 100644 --- a/cmd/slackdump/main.go +++ b/cmd/slackdump/main.go @@ -145,7 +145,7 @@ func parseCmdLine(args []string) (params, error) { fs.IntVar(&p.appCfg.Options.Tier3Retries, "t3-retries", slackdump.DefOptions.Tier3Retries, "rate limit retries for conversation.") fs.IntVar(&p.appCfg.Options.Tier2Retries, "t2-retries", slackdump.DefOptions.Tier2Retries, "rate limit retries for channel listing.") fs.IntVar(&p.appCfg.Options.DownloadRetries, "dl-retries", slackdump.DefOptions.DownloadRetries, "rate limit retries for file downloads.") - fs.IntVar(&p.appCfg.Options.ConversationsPerReq, "cpr", slackdump.DefOptions.ConversationsPerReq, "number of file download worker threads.") + fs.IntVar(&p.appCfg.Options.ConversationsPerReq, "cpr", slackdump.DefOptions.ConversationsPerReq, "number of conversation `items` per request.") fs.UintVar(&p.appCfg.Options.Tier3Boost, "limiter-boost", slackdump.DefOptions.Tier3Boost, "same as -t3-boost.") fs.UintVar(&p.appCfg.Options.Tier3Boost, "t3-boost", slackdump.DefOptions.Tier3Boost, "Tier-3 rate limiter boost in `events` per minute, will be added to the\nbase slack tier event per minute value.") fs.UintVar(&p.appCfg.Options.Tier3Burst, "limiter-burst", slackdump.DefOptions.Tier3Burst, "same as -t3-burst.") From 983eea3eeef3bafac8407b85ed6004abe68a68c7 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Wed, 23 Feb 2022 21:01:08 +1000 Subject: [PATCH 4/5] update README --- Makefile | 5 + README.rst | 232 +++++++++++++++------ slackdump.1 | 588 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 759 insertions(+), 66 deletions(-) create mode 100644 slackdump.1 diff --git a/Makefile b/Makefile index 28779f79..90258e94 100644 --- a/Makefile +++ b/Makefile @@ -40,3 +40,8 @@ clean: test: go test -race -cover -count=3 ./... + +man: slackdump.1 + +slackdump.1: README.rst + rst2man.py $< $@ --syntax-highlight=none diff --git a/README.rst b/README.rst index a83c8e4f..6da2acdc 100644 --- a/README.rst +++ b/README.rst @@ -221,68 +221,166 @@ with a brief description, run:: Command line flags are described as of version ``v1.3.1``. --V print version and exit --c same as -list-channels --cookie along with ``-t`` sets the authentication values. Can also be - set using ``COOKIE`` environment variable. Must contain the - value of ``d=`` cookie. --cpr number of conversation items per request. (default 200). This is - the amount of individual messages that will be fetched from - Slack API per single API request. --dl-retries rate limit retries for file downloads. (default 3). If - the file download process hits the Slack Rate Limit - reponse (HTTP ERROR 429), slackdump will retry the - download this number of times, for each file. --download enable files download. If this flag is specified, slackdump - will download all attachements. --download-workers number of file download worker threads. (default 4). - File download is performed with multiple - goroutines. This is the number of goroutines that - will be downloading files. You generally wouldn't - need to modify this value. --dump-from timestamp of the oldest message to fetch from - (i.e. 2020-12-31T23:59:59). Allows setting the lower - boundary of the timeframe for conversation dump. This is - useful when you don't need everything from the beginning - of times. --dump-to timestamp of the latest message to fetch to - (i.e. 2020-12-31T23:59:59). Same as above, but for upper - boundary. --f same as -download --ft output file naming template. This parameter allows to define - custom naming for output conversation files. See "Filename - templates" section for explanation and examples. --i specify the input file with Channel IDs or URLs to be used instead - of giving the list on the command line, one per line. Use "-" - to read input from STDIN. Example: ``-i my_links.txt``. --limiter-boost same as -t3-boost. (default 120) --limiter-burst same as -t3-burst. (default 1) --list-channels list channels (aka conversations) and their IDs for export. --list-users list users and their IDs. --o output filename for users and channels. Use '-' for - standard output. (default "-") --r report (output) format. One of 'json' or 'text'. - For channels and users - will output only in the specified - format. For messages - if 'text' is requested, - the text file will be generated along with json. --t Specify slack API_token, (environment: SLACK_TOKEN). - This should be used along with ``--cookie`` flag. --t2-boost Tier-2 limiter boost in events per minute (affects users - and channels). --t2-burst Tier-2 limiter burst in events (affects users and channels). (default 1) --t2-retries rate limit retries for channel listing. (default 20) --t3-boost Tier-3 rate limiter boost in events per minute, will be added to the - base slack tier event per minute value. (default 120) --t3-burst allow up to N burst events per second. Default value is safe. (default 1) --t3-retries rate limit retries for conversation. (default 3) --trace trace file (optional) (default "trace.out") --u same as -list-users --user-cache-age user cache lifetime duration. Set this to 0 to disable cache. (default 4h0m0s) --user-cache-file user cache filename. (default "users.json") --v verbose messages +\-V + print version and exit +\-c + same as -list-channels + +\-cookie + along with ``-t`` sets the authentication values. Can also be set + using ``COOKIE`` environment variable. Must contain the value of + ``d=`` cookie. + +\-cpr + number of conversation items per request. (default 200). This is + the amount of individual messages that will be fetched from Slack + API per single API request. + +\-dl-retries number + rate limit retries for file downloads. (default 3). If the file + download process hits the Slack Rate Limit reponse (HTTP ERROR + 429), slackdump will retry the download this number of times, for + each file. + +\-download + enable files download. If this flag is specified, slackdump will + download all attachments, including the ones in threads. + +\-download-workers + number of file download worker threads. (default 4). File download + is performed with multiple goroutines. This is the number of + goroutines that will be downloading files. You generally wouldn't + need to modify this value. + +\-dump-from + timestamp of the oldest message to fetch from + (i.e. 2020-12-31T23:59:59). Allows setting the lower boundary of + the timeframe for conversation dump. This is useful when you don't + need everything from the beginning of times. + +\-dump-to + timestamp of the latest message to fetch to + (i.e. 2020-12-31T23:59:59). Same as above, but for upper boundary. + +\-f + shorthand for -download (means "files") + +\-ft + output file naming template. This parameter allows to define + custom naming for output conversation files. + + It uses `Go templating`_ system. Available template tags: + + :{{.ID}}: channel ID + :{{.Name}}: channel Name + :{{.ThreadTS}}: thread timestamp. This tag can not be used on it's + own, it must be combined with at least one of the above tags. + + You can use any of the standard template functions. The default + value for this parameter outputs the channelID as the filename. For + threads, it will use channelID-threadTS. + + Below are some of the common templates you could use. + + :Channel ID and thread: + :: + + {{.ID}}{{if .ThreadTS}}-{{.ThreadTS}}{{end}} + + The output file will look like "``C480129421.json``" for a + channel if channel has ID=C480129421 and + "``C4840129421-1234567890.123456.json``" for a thread. This is + the default template. + + :Channel Name and thread: + + :: + + {{.Name}}{{if .ThreadTS}}({{.ThreadTS}}){{end}} + + The output file will look like "``general.json``" for the channel and + "``general(123457890.123456).json``" for a thread. + + +\-i + specify the input file with Channel IDs or URLs to be used instead + of giving the list on the command line, one per line. Use "-" to + read input from STDIN. Example: ``-i my_links.txt``. + +\-limiter-boost + same as -t3-boost. (default 120) + +\-limiter-burst + same as -t3-burst. (default 1) + +\-list-channels + list channels (aka conversations) and their IDs for export. The + default output format is "text". Use ``-r json`` to output + as JSON. + +\-list-users + list users and their IDs. The default output format is "text". + Use ``-r json`` to output as JSON. + +\-o + output filename for users and channels. Use '-' for standard + output. (default "-") + +\-r + report (output) format. One of 'json' or 'text'. For channels and + users - will output only in the specified format. For messages - + if 'text' is requested, the text file will be generated along with + json. + +\-t + Specify slack API token, (environment: ``SLACK_TOKEN``). + This should be used along with ``--cookie`` flag. + +\-t2-boost + Tier-2 limiter boost in events per minute (affects users and + channels APIs). + +\-t2-burst + Tier-2 limiter burst in events (affects users and + channels APIs). (default 1) + +\-t2-retries + rate limit retries for channel listing. (affects users and channels APIs). + (default 20) + +\-t3-boost + Tier-3 rate limiter boost in events per minute, will be added to + the base slack tier event per minute value. Affects conversation + APIs. (default 120) + +\-t3-burst + allow up to N burst events per second. Default value is + safe. Affects conversation APIs (default 1) + +\-t3-retries + rate limit retries for conversation. Affects conversation APIs. (default 3) + +\-trace filename + allows to specify the trace filename and enable tracing (optional). + Use this flag if requested by developer. The trace file does not contain any + sensitive or PII. + +\-u + shorthand for -list-users. + +\-user-cache-age + user cache lifetime duration. Set this to 0 to disable + cache. (default 4h0m0s) User cache is used to speedup consequent + runs of slackdump. Known issue - if you're changing slack + workspace, make sure to delete the cache file, or set this to 0. + +\-user-cache-file + user cache filename. (default "users.json") See note + for -user-cache-age above. + +\-v + verbose messages - - As a library ============ @@ -309,14 +407,15 @@ Use: FAQ === -Q: **Do I need to create a Slack application?** +:Q: **Do I need to create a Slack application?** + +:A: No, you don't. You need to grab that token and cookie from the + browser Slack session. See Usage_ at the top of the file. -A: No, you don't. You need to grab that token and cookie from the -browser Slack session. See Usage in the top of the file. +:Q: **I'm getting "invalid_auth" error** -Q: **I'm getting ``invalid_auth``** +:A: Go get the new Cookie from the browser and Token as well. -A: Go get the new Cookie from the browser. Bulletin Board @@ -331,6 +430,7 @@ Messages that were conveyed with the donations: .. _`Buy me a cup of tea`: https://www.paypal.com/donate/?hosted_button_id=GUHCLSM7E54ZW .. _`Join the discussion`: https://t.me/slackdump .. _`Read the set up guide on Medium.com`: https://medium.com/@gilyazov/downloading-your-private-slack-conversations-52e50428b3c2 +.. _`Go templating`: https://pkg.go.dev/html/template .. bulletin board links diff --git a/slackdump.1 b/slackdump.1 new file mode 100644 index 00000000..07b766de --- /dev/null +++ b/slackdump.1 @@ -0,0 +1,588 @@ +.\" Man page generated from reStructuredText. +. +.TH SLACK DUMPER "" "" "" +.SH NAME +Slack Dumper \- +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.sp +\fI\%Buy me a cup of tea\fP +.sp +\fI\%Join the discussion\fP +.sp +\fI\%Read the set up guide on Medium.com\fP +.sp +Purpose: dump slack messages, users and files using browser token and cookie. +.sp +Typical usecase scenarios: +.INDENT 0.0 +.IP \(bu 2 +You want to archive your private convesations from slack but the administrator +does not allow you to install applications. +.IP \(bu 2 +You are allowed to install applications in Slack but don\(aqt want to use the +"cloud" tools for privacy concerns \- god knows what those third party apps are +retaining in their "clouds". +.UNINDENT +.sp +The library is "fit\-for\-purpose" quality and provided AS\-IS. Can\(aqt +say it\(aqs ready for production, as it lacks most of the unit tests, but +will do for ad\-hoc use. +.sp +Slackdump accepts two types of input: URL link of the channel or +thread, or ID of the channel. +.SH USAGE +.INDENT 0.0 +.IP 1. 3 +Download the archive from the Releases page for your operating system. (NOTE: \fBMacOS users\fP should download \fBdarwin\fP release file). +.IP 2. 3 +Unpack +.IP 3. 3 +Change directory to where you have unpacked the archive. +.IP 4. 3 +Run \fB\&./slackdump \-h\fP to see help. +.UNINDENT +.SS How to authenticate +.SS Getting the authentication data +.INDENT 0.0 +.IP 1. 3 +Open up your Slack \fIin browser\fP and login. +.UNINDENT +.SS TOKEN +.INDENT 0.0 +.IP 1. 3 +Open your browser \fIDeveloper Console\fP\&. +.IP 2. 3 +Go to the Network tab +.IP 3. 3 +In the toolbar, switch to \fBFetch/XHR\fP view. +.IP 4. 3 +Open any channel or private conversation in Slack. You\(aqll see a +bunch of stuff appearing in Network panel. +.IP 5. 3 +In the list of requests, find the one starting with +\fBchannels.prefs.get?\fP, click it and click on \fIHeaders\fP tab in the +opened pane. +.IP 6. 3 +Scroll down, until you see \fBForm Data\fP +.IP 7. 3 +Grab the \fBtoken:\fP value (it starts with \fBxoxc\-\fP), by right +clicking the value and choosing "Copy Value". +.UNINDENT +.sp +\fBIf you don\(aqt see the token value\fP in Poogle Chrome \- switch to \fIPayload\fP tab, +your token is waiting for you there. +.SS COOKIE +.INDENT 0.0 +.IP 1. 3 +Switch to \fI\%Application\fP tab and select \fBCookies\fP in the left +navigation pane. +.IP 2. 3 +Find the cookie with the name "\fBd\fP". That\(aqs right, just the +letter "d". +.IP 3. 3 +Double\-click the Value of this cookie. +.IP 4. 3 +Press Ctrl+C or Cmd+C to copy it\(aqs value to clipboard. +.IP 5. 3 +Save it for later. +.UNINDENT +.SS Setting up the application +.INDENT 0.0 +.IP 1. 3 +Create the file named \fB\&.env\fP next to where the slackdump +executable in any text editor. Alternatively the file can +be named \fBsecrets.txt\fP or \fB\&.env.txt\fP\&. +.IP 2. 3 +Add the token and cookie values to it. End result +should look like this: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +SLACK_TOKEN=xoxc\-<...elided...> +COOKIE=12345472908twp<...elided...> +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 3. 3 +Save the file and close the editor. +.UNINDENT +.SS Dumping conversations +.sp +As it was already mentioned in the introduction, Slackdump supports +two ways of providing the conversation IDs that you want to save: +.INDENT 0.0 +.IP \(bu 2 +\fBBy ID\fP: it expects to see Conversation IDs. +.IP \(bu 2 +\fBBy URL\fP: it expects to see URLs. You can get URL by choosing +"Copy Link" in the Slack on the channel or thread. +.UNINDENT +.sp +IDs or URLs can be passed on the command line or read from a file +(using the \fB\-i\fP command line flag), in that file, every ID or URL +should be placed on a separate line. Slackdump can automatically +detect if it\(aqs an ID or a URL. +.SS Providing the list on the command line +.sp +Firstly, dump the channel list to choose what you want to dump: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-c +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You will get the output resembling the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +2021/10/31 17:32:34 initializing... +2021/10/31 17:32:35 retrieving data... +2021/10/31 17:32:35 done +ID Arch Saved What +CHXXXXXXX \- \- #everything +CHXXXXXXX \- \- #everyone +CHXXXXXXX \- \- #random +DHMAXXXXX \- \- @slackbot +DNF3XXXXX \- \- @alice +DLY4XXXXX \- \- @bob +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You\(aqll need the value in the \fBID\fP column. +.sp +To dump the channel, run the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump [ID2] ... [IDn] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +By default, slackdump generates a json file with the convesation. If +you want the convesation to be saved to a text file as well, use the +\fB\-r text\fP command line parameter. See example below. +.SS Example +.sp +Say, you want to dump convesations with @alice and @bob to the text +files and also want to save all the files that you all shared in those +convesations: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-r text \-f DNF3XXXXX DLY4XXXXX https://.... + ━━━┯━━━ ━┯ ━━━┯━━━━━ ━━━┯━━━━━ ━━━━┯━━━━━┅┅ + │ │ │ │ │ + │ │ │ ╰─: @alice │ + │ │ ╰───────────: @bob ┊ + │ ╰────────────────: save files + ╰──────────────────────: text file output + thread or conversation URL :────────╯ +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Conversation URL: +.sp +To get the conversation URL link, use this simple trick that they +won\(aqt teach you at school: +.INDENT 0.0 +.IP 1. 3 +In Slack, right click on the conversation you want to dump (in the +channel navigation pane on the left) +.IP 2. 3 +Choose "Copy link". +.UNINDENT +.sp +Thread URL: +.INDENT 0.0 +.IP 1. 3 +In Slack, open the thread that you want to dump. +.IP 2. 3 +The thread opens to the right of the main conversation window +.IP 3. 3 +On the first message of the thread, click on three vertical dots menu (not sure how it\(aqs properly called), choose "Copy link" +.UNINDENT +.sp +Run the slackdump and provide the URL link as an input: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-f https://xxxxxx.slack.com/archives/CHM82GX00/p1577694990000400 + ━┯ ━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + │ ╰─────: URL of the thread + ╰──────────────: save files +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Reading data from the file +.sp +Slackdump can read the list of the channels and URLs to dump from the +file. +.INDENT 0.0 +.IP 1. 3 +Create the file that will contain all the necessary IDs and/or +URLs, I\(aqll use "links.txt" in the example. +.IP 2. 3 +Copy/paste all the IDs and URLs into that file, one per line. +.IP 3. 3 +Run slackdump with "\-i" command line flag. "\-i" stands for +"input": +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-i links.txt + ━━━━┯━━━━━━━ + │ + ╰───────: instructs slackdump to use the file input +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.SS Dumping users +.sp +To view all users, run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-u +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +By default, slackdump exports users in text format. If you need to +output json, use \fB\-r json\fP flag. +.SS Dumping channels +.sp +To view channels, that are visible to your account, including group +conversations, archived chats and public channels, run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-c +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +By default, slackdump exports users in text format. If you need to +output json, use \fB\-r json\fP flag. +.SH COMMAND LINE FLAGS REFERENCE +.sp +In this section there will be some explanation provided for the +possible command line flags. +.sp +This doc may be out of date, to get the current command line flags +with a brief description, run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +slackdump \-h +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Command line flags are described as of version \fBv1.3.1\fP\&. +.INDENT 0.0 +.TP +.B \-V +print version and exit +.TP +.B \-c +same as \-list\-channels +.TP +.B \-cookie +along with \fB\-t\fP sets the authentication values. Can also be set +using \fBCOOKIE\fP environment variable. Must contain the value of +\fBd=\fP cookie. +.TP +.B \-cpr +number of conversation items per request. (default 200). This is +the amount of individual messages that will be fetched from Slack +API per single API request. +.TP +.B \-dl\-retries number +rate limit retries for file downloads. (default 3). If the file +download process hits the Slack Rate Limit reponse (HTTP ERROR +429), slackdump will retry the download this number of times, for +each file. +.TP +.B \-download +enable files download. If this flag is specified, slackdump will +download all attachments, including the ones in threads. +.TP +.B \-download\-workers +number of file download worker threads. (default 4). File download +is performed with multiple goroutines. This is the number of +goroutines that will be downloading files. You generally wouldn\(aqt +need to modify this value. +.TP +.B \-dump\-from +timestamp of the oldest message to fetch from +(i.e. 2020\-12\-31T23:59:59). Allows setting the lower boundary of +the timeframe for conversation dump. This is useful when you don\(aqt +need everything from the beginning of times. +.TP +.B \-dump\-to +timestamp of the latest message to fetch to +(i.e. 2020\-12\-31T23:59:59). Same as above, but for upper boundary. +.TP +.B \-f +shorthand for \-download (means "files") +.TP +.B \-ft +output file naming template. This parameter allows to define +custom naming for output conversation files. +.sp +It uses \fI\%Go templating\fP system. Available template tags: +.INDENT 7.0 +.TP +.B {{.ID}} +channel ID +.TP +.B {{.Name}} +channel Name +.TP +.B {{.ThreadTS}} +thread timestamp. This tag can not be used on it\(aqs +own, it must be combined with at least one of the above tags. +.UNINDENT +.sp +You can use any of the standard template functions. The default +value for this parameter outputs the channelID as the filename. For +threads, it will use channelID\-threadTS. +.sp +Below are some of the common templates you could use. +.INDENT 7.0 +.TP +.B Channel ID and thread +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +{{.ID}}{{if .ThreadTS}}\-{{.ThreadTS}}{{end}} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The output file will look like "\fBC480129421.json\fP" for a +channel if channel has ID=C480129421 and +"\fBC4840129421\-1234567890.123456.json\fP" for a thread. This is +the default template. +.TP +.B Channel Name and thread +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +{{.Name}}{{if .ThreadTS}}({{.ThreadTS}}){{end}} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The output file will look like "\fBgeneral.json\fP" for the channel and +"\fBgeneral(123457890.123456).json\fP" for a thread. +.UNINDENT +.TP +.B \-i +specify the input file with Channel IDs or URLs to be used instead +of giving the list on the command line, one per line. Use "\-" to +read input from STDIN. Example: \fB\-i my_links.txt\fP\&. +.TP +.B \-limiter\-boost +same as \-t3\-boost. (default 120) +.TP +.B \-limiter\-burst +same as \-t3\-burst. (default 1) +.TP +.B \-list\-channels +list channels (aka conversations) and their IDs for export. The +default output format is "text". Use \fB\-r json\fP to output +as JSON. +.TP +.B \-list\-users +list users and their IDs. The default output format is "text". +Use \fB\-r json\fP to output as JSON. +.TP +.B \-o +output filename for users and channels. Use \(aq\-\(aq for standard +output. (default "\-") +.TP +.B \-r +report (output) format. One of \(aqjson\(aq or \(aqtext\(aq. For channels and +users \- will output only in the specified format. For messages \- +if \(aqtext\(aq is requested, the text file will be generated along with +json. +.TP +.B \-t +Specify slack API token, (environment: \fBSLACK_TOKEN\fP). +This should be used along with \fB\-\-cookie\fP flag. +.TP +.B \-t2\-boost +Tier\-2 limiter boost in events per minute (affects users and +channels APIs). +.TP +.B \-t2\-burst +Tier\-2 limiter burst in events (affects users and +channels APIs). (default 1) +.TP +.B \-t2\-retries +rate limit retries for channel listing. (affects users and channels APIs). +(default 20) +.TP +.B \-t3\-boost +Tier\-3 rate limiter boost in events per minute, will be added to +the base slack tier event per minute value. Affects conversation +APIs. (default 120) +.TP +.B \-t3\-burst +allow up to N burst events per second. Default value is +safe. Affects conversation APIs (default 1) +.TP +.B \-t3\-retries +rate limit retries for conversation. Affects conversation APIs. (default 3) +.TP +.B \-trace filename +allows to specify the trace filename and enable tracing (optional). +Use this flag if requested by developer. The trace file does not contain any +sensitive or PII. +.TP +.B \-u +shorthand for \-list\-users. +.TP +.B \-user\-cache\-age +user cache lifetime duration. Set this to 0 to disable +cache. (default 4h0m0s) User cache is used to speedup consequent +runs of slackdump. Known issue \- if you\(aqre changing slack +workspace, make sure to delete the cache file, or set this to 0. +.TP +.B \-user\-cache\-file +user cache filename. (default "users.json") See note +for \-user\-cache\-age above. +.TP +.B \-v +verbose messages +.UNINDENT +.SH AS A LIBRARY +.sp +Download: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +go get github.com/rusq/slackdump +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Use: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +import "github.com/rusq/slackdump" + +func main() { + sd, err := slackdump.New(os.Getenv("TOKEN"), os.Getenv("COOKIE")) + if err != nil { + // handle + } + // ... read the docs +} +.ft P +.fi +.UNINDENT +.UNINDENT +.SH FAQ +.INDENT 0.0 +.TP +.B Q +\fBDo I need to create a Slack application?\fP +.TP +.B A +No, you don\(aqt. You need to grab that token and cookie from the +browser Slack session. See \fI\%Usage\fP at the top of the file. +.TP +.B Q +\fBI\(aqm getting "invalid_auth" error\fP +.TP +.B A +Go get the new Cookie from the browser and Token as well. +.UNINDENT +.SS Bulletin Board +.sp +Messages that were conveyed with the donations: +.INDENT 0.0 +.IP \(bu 2 +25/01/2022: Stay away from \fI\%TheSignChef.com\fP, ya hear, they don\(aqt pay what +they owe to their employees. +.UNINDENT +.\" bulletin board links +. +.\" Generated by docutils manpage writer. +. From 483d627eb8120c003f5d9aade703e1e4d21e8b64 Mon Sep 17 00:00:00 2001 From: Rustam Gilyazov <16064414+rusq@users.noreply.github.com> Date: Wed, 23 Feb 2022 21:12:44 +1000 Subject: [PATCH 5/5] fix tests and move tmpl from config to app --- cmd/slackdump/main.go | 4 +++- cmd/slackdump/main_test.go | 14 ++++++++------ internal/app/app.go | 15 ++++++++++++--- internal/app/config.go | 17 +++++++++-------- internal/app/config_test.go | 19 ++++++++----------- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/cmd/slackdump/main.go b/cmd/slackdump/main.go index 5abe2d14..6ef70512 100644 --- a/cmd/slackdump/main.go +++ b/cmd/slackdump/main.go @@ -22,6 +22,8 @@ const ( slackCookieEnv = "COOKIE" ) +const defFilenameTemplate = "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}" + var build = "dev" // secrets defines the names of the supported secret files that we load our @@ -136,7 +138,7 @@ func parseCmdLine(args []string) (params, error) { fs.StringVar(&p.appCfg.Output.Filename, "o", "-", "Output `filename` for users and channels. Use '-' for standard\nOutput.") fs.StringVar(&p.appCfg.Output.Format, "r", "", "report `format`. One of 'json' or 'text'") - fs.StringVar(&p.appCfg.FilenameTemplate, "ft", "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}", "output file naming template.") + fs.StringVar(&p.appCfg.FilenameTemplate, "ft", defFilenameTemplate, "output file naming template.") // options fs.BoolVar(&p.appCfg.Options.DumpFiles, "f", slackdump.DefOptions.DumpFiles, "same as -download") diff --git a/cmd/slackdump/main_test.go b/cmd/slackdump/main_test.go index a77f6ca6..c0980ffc 100644 --- a/cmd/slackdump/main_test.go +++ b/cmd/slackdump/main_test.go @@ -59,9 +59,10 @@ func Test_checkParameters(t *testing.T) { Token: "x", Cookie: "d", }, - Input: app.Input{List: []string{}}, - Output: app.Output{Filename: "-", Format: "text"}, - Options: slackdump.DefOptions, + FilenameTemplate: defFilenameTemplate, + Input: app.Input{List: []string{}}, + Output: app.Output{Filename: "-", Format: "text"}, + Options: slackdump.DefOptions, }}, false, }, @@ -77,9 +78,10 @@ func Test_checkParameters(t *testing.T) { Token: "x", Cookie: "d", }, - Input: app.Input{List: []string{}}, - Output: app.Output{Filename: "-", Format: "text"}, - Options: slackdump.DefOptions, + FilenameTemplate: defFilenameTemplate, + Input: app.Input{List: []string{}}, + Output: app.Output{Filename: "-", Format: "text"}, + Options: slackdump.DefOptions, }}, false, }, diff --git a/internal/app/app.go b/internal/app/app.go index 529ed846..2c077dc2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "html/template" "io" "os" "strings" @@ -19,14 +20,22 @@ const ( ) type App struct { - sd *slackdump.SlackDumper + sd *slackdump.SlackDumper + tmpl *template.Template cfg Config } // New creates a new slackdump app. func New(cfg Config) (*App, error) { - return &App{cfg: cfg}, nil + if err := cfg.Validate(); err != nil { + return nil, err + } + tmpl, err := cfg.compileTemplates() + if err != nil { + return nil, err + } + return &App{cfg: cfg, tmpl: tmpl}, nil } // init initialises the slack dumper app. @@ -107,7 +116,7 @@ func (app *App) newDumpFunc(s string) dumpFunc { func (app *App) renderFilename(c *slackdump.Conversation) string { var buf strings.Builder - if err := app.cfg.tmpl.ExecuteTemplate(&buf, filenameTmplName, c); err != nil { + if err := app.tmpl.ExecuteTemplate(&buf, filenameTmplName, c); err != nil { // this should nevar happen panic(err) } diff --git a/internal/app/config.go b/internal/app/config.go index f45979ea..2271d595 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -25,7 +25,6 @@ type Config struct { Latest TimeValue // latest time to dump conversations to FilenameTemplate string - tmpl *template.Template Options slackdump.Options } @@ -125,13 +124,15 @@ func (p *Config) Validate() error { return nil } -func (p *Config) compileValidateTemplate() error { - var err error - p.tmpl, err = template.New(filenameTmplName).Parse(p.FilenameTemplate) +func (cfg *Config) compileTemplates() (*template.Template, error) { + return template.New(filenameTmplName).Parse(cfg.FilenameTemplate) +} + +func (cfg *Config) compileValidateTemplate() error { + tmpl, err := cfg.compileTemplates() if err != nil { return err } - // are you ready for some filth? Here we go! // let's define some indicators @@ -152,15 +153,15 @@ func (p *Config) compileValidateTemplate() error { // now we render the template and check for OK/NotOK values in the output. var buf strings.Builder - if err := p.tmpl.ExecuteTemplate(&buf, filenameTmplName, tc); err != nil { + if err := tmpl.ExecuteTemplate(&buf, filenameTmplName, tc); err != nil { return err } if strings.Contains(buf.String(), NotOK) || len(buf.String()) == 0 { - return fmt.Errorf("invalid fields in the template: %q", p.FilenameTemplate) + return fmt.Errorf("invalid fields in the template: %q", cfg.FilenameTemplate) } if !strings.Contains(buf.String(), OK) { // must contain at least one OK - return fmt.Errorf("this does not resolve to anything useful: %q", p.FilenameTemplate) + return fmt.Errorf("this does not resolve to anything useful: %q", cfg.FilenameTemplate) } return nil } diff --git a/internal/app/config_test.go b/internal/app/config_test.go index 3644a8c8..796c02d6 100644 --- a/internal/app/config_test.go +++ b/internal/app/config_test.go @@ -1,7 +1,6 @@ package app import ( - "html/template" "testing" "github.com/rusq/slackdump" @@ -16,7 +15,6 @@ func TestConfig_compileValidateTemplate(t *testing.T) { Oldest TimeValue Latest TimeValue FilenameTemplate string - tmpl *template.Template Options slackdump.Options } tests := []struct { @@ -26,42 +24,42 @@ func TestConfig_compileValidateTemplate(t *testing.T) { }{ { "id is ok", - fields{FilenameTemplate: "{{.ID}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.ID}}"}, false, }, { "name is ok", - fields{FilenameTemplate: "{{.Name}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.Name}}"}, false, }, { "just threadTS is not ok", - fields{FilenameTemplate: "{{.ThreadTS}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.ThreadTS}}"}, true, }, { "threadTS and message ID is ok", - fields{FilenameTemplate: "{{.ID}}-{{.ThreadTS}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.ID}}-{{.ThreadTS}}"}, false, }, { "threadTS and message ID is ok (conditional)", - fields{FilenameTemplate: "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}"}, false, }, { "message is not ok", - fields{FilenameTemplate: "{{.Message}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.Message}}"}, true, }, { "unknown field is not ok", - fields{FilenameTemplate: "{{.Who_dis}}", tmpl: template.New("")}, + fields{FilenameTemplate: "{{.Who_dis}}"}, true, }, { "empty not ok", - fields{FilenameTemplate: "", tmpl: template.New("")}, + fields{FilenameTemplate: ""}, true, }, } @@ -75,7 +73,6 @@ func TestConfig_compileValidateTemplate(t *testing.T) { Oldest: tt.fields.Oldest, Latest: tt.fields.Latest, FilenameTemplate: tt.fields.FilenameTemplate, - tmpl: tt.fields.tmpl, Options: tt.fields.Options, } if err := p.compileValidateTemplate(); (err != nil) != tt.wantErr {