Merge pull request #34 from rusq/i10

Output file naming templates

Makefile

@@ -40,3 +40,8 @@ clean:
 
 test:
 	go test -race -cover -count=3 ./...
+
+man: slackdump.1
+
+slackdump.1: README.rst
+	rst2man.py $< $@ --syntax-highlight=none

README.rst

@@ -87,7 +87,7 @@ Setting up the application
 
 
 Dumping conversations
-~~~~~~~~~~~~~~~~~~~~~
+---------------------
 
 As it was already mentioned in the introduction, Slackdump supports
 two ways of providing the conversation IDs that you want to save:
@@ -102,7 +102,7 @@ should be placed on a separate line.  Slackdump can automatically
 detect if it's an ID or a URL.
 
 Providing the list on the command line
-++++++++++++++++++++++++++++++++++++++
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Firstly, dump the channel list to choose what you want to dump::
 
@@ -132,7 +132,7 @@ you want the convesation to be saved to a text file as well, use the
 ``-r text`` command line parameter.  See example below.
 
 Example
-^^^^^^^
++++++++
 
 Say, you want to dump convesations with @alice and @bob to the text
 files and also want to save all the files that you all shared in those
@@ -171,7 +171,7 @@ Run the slackdump and provide the URL link as an input::
 	     
 
 Reading data from the file
-++++++++++++++++++++++++++
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Slackdump can read the list of the channels and URLs to dump from the
 file.
@@ -188,7 +188,7 @@ file.
 		   ╰───────: instructs slackdump to use the file input
 
 Dumping users
-~~~~~~~~~~~~~
+-------------
 
 To view all users, run::
 
@@ -197,8 +197,8 @@ To view all users, run::
 By default, slackdump exports users in text format.  If you need to
 output json, use ``-r json`` flag.
 
-Dumping chanels
-~~~~~~~~~~~~~~~
+Dumping channels
+----------------
 
 To view channels, that are visible to your account, including group
 conversations, archived chats and public channels, run::
@@ -208,7 +208,179 @@ conversations, archived chats and public channels, run::
 By default, slackdump exports users in text format.  If you need to
 output json, use ``-r json`` flag.
 
-
+Command line flags reference
+============================
+
+In this section there will be some explanation provided for the
+possible command line flags.
+
+This doc may be out of date, to get the current command line flags
+with a brief description, run::
+
+  slackdump -h
+
+Command line flags are described as of version ``v1.3.1``.
+
+\-V
+   print version and exit
+\-c
+   same as -list-channels
+
+\-cookie
+   along with ``-t`` sets the authentication values.  Can also be set
+   using ``COOKIE`` environment variable.  Must contain the value of
+   ``d=`` cookie.
+
+\-cpr
+   number of conversation items per request. (default 200).  This is
+   the amount of individual messages that will be fetched from Slack
+   API per single API request.
+
+\-dl-retries number
+   rate limit retries for file downloads. (default 3).  If the file
+   download process hits the Slack Rate Limit reponse (HTTP ERROR
+   429), slackdump will retry the download this number of times, for
+   each file.
+
+\-download
+   enable files download.  If this flag is specified, slackdump will
+   download all attachments, including the ones in threads.
+
+\-download-workers
+   number of file download worker threads. (default 4).  File download
+   is performed with multiple goroutines.  This is the number of
+   goroutines that will be downloading files.  You generally wouldn't
+   need to modify this value.
+
+\-dump-from
+   timestamp of the oldest message to fetch from
+   (i.e. 2020-12-31T23:59:59).  Allows setting the lower boundary of
+   the timeframe for conversation dump.  This is useful when you don't
+   need everything from the beginning of times.
+
+\-dump-to
+   timestamp of the latest message to fetch to
+   (i.e. 2020-12-31T23:59:59).  Same as above, but for upper boundary.
+
+\-f
+   shorthand for -download (means "files")
+
+\-ft
+   output file naming template.  This parameter allows to define
+   custom naming for output conversation files.
+
+   It uses `Go templating`_ system.  Available template tags:
+
+   :{{.ID}}: channel ID
+   :{{.Name}}: channel Name
+   :{{.ThreadTS}}: thread timestamp.  This tag can not be used on it's
+      own, it must be combined with at least one of the above tags.
+
+   You can use any of the standard template functions.  The default
+   value for this parameter outputs the channelID as the filename.  For
+   threads, it will use channelID-threadTS.
+
+   Below are some of the common templates you could use.
+
+   :Channel ID and thread:
+      ::
+
+	 {{.ID}}{{if .ThreadTS}}-{{.ThreadTS}}{{end}}
+
+      The output file will look like "``C480129421.json``" for a
+      channel if channel has ID=C480129421 and
+      "``C4840129421-1234567890.123456.json``" for a thread.  This is
+      the default template.
+
+   :Channel Name and thread:
+
+      ::
+
+	 {{.Name}}{{if .ThreadTS}}({{.ThreadTS}}){{end}}
+
+      The output file will look like "``general.json``" for the channel and
+      "``general(123457890.123456).json``" for a thread.
+
+
+\-i
+   specify the input file with Channel IDs or URLs to be used instead
+   of giving the list on the command line, one per line.  Use "-" to
+   read input from STDIN.  Example: ``-i my_links.txt``.
+
+\-limiter-boost
+   same as -t3-boost. (default 120)
+
+\-limiter-burst
+   same as -t3-burst. (default 1)
+
+\-list-channels
+   list channels (aka conversations) and their IDs for export.  The
+   default output format is "text".  Use ``-r json`` to output
+   as JSON.
+
+\-list-users
+   list users and their IDs.  The default output format is "text".
+   Use ``-r json`` to output as JSON.
+
+\-o
+   output filename for users and channels.  Use '-' for standard
+   output. (default "-")
+
+\-r
+   report (output) format.  One of 'json' or 'text'. For channels and
+   users - will output only in the specified format.  For messages -
+   if 'text' is requested, the text file will be generated along with
+   json.
+
+\-t
+   Specify slack API token, (environment: ``SLACK_TOKEN``).
+   This should be used along with ``--cookie`` flag.
+
+\-t2-boost
+   Tier-2 limiter boost in events per minute (affects users and
+   channels APIs).
+
+\-t2-burst
+   Tier-2 limiter burst in events (affects users and
+   channels APIs). (default 1)
+
+\-t2-retries
+   rate limit retries for channel listing. (affects users and channels APIs).
+   (default 20)
+
+\-t3-boost
+   Tier-3 rate limiter boost in events per minute, will be added to
+   the base slack tier event per minute value.  Affects conversation
+   APIs. (default 120)
+
+\-t3-burst
+   allow up to N burst events per second.  Default value is
+   safe. Affects conversation APIs (default 1)
+
+\-t3-retries
+   rate limit retries for conversation.  Affects conversation APIs. (default 3)
+
+\-trace filename
+   allows to specify the trace filename and enable tracing (optional).
+   Use this flag if requested by developer.  The trace file does not contain any
+   sensitive or PII.
+
+\-u
+   shorthand for -list-users.
+
+\-user-cache-age
+   user cache lifetime duration. Set this to 0 to disable
+   cache. (default 4h0m0s) User cache is used to speedup consequent
+   runs of slackdump.  Known issue - if you're changing slack
+   workspace, make sure to delete the cache file, or set this to 0.
+
+\-user-cache-file
+   user cache filename. (default "users.json") See note
+   for -user-cache-age above.
+
+\-v
+   verbose messages
+
 As a library
 ============
 
@@ -235,14 +407,15 @@ Use:
 FAQ
 ===
 
-Q: **Do I need to create a Slack application?**
+:Q: **Do I need to create a Slack application?**
+
+:A: No, you don't.  You need to grab that token and cookie from the
+    browser Slack session.  See Usage_ at the top of the file.
 
-A: No, you don't.  You need to grab that token and cookie from the
-browser Slack session.  See Usage in the top of the file.
+:Q: **I'm getting "invalid_auth" error**
 
-Q: **I'm getting ``invalid_auth``**
+:A: Go get the new Cookie from the browser and Token as well.
 
-A: Go get the new Cookie from the browser.
 
 
 Bulletin Board
@@ -257,6 +430,7 @@ Messages that were conveyed with the donations:
 .. _`Buy me a cup of tea`: https://www.paypal.com/donate/?hosted_button_id=GUHCLSM7E54ZW
 .. _`Join the discussion`: https://t.me/slackdump
 .. _`Read the set up guide on Medium.com`: https://medium.com/@gilyazov/downloading-your-private-slack-conversations-52e50428b3c2
+.. _`Go templating`: https://pkg.go.dev/html/template
 
 ..
   bulletin board links

cmd/slackdump/main.go

@@ -22,6 +22,8 @@ const (
 	slackCookieEnv = "COOKIE"
 )
 
+const defFilenameTemplate = "{{.ID}}{{ if .ThreadTS}}-{{.ThreadTS}}{{end}}"
+
 var build = "dev"
 
 // secrets defines the names of the supported secret files that we load our
@@ -136,14 +138,16 @@ func parseCmdLine(args []string) (params, error) {
 	fs.StringVar(&p.appCfg.Output.Filename, "o", "-", "Output `filename` for users and channels.  Use '-' for standard\nOutput.")
 	fs.StringVar(&p.appCfg.Output.Format, "r", "", "report `format`.  One of 'json' or 'text'")
 
+	fs.StringVar(&p.appCfg.FilenameTemplate, "ft", defFilenameTemplate, "output file naming template.")
+
 	// options
 	fs.BoolVar(&p.appCfg.Options.DumpFiles, "f", slackdump.DefOptions.DumpFiles, "same as -download")
 	fs.BoolVar(&p.appCfg.Options.DumpFiles, "download", slackdump.DefOptions.DumpFiles, "enable files download.")
 	fs.IntVar(&p.appCfg.Options.Workers, "download-workers", slackdump.DefOptions.Workers, "number of file download worker threads.")
 	fs.IntVar(&p.appCfg.Options.Tier3Retries, "t3-retries", slackdump.DefOptions.Tier3Retries, "rate limit retries for conversation.")
 	fs.IntVar(&p.appCfg.Options.Tier2Retries, "t2-retries", slackdump.DefOptions.Tier2Retries, "rate limit retries for channel listing.")
 	fs.IntVar(&p.appCfg.Options.DownloadRetries, "dl-retries", slackdump.DefOptions.DownloadRetries, "rate limit retries for file downloads.")
-	fs.IntVar(&p.appCfg.Options.ConversationsPerReq, "cpr", slackdump.DefOptions.ConversationsPerReq, "number of file download worker threads.")
+	fs.IntVar(&p.appCfg.Options.ConversationsPerReq, "cpr", slackdump.DefOptions.ConversationsPerReq, "number of conversation `items` per request.")
 	fs.UintVar(&p.appCfg.Options.Tier3Boost, "limiter-boost", slackdump.DefOptions.Tier3Boost, "same as -t3-boost.")
 	fs.UintVar(&p.appCfg.Options.Tier3Boost, "t3-boost", slackdump.DefOptions.Tier3Boost, "Tier-3 rate limiter boost in `events` per minute, will be added to the\nbase slack tier event per minute value.")
 	fs.UintVar(&p.appCfg.Options.Tier3Burst, "limiter-burst", slackdump.DefOptions.Tier3Burst, "same as -t3-burst.")

cmd/slackdump/main_test.go

@@ -59,9 +59,10 @@ func Test_checkParameters(t *testing.T) {
 					Token:  "x",
 					Cookie: "d",
 				},
-				Input:   app.Input{List: []string{}},
-				Output:  app.Output{Filename: "-", Format: "text"},
-				Options: slackdump.DefOptions,
+				FilenameTemplate: defFilenameTemplate,
+				Input:            app.Input{List: []string{}},
+				Output:           app.Output{Filename: "-", Format: "text"},
+				Options:          slackdump.DefOptions,
 			}},
 			false,
 		},
@@ -77,9 +78,10 @@ func Test_checkParameters(t *testing.T) {
 					Token:  "x",
 					Cookie: "d",
 				},
-				Input:   app.Input{List: []string{}},
-				Output:  app.Output{Filename: "-", Format: "text"},
-				Options: slackdump.DefOptions,
+				FilenameTemplate: defFilenameTemplate,
+				Input:            app.Input{List: []string{}},
+				Output:           app.Output{Filename: "-", Format: "text"},
+				Options:          slackdump.DefOptions,
 			}},
 			false,
 		},