Documentation re-generated for 5.0 release.

CopyIndexes.mkdn

@@ -4,7 +4,7 @@ es-copy-index.pl - Copy an index from one cluster to another
 
 # VERSION
 
-version 4.8
+version 5.0
 
 # SYNOPSIS
 
@@ -28,6 +28,10 @@ From App::ElasticSearch::Utilities:
     --local         Use localhost as the elasticsearch host
     --host          ElasticSearch host to connect to
     --port          HTTP port for your cluster
+    --proto         Defaults to 'http', can also be 'https'
+    --http-username HTTP Basic Auth username
+    --http-password HTTP Basic Auth password (if not specified, and --http-user is, you will be prompted)
+    --password-exec Script to run to get the users password
     --noop          Any operations other than GET are disabled, can be negated with --no-noop
     --timeout       Timeout to ElasticSearch, default 30
     --keep-proxy    Do not remove any proxy settings from %ENV
@@ -38,6 +42,8 @@ From App::ElasticSearch::Utilities:
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: all
 
+See also the "CONNECTION ARGUMENTS" and "INDEX SELECTION ARGUMENTS" sections from App::ElasticSearch::Utilities.
+
 From CLI::Helpers:
 
     --data-file         Path to a file to write lines tagged with 'data => 1'

Maintenance.mkdn

@@ -4,7 +4,7 @@ es-daily-index-maintenance.pl - Run to prune old indexes and optimize existing
 
 # VERSION
 
-version 4.8
+version 5.0
 
 # SYNOPSIS
 
@@ -32,6 +32,10 @@ From App::ElasticSearch::Utilities:
     --local         Use localhost as the elasticsearch host
     --host          ElasticSearch host to connect to
     --port          HTTP port for your cluster
+    --proto         Defaults to 'http', can also be 'https'
+    --http-username HTTP Basic Auth username
+    --http-password HTTP Basic Auth password (if not specified, and --http-user is, you will be prompted)
+    --password-exec Script to run to get the users password
     --noop          Any operations other than GET are disabled, can be negated with --no-noop
     --timeout       Timeout to ElasticSearch, default 30
     --keep-proxy    Do not remove any proxy settings from %ENV
@@ -42,6 +46,8 @@ From App::ElasticSearch::Utilities:
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: all
 
+See also the "CONNECTION ARGUMENTS" and "INDEX SELECTION ARGUMENTS" sections from App::ElasticSearch::Utilities.
+
 From CLI::Helpers:
 
     --data-file         Path to a file to write lines tagged with 'data => 1'

README.mkdn

@@ -4,7 +4,7 @@ App::ElasticSearch::Utilities - Utilities for Monitoring ElasticSearch
 
 # VERSION
 
-version 4.8
+version 5.0
 
 # SYNOPSIS
 
@@ -51,6 +51,19 @@ distribution.
 
 Grab the value of the global value from the es-utils.yaml files.
 
+## es\_basic\_auth($host)
+
+Get the user/password combination for this host.  This is called from LWP::UserAgent if
+it recieves a 401, so the auth condition must be satisfied.
+
+Returns the username and password as a list.
+
+## es\_pass\_exec(host, username)
+
+Called from es\_basic\_auth to exec a program, capture the password
+and return it to the caller.  This allows the use of password vaults
+and keychains.
+
 ## es\_pattern
 
 Returns a hashref of the pattern filter used to get the indexes
@@ -189,6 +202,10 @@ From App::ElasticSearch::Utilities:
     --local         Use localhost as the elasticsearch host
     --host          ElasticSearch host to connect to
     --port          HTTP port for your cluster
+    --proto         Defaults to 'http', can also be 'https'
+    --http-username HTTP Basic Auth username
+    --http-password HTTP Basic Auth password (if not specified, and --http-user is, you will be prompted)
+    --password-exec Script to run to get the users password
     --noop          Any operations other than GET are disabled, can be negated with --no-noop
     --timeout       Timeout to ElasticSearch, default 30
     --keep-proxy    Do not remove any proxy settings from %ENV
@@ -199,14 +216,180 @@ From App::ElasticSearch::Utilities:
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: all
 
+See also the "CONNECTION ARGUMENTS" and "INDEX SELECTION ARGUMENTS" sections from App::ElasticSearch::Utilities.
+
 # ARGUMENT GLOBALS
 
 Some options may be specified in the **/etc/es-utils.yaml** or **$HOME/.es-utils.yaml** file:
 
     ---
+    base: logstash
+    days: 7
     host: esproxy.example.com
     port: 80
     timeout: 10
+    proto: https
+    http-username: bob
+    password-exec: /home/bob/bin/get-es-passwd.sh
+
+# CONNECTION ARGUMENTS
+
+Arguments for establishing a connection with the cluster.  Unless specified otherwise, these options
+can all be set in the globals file.
+
+- **local**
+
+    Assume ElasticSearch is running locally, connect to localhost.
+
+- **host**
+
+    Use a different hostname or IP address to connect.
+
+- **port**
+
+    Defaults to 9200.
+
+- **proto**
+
+    Defaults to 'http', can also be 'https'.
+
+- **http-username**
+
+    If HTTP Basic Authentication is required, use this username.
+
+    See also the ["HTTP Basic Authentication"](#http-basic-authentication) section for more details
+
+- **http-password**
+
+    If HTTP Basic Authentication is required, use this password, **\*\*INSECURE\*\***, set
+    in globals, netrc, or use the **password-exec** option below.
+
+- **password-exec**
+
+    If HTTP Basic Authentication is required, run this command, passing the arguments:
+
+        <command_to_run> <es_host> <es_username>
+
+    The script expects the last line to contain the password in plaintext.
+
+- **noop**
+
+    Prevents any communication to the cluster from making changes to the settings or data contained therein.
+    In short, it prevents anything but HEAD and GET requests, **except** POST requests to the \_search endpoint.
+
+- **timeout**
+
+    Timeout for connections and requests, defaults to 10.
+
+- **keep-proxy**
+
+    By default, HTTP proxy environment variables are stripped. Use this option to keep your proxy environment variables
+    in tact.
+
+# INDEX SELECTION ARGUMENTS
+
+- **base**
+
+    In an environment using monthly, weekly, daily, or hourly indexes.  The base index name is everything without the date.
+    Parsing for bases, also provides splitting and matching on segments of the index name delineated by the '-' character.
+    If we have the following indexes:
+
+        web-dc1-YYYY.MM.DD
+        web-dc2-YYYY.MM.DD
+        logstash-dc1-YYYY.MM.DD
+        logstash-dc2-YYYY.MM.DD
+
+    Valid bases would be:
+
+        web
+        web-dc1
+        web-dc2
+        logstash
+        logstash-dc1
+        logstash-dc2
+        dc1
+        dc2
+
+    Combining that with the days option can provide a way to select many indexes at once.
+
+- **days**
+
+    How many days backwards you want your operation to be relevant.
+
+- **datesep**
+
+    Default is '.' Can be set to an empty string for no separator.
+
+- **pattern**
+
+    A pattern to match the indexes.  Can expand the following key words and characters:
+
+        '*'    expanded to '.*'
+        'ANY'  expanded to '.*'
+        'DATE' expanded to a pattern to match a date, based on datesep
+
+    The indexes are compared against this pattern.
+
+# HTTP Basic Authentication
+
+The implementation for HTTP Basic Authentication leverages the LWP::UserAgent's underlying HTTP 401
+detection and is automatic.  There is no way to force basic authentication, it has to be requested
+by the server.  If the server does request it, here's what you need to know about how usernames and
+passwords are resolved.
+
+The username is selected by going through these mechanisms until one is found:
+
+    --http-username
+    'http-username' in /etc/es-utils.yml or ~/.es-utils.yml
+    Netrc element matching the hostname of the request
+    CLI::Helpers prompt()
+
+Once the username has been resolved, the following mechanisms are tried in order:
+
+    --http-password
+    'http-password' in /etc/es-utils.yml or ~/.es-utils.yml
+    Netrc element matching the hostname of the request
+    Password executable defined by --password-exec
+    'password-exec' in /etc/es-utils.yml, ~/.es-utils.yml
+    CLI::Helpers prompt()
+
+## Password Exec
+
+It is **BAD** practice to specify passwords as a command line argument, or store it in a plaintext
+file.  There are cases where this may be necessary, but it is not recommended.  The best method for securing  your
+password is to use the **password-exec** option.
+
+This option must point to an executable script.  That script will be passed two arguments, the hostname and the username
+for the request.  It expects the password printed to STDOUT as the last line of output.  Here's an example password-exec setup
+using Apple Keychain:
+
+    #!/bin/sh
+
+    HOSTNAME=$1;
+    USERNAME=$2;
+
+    /usr/bin/security find-generic-password -w -a "$USERNAME" -s "$HOSTNAME"
+
+If we save this to "$HOME/bin/get-passwd.sh" we can execute a script
+like this:
+
+    $ es-search.pl --http-username bob --password-exec $HOME/bin/get-passwd.sh \
+                    --base secure-data --fields
+
+Though it's probably best to set this in your ~/.es-utils.yml file:
+
+    ---
+    host: secured-cluster.example.org
+    port: 443
+    proto: https
+    http-username: bob
+    password-exec: /home/bob/bin/get-passwd.sh
+
+### CLI::Helpers and Password Prompting
+
+If all the fails to yield a password, the last resort is to use CLI::Helpers::prompt() to ask the user for their
+password.  If the user is using version 1.1 or higher of CLI::Helpers, this call will turn off echo and readline magic
+for the password prompt.
 
 # INSTALL
 

Searching.mkdn

@@ -4,7 +4,7 @@ es-search.pl - Provides a CLI for quick searches of data in ElasticSearch daily
 
 # VERSION
 
-version 4.8
+version 5.0
 
 # SYNOPSIS
 
@@ -40,6 +40,10 @@ From App::ElasticSearch::Utilities:
     --local         Use localhost as the elasticsearch host
     --host          ElasticSearch host to connect to
     --port          HTTP port for your cluster
+    --proto         Defaults to 'http', can also be 'https'
+    --http-username HTTP Basic Auth username
+    --http-password HTTP Basic Auth password (if not specified, and --http-user is, you will be prompted)
+    --password-exec Script to run to get the users password
     --noop          Any operations other than GET are disabled, can be negated with --no-noop
     --timeout       Timeout to ElasticSearch, default 30
     --keep-proxy    Do not remove any proxy settings from %ENV
@@ -50,6 +54,8 @@ From App::ElasticSearch::Utilities:
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: all
 
+See also the "CONNECTION ARGUMENTS" and "INDEX SELECTION ARGUMENTS" sections from App::ElasticSearch::Utilities.
+
 From CLI::Helpers:
 
     --data-file         Path to a file to write lines tagged with 'data => 1'