From 48ddb890ce350afa3ddbc8602bb282c9c185aebf Mon Sep 17 00:00:00 2001 From: Karol Bucek Date: Mon, 4 Apr 2022 08:27:29 +0200 Subject: [PATCH] Feat: add ssl_supported_protocols option (#1055) --- .ci/Dockerfile.elasticsearch | 5 ++ .ci/docker-compose.override.yml | 4 ++ .ci/logstash-run.sh | 16 +++-- .travis.yml | 1 + CHANGELOG.md | 3 + docs/index.asciidoc | 22 +++++- .../outputs/elasticsearch/http_client.rb | 4 +- .../elasticsearch/http_client_builder.rb | 5 ++ .../elasticsearch/api_configs.rb | 2 + logstash-output-elasticsearch.gemspec | 2 +- spec/integration/outputs/index_spec.rb | 67 ++++++++++++++++--- spec/unit/outputs/elasticsearch_ssl_spec.rb | 2 +- 12 files changed, 114 insertions(+), 19 deletions(-) diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch index 8ad98bf23..5ad0cec1b 100644 --- a/.ci/Dockerfile.elasticsearch +++ b/.ci/Dockerfile.elasticsearch @@ -6,6 +6,7 @@ ARG es_path=/usr/share/elasticsearch ARG es_yml=$es_path/config/elasticsearch.yml ARG SECURE_INTEGRATION ARG ES_SSL_KEY_INVALID +ARG ES_SSL_SUPPORTED_PROTOCOLS RUN rm -f $es_path/config/scripts @@ -25,6 +26,10 @@ RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then \ fi \ fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi +RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ ! -z "$ES_SSL_SUPPORTED_PROTOCOLS" ] ; then echo "xpack.security.http.ssl.supported_protocols: ${ES_SSL_SUPPORTED_PROTOCOLS}" >> $es_yml; fi +RUN cat $es_yml + +RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd admin -p elastic -r superuser; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml index 9812f5680..260b0e503 100644 --- a/.ci/docker-compose.override.yml +++ b/.ci/docker-compose.override.yml @@ -12,6 +12,7 @@ services: - INTEGRATION=${INTEGRATION:-false} - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false} + - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS elasticsearch: build: @@ -22,6 +23,9 @@ services: - INTEGRATION=${INTEGRATION:-false} - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false} + - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS + environment: + - ES_JAVA_OPTS=-Xms640m -Xmx640m command: /usr/share/elasticsearch/elasticsearch-run.sh tty: true ports: diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh index 990e05038..3e7bd084a 100755 --- a/.ci/logstash-run.sh +++ b/.ci/logstash-run.sh @@ -1,26 +1,32 @@ #!/bin/bash + +env + set -ex export PATH=$BUILD_DIR/gradle/bin:$PATH if [[ "$SECURE_INTEGRATION" == "true" ]]; then - ES_URL="https://elasticsearch:9200 -k" + ES_URL="https://elasticsearch:9200" else ES_URL="http://elasticsearch:9200" fi +# CentOS 7 using curl defaults does not enable TLSv1.3 +CURL_OPTS="-k --tlsv1.2 --tls-max 1.3" + wait_for_es() { count=120 - while ! curl -s $ES_URL >/dev/null && [[ $count -ne 0 ]]; do + while ! curl $CURL_OPTS $ES_URL >/dev/null && [[ $count -ne 0 ]]; do count=$(( $count - 1 )) [[ $count -eq 0 ]] && exit 1 sleep 1 done - echo $(curl -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") + echo $(curl $CURL_OPTS -vi $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") } if [[ "$INTEGRATION" != "true" ]]; then - bundle exec rspec -fd spec/unit -t ~integration -t ~secure_integration + jruby -rbundler/setup -S rspec -fd spec/unit -t ~integration -t ~secure_integration else if [[ "$SECURE_INTEGRATION" == "true" ]]; then @@ -32,5 +38,5 @@ else echo "Waiting for elasticsearch to respond..." ES_VERSION=$(wait_for_es) echo "Elasticsearch $ES_VERSION is Up!" - bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag es_version:$ES_VERSION spec/integration + jruby -rbundler/setup -S rspec -fd $extra_tag_args --tag update_tests:painless --tag es_version:$ES_VERSION spec/integration fi diff --git a/.travis.yml b/.travis.yml index 6534b863a..9eeaa6c81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,3 +12,4 @@ env: - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info + - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info diff --git a/CHANGELOG.md b/CHANGELOG.md index 612c45921..b8fbb75e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 11.5.0 + - Feat: add ssl_supported_protocols option [#1055](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1055) + ## 11.4.2 - [DOC] Add `v8` to supported values for ecs_compatiblity defaults [#1059](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1059) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index f7ba26dbd..0ad2beac8 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -355,6 +355,7 @@ This plugin supports the following configuration options plus the | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |a valid filesystem path|No | <> |<>|No | <> |<>|No @@ -1004,6 +1005,23 @@ Option to validate the server's certificate. Disabling this severely compromises For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf +[id="plugins-{type}s-{plugin}-ssl_supported_protocols"] +===== `ssl_supported_protocols` + + * Value type is <> + * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'` + * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`. + `'TLSv1.1'` is not considered secure and is only provided for legacy applications. + +List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster. + +For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the +`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash. + +NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash, +the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in +the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list. + [id="plugins-{type}s-{plugin}-template"] ===== `template` @@ -1018,8 +1036,8 @@ If not set, the included template will be used. * Value type is <> * Default value depends on whether <> is enabled: - ** ECS Compatibility disabled: `logstash` - ** ECS Compatibility enabled: `ecs-logstash` + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` This configuration option defines how the template is named inside Elasticsearch. diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index 9efaeaf12..975e348ab 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -283,11 +283,11 @@ def uris end def client_settings - @options[:client_settings] || {} + @_client_settings ||= @options[:client_settings] || {} end def ssl_options - client_settings.fetch(:ssl, {}) + @_ssl_options ||= client_settings.fetch(:ssl, {}) end def http_compression diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb index fb181fa59..ce59bae48 100644 --- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb +++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb @@ -132,11 +132,16 @@ def self.setup_ssl(logger, params) ssl_options[:keystore] = keystore ssl_options[:keystore_password] = keystore_password.value if keystore_password end + if !params["ssl_certificate_verification"] logger.warn "You have enabled encryption but DISABLED certificate verification, " + "to make sure your data is secure remove `ssl_certificate_verification => false`" ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname end + + protocols = params['ssl_supported_protocols'] + ssl_options[:protocols] = protocols if protocols && protocols.any? + { ssl: ssl_options } end diff --git a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb index 1721a0c00..67bfeaad3 100644 --- a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +++ b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb @@ -66,6 +66,8 @@ module APIConfigs # Set the keystore password :keystore_password => { :validate => :password }, + :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true }, + # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use # this with master nodes, you probably want to disable HTTP on them by setting diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index 6fb9b14ea..154793130 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.4.2' + s.version = '11.5.0' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb index 6e8170528..21b46dabb 100644 --- a/spec/integration/outputs/index_spec.rb +++ b/spec/integration/outputs/index_spec.rb @@ -60,25 +60,48 @@ let(:curl_opts) { nil } + let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc + let(:es_admin_pass) { 'elastic' } + def curl_and_get_json_response(url, method: :get); require 'open3' + cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}" begin - stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}") + out, err, status = Open3.capture3(cmd) rescue Errno::ENOENT fail "curl not available, make sure curl binary is installed and available on $PATH" end if status.success? - LogStash::Json.load(stdout) + http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n + + if http_status.strip[0].to_i > 2 + error = (LogStash::Json.load(out)['error']) rescue nil + if error + fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}" + else + warn out + fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}" + end + end + + LogStash::Json.load(out) else - fail "curl failed: #{status}\n #{stdout}" + warn out + fail "#{cmd.inspect} process failed: #{status}\n\n#{err}" end end + let(:initial_events) { [] } + before do subject.register - subject.multi_receive([]) + subject.multi_receive(initial_events) if initial_events end - + + after do + subject.do_close + end + shared_examples "an indexer" do |secure| it "ships events" do subject.multi_receive(events) @@ -146,17 +169,17 @@ def curl_and_get_json_response(url, method: :get); require 'open3' let(:user) { "simpleuser" } let(:password) { "abc123" } let(:cacert) { "spec/fixtures/test_certs/ca.crt" } - let(:es_url) {"https://elasticsearch:9200"} + let(:es_url) { "https://#{get_host_port}" } let(:config) do { - "hosts" => ["elasticsearch:9200"], + "hosts" => [ get_host_port ], "user" => user, "password" => password, "ssl" => true, "cacert" => cacert, "index" => index } - end + end let(:curl_opts) { "-u #{user}:#{password}" } @@ -197,6 +220,8 @@ def curl_and_get_json_response(url, method: :get); require 'open3' else + let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password + it_behaves_like("an indexer", true) describe "with a password requiring escaping" do @@ -219,6 +244,32 @@ def curl_and_get_json_response(url, method: :get); require 'open3' include_examples("an indexer", true) end + context 'with enforced TLSv1.3 protocol' do + let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] } + + it_behaves_like("an indexer", true) + end + + context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do + let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] } + + let(:initial_events) { nil } + + it "does not ship events" do + curl_and_get_json_response index_url, method: :put # make sure index exists + Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop + sleep 2.5 + + curl_and_get_json_response "#{es_url}/_refresh", method: :post + + result = curl_and_get_json_response "#{index_url}/_count?q=*" + cur_count = result["count"] + expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a + # [Manticore::ClientProtocolException] Received fatal alert: protocol_version + end + + end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3' + end end diff --git a/spec/unit/outputs/elasticsearch_ssl_spec.rb b/spec/unit/outputs/elasticsearch_ssl_spec.rb index 3186eefc4..b7b9857c8 100644 --- a/spec/unit/outputs/elasticsearch_ssl_spec.rb +++ b/spec/unit/outputs/elasticsearch_ssl_spec.rb @@ -33,7 +33,7 @@ it "should pass the flag to the ES client" do expect(::Manticore::Client).to receive(:new) do |args| - expect(args[:ssl]).to eq(:enabled => true, :verify => :disable) + expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable) end.and_return(manticore_double) subject.register