Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Elasticsearch upgrade #210

Merged
merged 7 commits into from
Mar 23, 2023
Merged

Elasticsearch upgrade #210

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
436deed
make sure settings hash is what elasticsearch expects
wkdewey Oct 21, 2022
8a35aa2
change where mappings are posted for es upgrade
wkdewey Oct 21, 2022
2c7fa5c
add headers to ES requests for authorization
wkdewey Oct 25, 2022
25f0a2b
add method to construct basic auth header from options
wkdewey Oct 25, 2022
0ec2665
remove debugging code
wkdewey Oct 25, 2022
7b90a09
update conditional logic for status code, dynamic_templates key
wkdewey Oct 25, 2022
f982173
change endpoint for delete_by_query for ES8 compatibility
wkdewey Jan 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions lib/datura/elasticsearch/alias.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def self.add
{ add: { alias: ali, index: idx } }
]
}
RestClient.post(base_url, data.to_json, { content_type: :json }) { |res, req, result|
RestClient.post(base_url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
puts "Successfully added alias #{ali}. Current alias list:"
Expand All @@ -40,15 +40,15 @@ def self.delete

url = File.join(options["es_path"], idx, "_alias", ali)

res = JSON.parse(RestClient.delete(url))
res = JSON.parse(RestClient.delete(url, @auth_header))
puts JSON.pretty_generate(res)
list
end

def self.list
options = Datura::Options.new({}).all

res = RestClient.get(File.join(options["es_path"], "_aliases"))
res = RestClient.get(File.join(options["es_path"], "_aliases"), )
JSON.pretty_generate(JSON.parse(res))
end

Expand Down
4 changes: 2 additions & 2 deletions lib/datura/elasticsearch/data.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def self.clear_all(options)
if confirm == "Yes I'm sure"
url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true")
json = { "query" => { "match_all" => {} } }
RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result|
RestClient.post(url, json.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
Expand All @@ -66,7 +66,7 @@ def self.clear_index(options)

if confirmation
data = self.build_clear_data(options)
RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result|
RestClient.post(url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
Expand Down
18 changes: 11 additions & 7 deletions lib/datura/elasticsearch/index.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
require "json"
require "rest-client"
require "yaml"
require "base64"

require_relative "./../elasticsearch.rb"

Expand All @@ -25,6 +26,7 @@ def initialize(options = nil, schema_mapping: false)

# yaml settings (if exist) and mappings
@requested_schema = YAML.load_file(@options["es_schema"])
@auth_header = Datura::Helpers.construct_auth_header(@options)
# if requested, grab the mapping currently associated with this index
# otherwise wait until after the requested schema is loaded
get_schema_mapping if schema_mapping
Expand All @@ -34,15 +36,15 @@ def create
json = @requested_schema["settings"].to_json
puts "Creating ES index for API version #{@options["api_version"]}: #{@pretty_url}"
if json && json != "null"
RestClient.put(@pretty_url, json, { content_type: :json }) { |res, req, result|
RestClient.put(@pretty_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
raise "#{result.code} error creating Elasticsearch index: #{res}"
end
}
else
RestClient.put(@pretty_url, nil) { |res, req, result|
RestClient.put(@pretty_url, nil, @auth_header) { |res, req, result|
if result.code == "200"
puts res
else
Expand All @@ -55,15 +57,15 @@ def create
def delete
puts "Deleting #{@options["es_index"]} via url #{@pretty_url}"

RestClient.delete(@pretty_url) { |res, req, result|
RestClient.delete(@pretty_url, @auth_header) { |res, req, result|
if result.code != "200"
raise "#{result.code} error deleting Elasticsearch index: #{res}"
end
}
end

def get_schema
RestClient.get(@mapping_url) { |res, req, result|
RestClient.get(@mapping_url, @auth_header) { |res, req, result|
if result.code == "200"
JSON.parse(res)
else
Expand Down Expand Up @@ -110,7 +112,7 @@ def set_schema
json = @requested_schema["mappings"].to_json

puts "Setting schema: #{@mapping_url}"
RestClient.put(@mapping_url, json, { content_type: :json }) { |res, req, result|
RestClient.put(@mapping_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
Expand Down Expand Up @@ -206,8 +208,9 @@ def self.clear_all(options)
confirm = STDIN.gets.chomp
if confirm == "Yes I'm sure"
url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true")
auth_header = Datura::Helpers.construct_auth_header(options)
json = { "query" => { "match_all" => {} } }
RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result|
RestClient.post(url, json.to_json, auth_header.merge({ content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
Expand All @@ -226,7 +229,8 @@ def self.clear_index(options)

if confirmation
data = self.build_clear_data(options)
RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result|
auth_header = Datura::Helpers.construct_auth_header(options)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note for future reference on committing. This helper method isn't included in this commit also, so the code would be broken at this commit. It's added in the next commit, but in the future these kinds of things need to be committed together.

RestClient.post(url, data.to_json, auth_header.merge({content_type: :json })) { |res, req, result|
if result.code == "200"
puts res
else
Expand Down
5 changes: 4 additions & 1 deletion lib/datura/file_type.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def initialize(location, options)
@out_html = File.join(output, "html")
@out_iiif = File.join(output, "iiif")
@out_solr = File.join(output, "solr")
@auth_header = Datura::Helpers.construct_auth_header(options)
Datura::Helpers.make_dirs(@out_es, @out_html, @out_iiif, @out_solr)
# script locations set in child classes
end
Expand Down Expand Up @@ -68,7 +69,9 @@ def post_es(es)
# NOTE: If you need to do partial updates rather than replacement of doc
# you will need to add _update at the end of this URL
begin
RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } )
puts @auth_header
byebug
RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) )
rescue => e
error = "Error transforming or posting to ES for #{self.filename(false)}: #{e}"
end
Expand Down