diff --git a/.travis.yml b/.travis.yml index 17c17c020..d164a0b0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,7 @@ services: branches: only: - master - - travis - - 5.x - 6.x - - 2.x matrix: include: @@ -29,7 +26,7 @@ matrix: jdk: oraclejdk8 env: RAILS_VERSIONS=5.0 - - rvm: 2.6.1 + - rvm: 2.6.2 jdk: oraclejdk8 env: RAILS_VERSIONS=4.0,5.0,6.0 @@ -39,12 +36,13 @@ matrix: env: global: - - ELASTICSEARCH_VERSION=7.0.0-beta1 + - ELASTICSEARCH_VERSION=7.0.0 + - TEST_ES_SERVER=http://localhost:9250 - TEST_CLUSTER_PORT=9250 - QUIET=true before_install: - - TEST_CLUSTER_PORT=9250 source ./travis_before_script.sh + - source ./travis_before_script.sh - gem update --system - gem update bundler - gem --version diff --git a/Rakefile b/Rakefile index 99e4e2c8d..07dbe81ee 100644 --- a/Rakefile +++ b/Rakefile @@ -22,6 +22,33 @@ subprojects << 'elasticsearch-model' unless defined?(JRUBY_VERSION) __current__ = Pathname( File.expand_path('..', __FILE__) ) +def admin_client + $admin_client ||= begin + transport_options = {} + test_suite = ENV['TEST_SUITE'].freeze + + if hosts = ENV['TEST_ES_SERVER'] || ENV['ELASTICSEARCH_HOSTS'] + split_hosts = hosts.split(',').map do |host| + /(http\:\/\/)?(\S+)/.match(host)[2] + end + + host, port = split_hosts.first.split(':') + end + + if test_suite == 'security' + transport_options.merge!(:ssl => { verify: false, + ca_path: CERT_DIR }) + + password = ENV['ELASTIC_PASSWORD'] + user = ENV['ELASTIC_USER'] || 'elastic' + url = "https://#{user}:#{password}@#{host}:#{port}" + else + url = "http://#{host || 'localhost'}:#{port || 9200}" + end + Elasticsearch::Client.new(host: url, transport_options: transport_options) + end +end + task :default do system "rake --tasks" end @@ -53,9 +80,7 @@ namespace :bundle do subprojects.each do |project| sh "rm -f #{__current__.join(project)}/Gemfile.lock" end - sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/3.0.gemfile.lock" - sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/4.0.gemfile.lock" - sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/5.0.gemfile.lock" + sh "rm -f #{__current__.join('elasticsearch-model/gemfiles')}/*.lock" end end @@ -132,7 +157,7 @@ namespace :test do end desc "Run all tests in all subprojects" - task :all do + task :all => :wait_for_green do subprojects.each do |project| puts '-'*80 sh "cd #{project} && " + @@ -163,6 +188,32 @@ namespace :test do end end + +desc "Wait for elasticsearch cluster to be in green state" +task :wait_for_green do + require 'elasticsearch' + + ready = nil + 5.times do |i| + begin + puts "Attempting to wait for green status: #{i + 1}" + if admin_client.cluster.health(wait_for_status: 'green', timeout: '50s') + ready = true + break + end + rescue Elasticsearch::Transport::Transport::Errors::RequestTimeout => ex + puts "Couldn't confirm green status.\n#{ex.inspect}." + rescue Faraday::ConnectionFailed => ex + puts "Couldn't connect to Elasticsearch.\n#{ex.inspect}." + sleep(30) + end + end + unless ready + puts "Couldn't connect to Elasticsearch, aborting program." + exit(1) + end +end + desc "Generate documentation for all subprojects" task :doc do subprojects.each do |project| diff --git a/elasticsearch-model/.gitignore b/elasticsearch-model/.gitignore index 52f8d0334..37746eee1 100644 --- a/elasticsearch-model/.gitignore +++ b/elasticsearch-model/.gitignore @@ -16,4 +16,6 @@ test/tmp test/version_tmp tmp -gemfiles/*.gemfile.lock + +gemfiles/*.lock + diff --git a/elasticsearch-model/gemfiles/6.0.gemfile b/elasticsearch-model/gemfiles/6.0.gemfile index 035a754cc..484a9df3b 100644 --- a/elasticsearch-model/gemfiles/6.0.gemfile +++ b/elasticsearch-model/gemfiles/6.0.gemfile @@ -20,6 +20,7 @@ # $ BUNDLE_GEMFILE=./gemfiles/6.0.gemfile bundle install # $ BUNDLE_GEMFILE=./gemfiles/6.0.gemfile bundle exec rake test:integration + source 'https://rubygems.org' gemspec path: '../' diff --git a/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb index 7cd8456ac..b4df631f4 100644 --- a/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb +++ b/elasticsearch-model/lib/elasticsearch/model/adapters/multiple.rb @@ -75,8 +75,8 @@ def __records_for_klass(klass, ids) klass.where(klass.primary_key => ids) when Elasticsearch::Model::Adapter::Mongoid.equal?(adapter) klass.where(:id.in => ids) - else - klass.find(ids) + else + klass.find(ids) end end @@ -108,13 +108,21 @@ def __ids_by_type def __type_for_hit(hit) @@__types ||= {} - @@__types[ "#{hit[:_index]}::#{hit[:_type]}" ] ||= begin + key = "#{hit[:_index]}::#{hit[:_type]}" if hit[:_type] && hit[:_type] != '_doc' + key = hit[:_index] unless key + + @@__types[key] ||= begin Registry.all.detect do |model| - model.index_name == hit[:_index] && model.document_type == hit[:_type] + (model.index_name == hit[:_index] && __no_type?(hit)) || + (model.index_name == hit[:_index] && model.document_type == hit[:_type]) end end end + def __no_type?(hit) + hit[:_type].nil? || hit[:_type] == '_doc' + end + # Returns the adapter registered for a particular `klass` or `nil` if not available # # @api private diff --git a/elasticsearch-model/lib/elasticsearch/model/indexing.rb b/elasticsearch-model/lib/elasticsearch/model/indexing.rb index da3c7f79b..69baf597e 100644 --- a/elasticsearch-model/lib/elasticsearch/model/indexing.rb +++ b/elasticsearch-model/lib/elasticsearch/model/indexing.rb @@ -56,9 +56,7 @@ class Mappings # @private TYPES_WITH_EMBEDDED_PROPERTIES = %w(object nested) - def initialize(type, options={}) - raise ArgumentError, "`type` is missing" if type.nil? - + def initialize(type = nil, options={}) @type = type @options = options @mapping = {} @@ -89,7 +87,11 @@ def indexes(name, options={}, &block) end def to_hash - { @type.to_sym => @options.merge( properties: @mapping ) } + if @type + { @type.to_sym => @options.merge( properties: @mapping ) } + else + @options.merge( properties: @mapping ) + end end def as_json(options={}) @@ -246,10 +248,12 @@ def create_index!(options={}) delete_index!(options.merge index: target_index) if options[:force] unless index_exists?(index: target_index) - self.client.indices.create index: target_index, - body: { - settings: settings, - mappings: mappings } + options.delete(:force) + self.client.indices.create({ index: target_index, + body: { + settings: settings, + mappings: mappings } + }.merge(options)) end end diff --git a/elasticsearch-model/lib/elasticsearch/model/naming.rb b/elasticsearch-model/lib/elasticsearch/model/naming.rb index 961959d61..9db0a9b08 100644 --- a/elasticsearch-model/lib/elasticsearch/model/naming.rb +++ b/elasticsearch-model/lib/elasticsearch/model/naming.rb @@ -108,10 +108,7 @@ def default_index_name self.model_name.collection.gsub(/\//, '-') end - def default_document_type - DEFAULT_DOC_TYPE - end - + def default_document_type; end end module InstanceMethods diff --git a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/basic_spec.rb b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/basic_spec.rb index 669b28bfc..eef1bd42c 100644 --- a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/basic_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/basic_spec.rb @@ -19,339 +19,377 @@ describe Elasticsearch::Model::Adapter::ActiveRecord do - before(:all) do - ActiveRecord::Schema.define(:version => 1) do - create_table :articles do |t| - t.string :title - t.string :body - t.integer :clicks, :default => 0 - t.datetime :created_at, :default => 'NOW()' + context 'when a document_type is not defined for the Model' do + + before do + ActiveRecord::Schema.define(:version => 1) do + create_table :article_no_types do |t| + t.string :title + t.string :body + t.integer :clicks, :default => 0 + t.datetime :created_at, :default => 'NOW()' + end end - end - Article.delete_all - Article.__elasticsearch__.create_index!(force: true) + ArticleNoType.delete_all + ArticleNoType.__elasticsearch__.create_index!(force: true) - Article.create!(title: 'Test', body: '', clicks: 1) - Article.create!(title: 'Testing Coding', body: '', clicks: 2) - Article.create!(title: 'Coding', body: '', clicks: 3) + ArticleNoType.create!(title: 'Test', body: '', clicks: 1) + ArticleNoType.create!(title: 'Testing Coding', body: '', clicks: 2) + ArticleNoType.create!(title: 'Coding', body: '', clicks: 3) - Article.__elasticsearch__.refresh_index! - end + ArticleNoType.__elasticsearch__.refresh_index! + end - describe 'indexing a document' do + describe 'indexing a document' do - let(:search_result) do - Article.search('title:test') - end + let(:search_result) do + ArticleNoType.search('title:test') + end - it 'allows searching for documents' do - expect(search_result.results.size).to be(2) - expect(search_result.records.size).to be(2) + it 'allows searching for documents' do + expect(search_result.results.size).to be(2) + expect(search_result.records.size).to be(2) + end end end - describe '#results' do + context 'when a document_type is defined for the Model' do - let(:search_result) do - Article.search('title:test') - end + before(:all) do + ActiveRecord::Schema.define(:version => 1) do + create_table :articles do |t| + t.string :title + t.string :body + t.integer :clicks, :default => 0 + t.datetime :created_at, :default => 'NOW()' + end + end - it 'returns an instance of Response::Result' do - expect(search_result.results.first).to be_a(Elasticsearch::Model::Response::Result) - end + Article.delete_all + Article.__elasticsearch__.create_index!(force: true, include_type_name: true) + + Article.create!(title: 'Test', body: '', clicks: 1) + Article.create!(title: 'Testing Coding', body: '', clicks: 2) + Article.create!(title: 'Coding', body: '', clicks: 3) - it 'prooperly loads the document' do - expect(search_result.results.first.title).to eq('Test') + Article.__elasticsearch__.refresh_index! end - context 'when the result contains other data' do + describe 'indexing a document' do let(:search_result) do - Article.search(query: { match: { title: 'test' } }, highlight: { fields: { title: {} } }) + Article.search('title:test') end - it 'allows access to the Elasticsearch result' do - expect(search_result.results.first.title).to eq('Test') - expect(search_result.results.first.title?).to be(true) - expect(search_result.results.first.boo?).to be(false) - expect(search_result.results.first.highlight?).to be(true) - expect(search_result.results.first.highlight.title?).to be(true) - expect(search_result.results.first.highlight.boo?).to be(false) + it 'allows searching for documents' do + expect(search_result.results.size).to be(2) + expect(search_result.records.size).to be(2) end end - end - describe '#records' do + describe '#results' do - let(:search_result) do - Article.search('title:test') - end - - it 'returns an instance of the model' do - expect(search_result.records.first).to be_a(Article) - end + let(:search_result) do + Article.search('title:test') + end - it 'prooperly loads the document' do - expect(search_result.records.first.title).to eq('Test') - end - end + it 'returns an instance of Response::Result' do + expect(search_result.results.first).to be_a(Elasticsearch::Model::Response::Result) + end - describe 'Enumerable' do + it 'prooperly loads the document' do + expect(search_result.results.first.title).to eq('Test') + end - let(:search_result) do - Article.search('title:test') - end + context 'when the result contains other data' do - it 'allows iteration over results' do - expect(search_result.results.map(&:_id)).to eq(['1', '2']) - end + let(:search_result) do + Article.search(query: { match: { title: 'test' } }, highlight: { fields: { title: {} } }) + end - it 'allows iteration over records' do - expect(search_result.records.map(&:id)).to eq([1, 2]) + it 'allows access to the Elasticsearch result' do + expect(search_result.results.first.title).to eq('Test') + expect(search_result.results.first.title?).to be(true) + expect(search_result.results.first.boo?).to be(false) + expect(search_result.results.first.highlight?).to be(true) + expect(search_result.results.first.highlight.title?).to be(true) + expect(search_result.results.first.highlight.boo?).to be(false) + end + end end - end - - describe '#id' do - let(:search_result) do - Article.search('title:test') - end + describe '#records' do - it 'returns the id' do - expect(search_result.results.first.id).to eq('1') - end - end + let(:search_result) do + Article.search('title:test') + end - describe '#id' do + it 'returns an instance of the model' do + expect(search_result.records.first).to be_a(Article) + end - let(:search_result) do - Article.search('title:test') + it 'prooperly loads the document' do + expect(search_result.records.first.title).to eq('Test') + end end - it 'returns the type' do - expect(search_result.results.first.type).to eq('article') - end - end + describe 'Enumerable' do - describe '#each_with_hit' do + let(:search_result) do + Article.search('title:test') + end - let(:search_result) do - Article.search('title:test') - end + it 'allows iteration over results' do + expect(search_result.results.map(&:_id)).to eq(['1', '2']) + end - it 'returns the record with the Elasticsearch hit' do - search_result.records.each_with_hit do |r, h| - expect(h._score).not_to be_nil - expect(h._source.title).not_to be_nil + it 'allows iteration over records' do + expect(search_result.records.map(&:id)).to eq([1, 2]) end end - end - describe 'search results order' do + describe '#id' do - let(:search_result) do - Article.search(query: { match: { title: 'code' }}, sort: { clicks: :desc }) - end + let(:search_result) do + Article.search('title:test') + end - it 'preserves the search results order when accessing a single record' do - expect(search_result.records[0].clicks).to be(3) - expect(search_result.records[1].clicks).to be(2) - expect(search_result.records.first).to eq(search_result.records[0]) + it 'returns the id' do + expect(search_result.results.first.id).to eq('1') + end end - it 'preserves the search results order for the list of records' do - search_result.records.each_with_hit do |r, h| - expect(r.id.to_s).to eq(h._id) + describe '#id' do + + let(:search_result) do + Article.search('title:test') end - search_result.records.map_with_hit do |r, h| - expect(r.id.to_s).to eq(h._id) + it 'returns the type' do + expect(search_result.results.first.type).to eq('article') end end - end - describe 'a paged collection' do + describe '#each_with_hit' do - let(:search_result) do - Article.search(query: { match: { title: { query: 'test' } } }, - size: 2, - from: 1) - end + let(:search_result) do + Article.search('title:test') + end - it 'applies the paged options to the search' do - expect(search_result.results.size).to eq(1) - expect(search_result.results.first.title).to eq('Testing Coding') - expect(search_result.records.size).to eq(1) - expect(search_result.records.first.title).to eq('Testing Coding') + it 'returns the record with the Elasticsearch hit' do + search_result.records.each_with_hit do |r, h| + expect(h._score).not_to be_nil + expect(h._source.title).not_to be_nil + end + end end - end - describe '#destroy' do + describe 'search results order' do - before do - Article.create!(title: 'destroy', body: '', clicks: 1) - Article.__elasticsearch__.refresh_index! - Article.where(title: 'destroy').first.destroy + let(:search_result) do + Article.search(query: { match: { title: 'code' }}, sort: { clicks: :desc }) + end - Article.__elasticsearch__.refresh_index! - end + it 'preserves the search results order when accessing a single record' do + expect(search_result.records[0].clicks).to be(3) + expect(search_result.records[1].clicks).to be(2) + expect(search_result.records.first).to eq(search_result.records[0]) + end - let(:search_result) do - Article.search('title:test') - end + it 'preserves the search results order for the list of records' do + search_result.records.each_with_hit do |r, h| + expect(r.id.to_s).to eq(h._id) + end - it 'removes the document from the index' do - expect(Article.count).to eq(3) - expect(search_result.results.size).to eq(2) - expect(search_result.records.size).to eq(2) + search_result.records.map_with_hit do |r, h| + expect(r.id.to_s).to eq(h._id) + end + end end - end - describe 'full document updates' do + describe 'a paged collection' do - before do - article = Article.create!(title: 'update', body: '', clicks: 1) - Article.__elasticsearch__.refresh_index! - article.title = 'Writing' - article.save + let(:search_result) do + Article.search(query: { match: { title: { query: 'test' } } }, + size: 2, + from: 1) + end - Article.__elasticsearch__.refresh_index! + it 'applies the paged options to the search' do + expect(search_result.results.size).to eq(1) + expect(search_result.results.first.title).to eq('Testing Coding') + expect(search_result.records.size).to eq(1) + expect(search_result.records.first.title).to eq('Testing Coding') + end end - let(:search_result) do - Article.search('title:write') - end + describe '#destroy' do - it 'applies the update' do - expect(search_result.results.size).to eq(1) - expect(search_result.records.size).to eq(1) - end - end + before do + Article.create!(title: 'destroy', body: '', clicks: 1) + Article.__elasticsearch__.refresh_index! + Article.where(title: 'destroy').first.destroy - describe 'attribute updates' do + Article.__elasticsearch__.refresh_index! + end - before do - article = Article.create!(title: 'update', body: '', clicks: 1) - Article.__elasticsearch__.refresh_index! - article.title = 'special' - article.save + let(:search_result) do + Article.search('title:test') + end - Article.__elasticsearch__.refresh_index! + it 'removes the document from the index' do + expect(Article.count).to eq(3) + expect(search_result.results.size).to eq(2) + expect(search_result.records.size).to eq(2) + end end - let(:search_result) do - Article.search('title:special') - end + describe 'full document updates' do - it 'applies the update' do - expect(search_result.results.size).to eq(1) - expect(search_result.records.size).to eq(1) - end - end + before do + article = Article.create!(title: 'update', body: '', clicks: 1) + Article.__elasticsearch__.refresh_index! + article.title = 'Writing' + article.save - describe '#save' do + Article.__elasticsearch__.refresh_index! + end - before do - article = Article.create!(title: 'save', body: '', clicks: 1) + let(:search_result) do + Article.search('title:write') + end - ActiveRecord::Base.transaction do - article.body = 'dummy' - article.save + it 'applies the update' do + expect(search_result.results.size).to eq(1) + expect(search_result.records.size).to eq(1) + end + end + + describe 'attribute updates' do + before do + article = Article.create!(title: 'update', body: '', clicks: 1) + Article.__elasticsearch__.refresh_index! article.title = 'special' article.save + + Article.__elasticsearch__.refresh_index! end - article.__elasticsearch__.update_document - Article.__elasticsearch__.refresh_index! - end + let(:search_result) do + Article.search('title:special') + end - let(:search_result) do - Article.search('body:dummy') + it 'applies the update' do + expect(search_result.results.size).to eq(1) + expect(search_result.records.size).to eq(1) + end end - it 'applies the save' do - expect(search_result.results.size).to eq(1) - expect(search_result.records.size).to eq(1) - end - end + describe '#save' do - describe 'a DSL search' do + before do + article = Article.create!(title: 'save', body: '', clicks: 1) - let(:search_result) do - Article.search(query: { match: { title: { query: 'test' } } }) - end + ActiveRecord::Base.transaction do + article.body = 'dummy' + article.save - it 'returns the results' do - expect(search_result.results.size).to eq(2) - expect(search_result.records.size).to eq(2) - end - end + article.title = 'special' + article.save + end - describe 'chaining SQL queries on response.records' do + article.__elasticsearch__.update_document + Article.__elasticsearch__.refresh_index! + end - let(:search_result) do - Article.search(query: { match: { title: { query: 'test' } } }) - end + let(:search_result) do + Article.search('body:dummy') + end - it 'executes the SQL request with the chained query criteria' do - expect(search_result.records.size).to eq(2) - expect(search_result.records.where(title: 'Test').size).to eq(1) - expect(search_result.records.where(title: 'Test').first.title).to eq('Test') + it 'applies the save' do + expect(search_result.results.size).to eq(1) + expect(search_result.records.size).to eq(1) + end end - end - - describe 'ordering of SQL queries' do - context 'when order is called on the ActiveRecord query' do + describe 'a DSL search' do let(:search_result) do - Article.search query: { match: { title: { query: 'test' } } } + Article.search(query: { match: { title: { query: 'test' } } }) end - it 'allows the SQL query to be ordered independent of the Elasticsearch results order', unless: active_record_at_least_4? do - expect(search_result.records.order('title DESC').first.title).to eq('Testing Coding') - expect(search_result.records.order('title DESC')[0].title).to eq('Testing Coding') - end - - it 'allows the SQL query to be ordered independent of the Elasticsearch results order', if: active_record_at_least_4? do - expect(search_result.records.order(title: :desc).first.title).to eq('Testing Coding') - expect(search_result.records.order(title: :desc)[0].title).to eq('Testing Coding') + it 'returns the results' do + expect(search_result.results.size).to eq(2) + expect(search_result.records.size).to eq(2) end end - context 'when more methods are chained on the ActiveRecord query' do + describe 'chaining SQL queries on response.records' do let(:search_result) do - Article.search query: {match: {title: {query: 'test'}}} + Article.search(query: { match: { title: { query: 'test' } } }) end - it 'allows the SQL query to be ordered independent of the Elasticsearch results order', if: active_record_at_least_4? do - expect(search_result.records.distinct.order(title: :desc).first.title).to eq('Testing Coding') - expect(search_result.records.distinct.order(title: :desc)[0].title).to eq('Testing Coding') + it 'executes the SQL request with the chained query criteria' do + expect(search_result.records.size).to eq(2) + expect(search_result.records.where(title: 'Test').size).to eq(1) + expect(search_result.records.where(title: 'Test').first.title).to eq('Test') end end - end - describe 'access to the response via methods' do + describe 'ordering of SQL queries' do + + context 'when order is called on the ActiveRecord query' do + + let(:search_result) do + Article.search query: { match: { title: { query: 'test' } } } + end + + it 'allows the SQL query to be ordered independent of the Elasticsearch results order', unless: active_record_at_least_4? do + expect(search_result.records.order('title DESC').first.title).to eq('Testing Coding') + expect(search_result.records.order('title DESC')[0].title).to eq('Testing Coding') + end + + it 'allows the SQL query to be ordered independent of the Elasticsearch results order', if: active_record_at_least_4? do + expect(search_result.records.order(title: :desc).first.title).to eq('Testing Coding') + expect(search_result.records.order(title: :desc)[0].title).to eq('Testing Coding') + end + end + + context 'when more methods are chained on the ActiveRecord query' do + + let(:search_result) do + Article.search query: {match: {title: {query: 'test'}}} + end - let(:search_result) do - Article.search(query: { match: { title: { query: 'test' } } }, - aggregations: { - dates: { date_histogram: { field: 'created_at', interval: 'hour' } }, - clicks: { global: {}, aggregations: { min: { min: { field: 'clicks' } } } } - }, - suggest: { text: 'tezt', title: { term: { field: 'title', suggest_mode: 'always' } } }) + it 'allows the SQL query to be ordered independent of the Elasticsearch results order', if: active_record_at_least_4? do + expect(search_result.records.distinct.order(title: :desc).first.title).to eq('Testing Coding') + expect(search_result.records.distinct.order(title: :desc)[0].title).to eq('Testing Coding') + end + end end - it 'allows document keys to be access via methods' do - expect(search_result.aggregations.dates.buckets.first.doc_count).to eq(2) - expect(search_result.aggregations.clicks.doc_count).to eq(6) - expect(search_result.aggregations.clicks.min.value).to eq(1.0) - expect(search_result.aggregations.clicks.max).to be_nil - expect(search_result.suggestions.title.first.options.size).to eq(1) - expect(search_result.suggestions.terms).to eq(['test']) + describe 'access to the response via methods' do + + let(:search_result) do + Article.search(query: { match: { title: { query: 'test' } } }, + aggregations: { + dates: { date_histogram: { field: 'created_at', interval: 'hour' } }, + clicks: { global: {}, aggregations: { min: { min: { field: 'clicks' } } } } + }, + suggest: { text: 'tezt', title: { term: { field: 'title', suggest_mode: 'always' } } }) + end + + it 'allows document keys to be access via methods' do + expect(search_result.aggregations.dates.buckets.first.doc_count).to eq(2) + expect(search_result.aggregations.clicks.doc_count).to eq(6) + expect(search_result.aggregations.clicks.min.value).to eq(1.0) + expect(search_result.aggregations.clicks.max).to be_nil + expect(search_result.suggestions.title.first.options.size).to eq(1) + expect(search_result.suggestions.terms).to eq(['test']) + end end end end diff --git a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/namespaced_model_spec.rb b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/namespaced_model_spec.rb index 9f4422050..3f93eca9d 100644 --- a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/namespaced_model_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/namespaced_model_spec.rb @@ -27,7 +27,7 @@ end MyNamespace::Book.delete_all - MyNamespace::Book.__elasticsearch__.create_index!(force: true) + MyNamespace::Book.__elasticsearch__.create_index!(force: true, include_type_name: true) MyNamespace::Book.create!(title: 'Test') MyNamespace::Book.__elasticsearch__.refresh_index! end diff --git a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/parent_child_spec.rb b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/parent_child_spec.rb index 60d642103..675ab3b73 100644 --- a/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/parent_child_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/adapters/active_record/parent_child_spec.rb @@ -38,7 +38,7 @@ add_index(:answers, :question_id) unless index_exists?(:answers, :question_id) clear_tables(Question) - ParentChildSearchable.create_index!(force: true) + ParentChildSearchable.create_index!(force: true, include_type_name: true) q_1 = Question.create!(title: 'First Question', author: 'John') q_2 = Question.create!(title: 'Second Question', author: 'Jody') diff --git a/elasticsearch-model/spec/elasticsearch/model/indexing_spec.rb b/elasticsearch-model/spec/elasticsearch/model/indexing_spec.rb index f625b538e..3ee081969 100644 --- a/elasticsearch-model/spec/elasticsearch/model/indexing_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/indexing_spec.rb @@ -101,10 +101,8 @@ class NotFound < Exception; end expect(DummyIndexingModel.mappings).to be_a(Elasticsearch::Model::Indexing::Mappings) end - it 'raises an exception when there is no type passed to the #initialize method' do - expect { - Elasticsearch::Model::Indexing::Mappings.new - }.to raise_exception(ArgumentError) + it 'does not raise an exception when there is no type passed to the #initialize method' do + expect(Elasticsearch::Model::Indexing::Mappings.new) end it 'should be convertible to a hash' do @@ -115,7 +113,7 @@ class NotFound < Exception; end expect(Elasticsearch::Model::Indexing::Mappings.new(:mytype, { foo: 'bar' }).as_json).to eq(expected_mapping_hash) end - context 'when specific mappings are defined' do + context 'when a type is specified' do let(:mappings) do Elasticsearch::Model::Indexing::Mappings.new(:mytype) @@ -130,6 +128,65 @@ class NotFound < Exception; end expect(mappings.to_hash[:mytype][:properties][:foo][:type]).to eq('boolean') end + it 'uses text as the default field type' do + expect(mappings.to_hash[:mytype][:properties][:bar][:type]).to eq('text') + end + + context 'when the \'include_type_name\' option is specified' do + + let(:mappings) do + Elasticsearch::Model::Indexing::Mappings.new(:mytype, include_type_name: true) + end + + before do + mappings.indexes :foo, { type: 'boolean', include_in_all: false } + end + + it 'creates the correct mapping definition' do + expect(mappings.to_hash[:mytype][:properties][:foo][:type]).to eq('boolean') + end + + it 'sets the \'include_type_name\' option' do + expect(mappings.to_hash[:mytype][:include_type_name]).to eq(true) + end + end + end + + context 'when a type is not specified' do + + let(:mappings) do + Elasticsearch::Model::Indexing::Mappings.new + end + + before do + mappings.indexes :foo, { type: 'boolean', include_in_all: false } + mappings.indexes :bar + end + + it 'creates the correct mapping definition' do + expect(mappings.to_hash[:properties][:foo][:type]).to eq('boolean') + end + + it 'uses text as the default type' do + expect(mappings.to_hash[:properties][:bar][:type]).to eq('text') + end + end + + context 'when specific mappings are defined' do + + let(:mappings) do + Elasticsearch::Model::Indexing::Mappings.new(:mytype, include_type_name: true) + end + + before do + mappings.indexes :foo, { type: 'boolean', include_in_all: false } + mappings.indexes :bar + end + + it 'creates the correct mapping definition' do + expect(mappings.to_hash[:mytype][:properties][:foo][:type]).to eq('boolean') + end + it 'uses text as the default type' do expect(mappings.to_hash[:mytype][:properties][:bar][:type]).to eq('text') end @@ -186,6 +243,10 @@ class NotFound < Exception; end expect(mappings.to_hash[:mytype][:properties][:foo_nested_as_symbol][:properties]).not_to be_nil expect(mappings.to_hash[:mytype][:properties][:foo_nested_as_symbol][:fields]).to be_nil end + + it 'defines the settings' do + expect(mappings.to_hash[:mytype][:include_type_name]).to be(true) + end end end @@ -197,7 +258,7 @@ class NotFound < Exception; end end let(:expected_mappings_hash) do - { _doc: { foo: "boo", bar: "bam", properties: {} } } + { foo: "boo", bar: "bam", properties: {} } end it 'sets the mappings' do @@ -213,7 +274,25 @@ class NotFound < Exception; end end it 'sets the mappings' do - expect(DummyIndexingModel.mapping.to_hash[:_doc][:properties][:foo][:type]).to eq('boolean') + expect(DummyIndexingModel.mapping.to_hash[:properties][:foo][:type]).to eq('boolean') + end + end + + context 'when the class has a document_type' do + + before do + DummyIndexingModel.instance_variable_set(:@mapping, nil) + DummyIndexingModel.document_type(:mytype) + DummyIndexingModel.mappings(foo: 'boo') + DummyIndexingModel.mappings(bar: 'bam') + end + + let(:expected_mappings_hash) do + { mytype: { foo: "boo", bar: "bam", properties: {} } } + end + + it 'sets the mappings' do + expect(DummyIndexingModel.mappings.to_hash).to eq(expected_mappings_hash) end end end @@ -755,8 +834,8 @@ class ::DummyIndexingModelForCreate context 'when options are not provided' do let(:expected_body) do - { mappings: { _doc: { properties: { foo: { analyzer: 'keyword', - type: 'text' } } } }, + { mappings: { properties: { foo: { analyzer: 'keyword', + type: 'text' } } }, settings: { index: { number_of_shards: 1 } } } end @@ -806,8 +885,8 @@ class ::DummyIndexingModelForCreate before do expect(DummyIndexingModelForCreate).to receive(:client).and_return(client) - expect(DummyIndexingModelForCreate).to receive(:index_exists?).and_return(false) expect(DummyIndexingModelForCreate).to receive(:delete_index!).and_return(true) + expect(DummyIndexingModelForCreate).to receive(:index_exists?).and_return(false) expect(indices).to receive(:create).and_raise(Exception) end @@ -827,8 +906,8 @@ class ::DummyIndexingModelForCreate end let(:expected_body) do - { mappings: { _doc: { properties: { foo: { analyzer: 'keyword', - type: 'text' } } } }, + { mappings: { properties: { foo: { analyzer: 'keyword', + type: 'text' } } }, settings: { index: { number_of_shards: 1 } } } end diff --git a/elasticsearch-model/spec/elasticsearch/model/naming_inheritance_spec.rb b/elasticsearch-model/spec/elasticsearch/model/naming_inheritance_spec.rb index 93b7ed95e..0d8efb9a3 100644 --- a/elasticsearch-model/spec/elasticsearch/model/naming_inheritance_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/naming_inheritance_spec.rb @@ -97,9 +97,9 @@ class ::Cat < ::Animal describe '#document_type' do - it 'returns the default document type' do - expect(TestBase.document_type).to eq('_doc') - expect(TestBase.new.document_type).to eq('_doc') + it 'returns nil' do + expect(TestBase.document_type).to be_nil + expect(TestBase.new.document_type).to be_nil end it 'returns the explicit document type' do diff --git a/elasticsearch-model/spec/elasticsearch/model/naming_spec.rb b/elasticsearch-model/spec/elasticsearch/model/naming_spec.rb index 9e2365bcd..bdd5aafad 100644 --- a/elasticsearch-model/spec/elasticsearch/model/naming_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/naming_spec.rb @@ -51,9 +51,9 @@ class DummyNamingModelInNamespace expect(::MyNamespace::DummyNamingModelInNamespace.new.index_name).to eq('my_namespace-dummy_naming_model_in_namespaces') end - it 'returns the document type' do - expect(DummyNamingModel.document_type).to eq('_doc') - expect(DummyNamingModel.new.document_type).to eq('_doc') + it 'returns nil' do + expect(DummyNamingModel.document_type).to be_nil + expect(DummyNamingModel.new.document_type).to be_nil end describe '#index_name' do @@ -141,8 +141,8 @@ class DummyNamingModelInNamespace describe '#document_type' do - it 'returns the document type' do - expect(DummyNamingModel.document_type).to eq('_doc') + it 'returns nil' do + expect(DummyNamingModel.document_type).to be_nil end context 'when the method is called with an argument' do diff --git a/elasticsearch-model/spec/elasticsearch/model/response/pagination/kaminari_spec.rb b/elasticsearch-model/spec/elasticsearch/model/response/pagination/kaminari_spec.rb index 3824a0d44..4ad8ab5a1 100644 --- a/elasticsearch-model/spec/elasticsearch/model/response/pagination/kaminari_spec.rb +++ b/elasticsearch-model/spec/elasticsearch/model/response/pagination/kaminari_spec.rb @@ -31,11 +31,6 @@ def self.document_type; 'bar'; end remove_classes(ModelClass) end - let(:response_document) do - { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, - 'hits' => { 'total' => 100, 'hits' => (1..100).to_a.map { |i| { _id: i } } } } - end - let(:search) do Elasticsearch::Model::Searching::SearchRequest.new(model, '*') end @@ -391,37 +386,87 @@ def self.document_type; 'bar'; end end end - context 'when the model is a single one' do + context 'when Elasticsearch version is < 7.0' do - let(:model) do - ModelClass + let(:response_document) do + { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, + 'hits' => { 'total' => 100, 'hits' => (1..100).to_a.map { |i| { _id: i } } } } end - let(:type_field) do - 'bar' - end + context 'when the model is a single one' do + + let(:model) do + ModelClass + end + + let(:type_field) do + 'bar' + end + + let(:index_field) do + 'foo' + end - let(:index_field) do - 'foo' + it_behaves_like 'a search request that can be paginated' end - it_behaves_like 'a search request that can be paginated' - end + context 'when the model is a multimodel' do + + let(:model) do + Elasticsearch::Model::Multimodel.new(ModelClass) + end - context 'when the model is a multimodel' do + let(:type_field) do + ['bar'] + end + + let(:index_field) do + ['foo'] + end - let(:model) do - Elasticsearch::Model::Multimodel.new(ModelClass) + it_behaves_like 'a search request that can be paginated' end + end - let(:type_field) do - ['bar'] + context 'when Elasticsearch version is >= 7.0' do + + let(:response_document) do + { 'took' => '5', 'timed_out' => false, '_shards' => {'one' => 'OK'}, + 'hits' => { 'total' => { 'value' => 100, 'relation' => 'eq' }, 'hits' => (1..100).to_a.map { |i| { _id: i } } } } end - let(:index_field) do - ['foo'] + context 'when the model is a single one' do + + let(:model) do + ModelClass + end + + let(:type_field) do + 'bar' + end + + let(:index_field) do + 'foo' + end + + it_behaves_like 'a search request that can be paginated' end - it_behaves_like 'a search request that can be paginated' + context 'when the model is a multimodel' do + + let(:model) do + Elasticsearch::Model::Multimodel.new(ModelClass) + end + + let(:type_field) do + ['bar'] + end + + let(:index_field) do + ['foo'] + end + + it_behaves_like 'a search request that can be paginated' + end end end diff --git a/elasticsearch-model/spec/support/app.rb b/elasticsearch-model/spec/support/app.rb index d5c2c8ace..e009f9371 100644 --- a/elasticsearch-model/spec/support/app.rb +++ b/elasticsearch-model/spec/support/app.rb @@ -30,6 +30,7 @@ require 'support/app/series' require 'support/app/mongoid_article' require 'support/app/article' +require 'support/app/article_no_type' require 'support/app/searchable' require 'support/app/category' require 'support/app/author' diff --git a/elasticsearch-model/spec/support/app/article_no_type.rb b/elasticsearch-model/spec/support/app/article_no_type.rb new file mode 100644 index 000000000..5a08746ba --- /dev/null +++ b/elasticsearch-model/spec/support/app/article_no_type.rb @@ -0,0 +1,37 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class ::ArticleNoType < ActiveRecord::Base + include Elasticsearch::Model + include Elasticsearch::Model::Callbacks + + settings index: {number_of_shards: 1, number_of_replicas: 0} do + mapping do + indexes :title, type: 'text', analyzer: 'snowball' + indexes :body, type: 'text' + indexes :clicks, type: 'integer' + indexes :created_at, type: 'date' + end + end + + def as_indexed_json(options = {}) + attributes + .symbolize_keys + .slice(:title, :body, :clicks, :created_at) + .merge(suggest_title: title) + end +end diff --git a/elasticsearch-model/spec/support/app/parent_and_child_searchable.rb b/elasticsearch-model/spec/support/app/parent_and_child_searchable.rb index e747bec2e..15b56186a 100644 --- a/elasticsearch-model/spec/support/app/parent_and_child_searchable.rb +++ b/elasticsearch-model/spec/support/app/parent_and_child_searchable.rb @@ -21,7 +21,7 @@ module ParentChildSearchable def create_index!(options={}) client = Question.__elasticsearch__.client - client.indices.delete index: INDEX_NAME rescue nil if options[:force] + client.indices.delete index: INDEX_NAME rescue nil if options.delete(:force) settings = Question.settings.to_hash.merge Answer.settings.to_hash mapping_properties = { join_field: { type: JOIN, @@ -31,10 +31,10 @@ def create_index!(options={}) Answer.mappings.to_hash[:doc][:properties]) mappings = { doc: { properties: merged_properties }} - client.indices.create index: INDEX_NAME, - body: { + client.indices.create({ index: INDEX_NAME, + body: { settings: settings.to_hash, - mappings: mappings } + mappings: mappings } }.merge(options)) end extend self diff --git a/elasticsearch-persistence/elasticsearch-persistence.gemspec b/elasticsearch-persistence/elasticsearch-persistence.gemspec index f20190052..3eb544f02 100644 --- a/elasticsearch-persistence/elasticsearch-persistence.gemspec +++ b/elasticsearch-persistence/elasticsearch-persistence.gemspec @@ -40,7 +40,7 @@ Gem::Specification.new do |s| s.required_ruby_version = ">= 1.9.3" - s.add_dependency "elasticsearch", '~> 6' + s.add_dependency "elasticsearch", '~> 7' s.add_dependency "elasticsearch-model", '~> 7' s.add_dependency "activesupport", '> 4' s.add_dependency "activemodel", '> 4' diff --git a/elasticsearch-persistence/lib/elasticsearch/persistence/repository.rb b/elasticsearch-persistence/lib/elasticsearch/persistence/repository.rb index e330ffb5c..c10b5c00b 100644 --- a/elasticsearch-persistence/lib/elasticsearch/persistence/repository.rb +++ b/elasticsearch-persistence/lib/elasticsearch/persistence/repository.rb @@ -80,16 +80,6 @@ def create(options = {}, &block) # @since 6.0.0 DEFAULT_INDEX_NAME = 'repository'.freeze - # The default document type. - # - # @return [ String ] The default document type. - # - # @note the document type will no longer be configurable in future versions - # of Elasticsearch. - # - # @since 6.0.0 - DEFAULT_DOC_TYPE = '_doc'.freeze - # The repository options. # # @return [ Hash ] @@ -141,8 +131,7 @@ def client # @since 6.0.0 def document_type @document_type ||= @options[:document_type] || - __get_class_value(:document_type) || - DEFAULT_DOC_TYPE + __get_class_value(:document_type) end # Get the index name used by the repository. diff --git a/elasticsearch-persistence/lib/elasticsearch/persistence/repository/dsl.rb b/elasticsearch-persistence/lib/elasticsearch/persistence/repository/dsl.rb index 7373f9acd..81185cfd4 100644 --- a/elasticsearch-persistence/lib/elasticsearch/persistence/repository/dsl.rb +++ b/elasticsearch-persistence/lib/elasticsearch/persistence/repository/dsl.rb @@ -44,7 +44,7 @@ module ClassMethods # # @since 6.0.0 def document_type(_type = nil) - @document_type ||= (_type || DEFAULT_DOC_TYPE) + @document_type ||= _type end # Get or set the class-level index name setting. diff --git a/elasticsearch-persistence/spec/repository/store_spec.rb b/elasticsearch-persistence/spec/repository/store_spec.rb index bc9b7ba61..569aa410f 100644 --- a/elasticsearch-persistence/spec/repository/store_spec.rb +++ b/elasticsearch-persistence/spec/repository/store_spec.rb @@ -331,7 +331,7 @@ def to_hash context 'when the document does not exist' do before do - repository.create_index! + repository.create_index!(include_type_name: true) end it 'raises an exception' do diff --git a/elasticsearch-persistence/spec/repository_spec.rb b/elasticsearch-persistence/spec/repository_spec.rb index 06d32ef0a..472687fe5 100644 --- a/elasticsearch-persistence/spec/repository_spec.rb +++ b/elasticsearch-persistence/spec/repository_spec.rb @@ -107,9 +107,6 @@ class RepositoryWithoutDSL expect(repository.client).to be_a(Elasticsearch::Transport::Client) end - it 'sets a default document type' do - expect(repository.document_type).to eq('_doc') - end it 'sets a default index name' do expect(repository.index_name).to eq('repository') @@ -282,7 +279,7 @@ class RepositoryWithDSL before do begin; repository.delete_index!; rescue; end - repository.create_index! + repository.create_index!(include_type_name: true) end it 'creates the index' do @@ -337,7 +334,7 @@ class RepositoryWithDSL end before do - repository.create_index! + repository.create_index!(include_type_name: true) end it 'refreshes the index' do @@ -364,7 +361,7 @@ class RepositoryWithDSL end before do - repository.create_index! + repository.create_index!(include_type_name: true) end it 'determines if the index exists' do @@ -501,10 +498,6 @@ class RepositoryWithoutDSL }.to raise_exception(NoMethodError) end - it 'sets a default on the instance' do - expect(RepositoryWithoutDSL.new.document_type).to eq('_doc') - end - it 'allows the value to be overridden with options on the instance' do expect(RepositoryWithoutDSL.new(document_type: 'notes').document_type).to eq('notes') end @@ -547,6 +540,33 @@ class RepositoryWithoutDSL repository.create_index! expect(repository.index_exists?).to eq(true) end + + context 'when the repository has a document type defined' do + + let(:repository) do + RepositoryWithoutDSL.new(client: DEFAULT_CLIENT, document_type: 'mytype') + end + + context 'when the server is version >= 7.0', if: server_version > '7.0' do + + context 'when the include_type_name option is specified' do + + it 'creates an index' do + repository.create_index!(include_type_name: true) + expect(repository.index_exists?).to eq(true) + end + end + + context 'when the include_type_name option is not specified' do + + it 'raises an error' do + expect { + repository.create_index! + }.to raise_exception(Elasticsearch::Transport::Transport::Errors::BadRequest) + end + end + end + end end describe '#delete_index!' do @@ -562,7 +582,7 @@ class RepositoryWithoutDSL end it 'deletes an index' do - repository.create_index! + repository.create_index!(include_type_name: true) repository.delete_index! expect(repository.index_exists?).to eq(false) end @@ -585,7 +605,7 @@ class RepositoryWithoutDSL end it 'refreshes an index' do - repository.create_index! + repository.create_index!(include_type_name: true) expect(repository.refresh_index!['_shards']).to be_a(Hash) end end @@ -607,7 +627,7 @@ class RepositoryWithoutDSL end it 'returns whether the index exists' do - repository.create_index! + repository.create_index!(include_type_name: true) expect(repository.index_exists?).to be(true) end end @@ -621,7 +641,7 @@ class RepositoryWithoutDSL end it 'sets a default on an instance' do - expect(RepositoryWithoutDSL.new.mapping.to_hash).to eq(_doc: { properties: {} }) + expect(RepositoryWithoutDSL.new.mapping.to_hash).to eq(properties: {}) end it 'allows the mapping to be set as an option' do diff --git a/elasticsearch-persistence/spec/spec_helper.rb b/elasticsearch-persistence/spec/spec_helper.rb index 2caf7bcee..73243fe19 100644 --- a/elasticsearch-persistence/spec/spec_helper.rb +++ b/elasticsearch-persistence/spec/spec_helper.rb @@ -48,3 +48,12 @@ class MyTestRepository # # @since 6.0.0 DEFAULT_REPOSITORY = MyTestRepository.new(index_name: 'my_test_repository', document_type: 'test') + +# Get the Elasticsearch server version. +# +# @return [ String ] The version of Elasticsearch. +# +# @since 7.0.0 +def server_version(client = nil) + (client || DEFAULT_CLIENT).info['version']['number'] +end diff --git a/travis_before_script.sh b/travis_before_script.sh index 014864175..547a5a040 100644 --- a/travis_before_script.sh +++ b/travis_before_script.sh @@ -1,6 +1,15 @@ #!/bin/bash -curl https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz | tar xz -C /tmp +if [ "$ELASTICSEARCH_VERSION" == "6.7.1" ] +then + url="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz" +else + url="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz" +fi + +echo "Downloading elasticsearch from $url" +curl $url | tar xz -C /tmp echo "Starting elasticsearch on port ${TEST_CLUSTER_PORT}" +/tmp/elasticsearch-${ELASTICSEARCH_VERSION}/bin/elasticsearch-keystore create /tmp/elasticsearch-${ELASTICSEARCH_VERSION}/bin/elasticsearch -E http.port=${TEST_CLUSTER_PORT} &> /dev/null &