Skip to content

Commit 5f50d13

Browse files
just3wsjust3ws
committed
WIP replacing Tire with ElasticSearch::Model
1 parent 2969400 commit 5f50d13

File tree

7 files changed

+129
-75
lines changed

7 files changed

+129
-75
lines changed

‎Rakefile

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,33 @@ Coderwall::Application.load_tasks
55

66
task default: :spec
77

8+
namespace :search do
9+
task test: :environment do
10+
Team.__elasticsearch__.client.indices.delete index: Team.index_name rescue nil
11+
Team.__elasticsearch__.client.indices.create index: Team.index_name, body: { settings: Team.settings.to_hash, mappings: Team.mappings.to_hash }
12+
Team.import
13+
14+
es = Team.__elasticsearch__.search({
15+
query: {
16+
multi_match: {
17+
query: 'group',
18+
fields: ['name']
19+
}
20+
},
21+
size: 10000
22+
}).to_a
23+
24+
pg = Team.where('name ilike ?', '%group%').to_a
25+
26+
puts "ES has #{es.count - pg.count}"
27+
ap (es.map(&:name).sort) - (pg.map(&:name).sort)
28+
29+
puts "PG has #{pg.count - es.count}"
30+
ap (pg.map(&:name).sort) - (es.map(&:name).sort)
31+
32+
end
33+
end
34+
835
namespace :team do
936
task migrate: :environment do
1037
puts '--- Beginning team migration ---'
@@ -83,17 +110,17 @@ namespace :team do
83110
#mongo_team_locations = mongo_team.team_locations
84111

85112
#if mongo_team_locations.count != pg_team_locations.count
86-
#puts "locations | pg:#{pg_team.id} | mongo:#{mongo_team.id}| #{mongo_team_locations.count} != #{pg_team_locations.count}"
113+
#puts "locations | pg:#{pg_team.id} | mongo:#{mongo_team.id}| #{mongo_team_locations.count} != #{pg_team_locations.count}"
87114
#end
88115

89116
## Ignoring:
90117
## - points_of_interest
91118
#pg_team.locations.each do |pg_team_location|
92-
#mongo_team_location = mongo_team.team_locations.select { |tl| tl.name == pg_team_location.name }.first
119+
#mongo_team_location = mongo_team.team_locations.select { |tl| tl.name == pg_team_location.name }.first
93120

94-
#%i(address city country description name state_code).each do |attr|
95-
#neq(attr, pg_team_location, mongo_team_location, false)
96-
#end
121+
#%i(address city country description name state_code).each do |attr|
122+
#neq(attr, pg_team_location, mongo_team_location, false)
123+
#end
97124
#end
98125

99126

@@ -124,9 +151,9 @@ namespace :team do
124151
#puts 'JOBS'
125152

126153
#pg_team.jobs.each do |pg_team_job|
127-
#mongo_team_job = Team.where(id: pg_team_job.team_document_id.to_s).first
154+
#mongo_team_job = Team.where(id: pg_team_job.team_document_id.to_s).first
128155

129-
#neq(:name, pg_team_job, mongo_team_job, false)
156+
#neq(:name, pg_team_job, mongo_team_job, false)
130157
#end
131158

132159
#puts 'FOLLOWERS'

‎app/models/concerns/team_search.rb

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
module TeamSearch
22
extend ActiveSupport::Concern
3+
34
included do
4-
include Elasticsearch::Model
5+
56
end
67
end

‎app/models/team.rb

Lines changed: 58 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,21 @@
1-
# encoding utf-8
2-
require 'search'
1+
# encodinsnowballg utf-8
2+
require 'elasticsearch/model'
33

44
class Team < ActiveRecord::Base
5+
include SearchModule
6+
include LeaderboardRedisRank
7+
include TeamAnalytics
8+
include TeamMigration
9+
510
DEFAULT_HEX_BRAND = '#343131'
611
LEADERBOARD_KEY = 'teams:leaderboard'
712
FEATURED_TEAMS_CACHE_KEY = 'featured_teams_results'
813
MAX_TEAM_SCORE = 400
914

10-
self.table_name = 'teams'
11-
12-
include SearchModule
1315
include TeamSearch
14-
include LeaderboardRedisRank
15-
include TeamAnalytics
16-
include TeamMigration
1716

1817
mount_uploader :avatar, TeamUploader
1918

20-
mapping team: {
21-
properties: {
22-
id: { type: 'string', index: 'not_analyzed' },
23-
slug: { type: 'string', index: 'not_analyzed' },
24-
name: { type: 'string', boost: 100, analyzer: 'snowball' },
25-
score: { type: 'float', index: 'not_analyzed' },
26-
size: { type: 'integer', index: 'not_analyzed' },
27-
avatar: { type: 'string', index: 'not_analyzed' },
28-
country: { type: 'string', boost: 50, analyzer: 'snowball' },
29-
url: { type: 'string', index: 'not_analyzed' },
30-
follow_path: { type: 'string', index: 'not_analyzed' },
31-
hiring: { type: 'boolean', index: 'not_analyzed' },
32-
total_member_count: { type: 'integer', index: 'not_analyzed' },
33-
completed_sections: { type: 'integer', index: 'not_analyzed' },
34-
team_members: { type: 'multi_field', fields: {
35-
username: { type: 'string', index: 'not_analyzed' },
36-
profile_url: { type: 'string', index: 'not_analyzed' },
37-
avatar: { type: 'string', index: 'not_analyzed' }
38-
} }
39-
}
40-
}
41-
4219
scope :featured, ->{ where(premium: true, valid_jobs: true, hide_from_featured: false) }
4320

4421

@@ -137,7 +114,7 @@ def self.completed_at_least(section_count = 6, page=1, per_page=Team.count, sear
137114

138115
def self.with_similar_names(name)
139116
name.gsub!(/ \-\./, '.*')
140-
teams = Team.any_of({ :name => /#{name}.*/i }).limit(3).to_a
117+
Team.where('name ilike ?', "#{name}%").limit(3).to_a
141118
end
142119

143120
def self.with_completed_section(section)
@@ -727,7 +704,7 @@ def coderwall?
727704
end
728705

729706
def reindex_search
730-
if Rails.env.development? or Rails.env.test? or self.destroyed?
707+
if %w(development test).include?(Rails.env) || self.destroyed?
731708
self.tire.update_index
732709
else
733710
IndexTeamJob.perform_async(id)
@@ -827,6 +804,54 @@ def deny_join_request(user)
827804
self.pending_join_requests.delete user.id
828805
end
829806

807+
808+
include Elasticsearch::Model
809+
include Elasticsearch::Model::Callbacks
810+
811+
#mapping team: {
812+
#properties: {
813+
#id: { type: 'string', index: 'not_analyzed' },
814+
#slug: { type: 'string', index: 'not_analyzed' },
815+
#name: { type: 'string', boost: 100, analyzer: 'snowball' },
816+
#score: { type: 'float', index: 'not_analyzed' },
817+
#size: { type: 'integer', index: 'not_analyzed' },
818+
#avatar: { type: 'string', index: 'not_analyzed' },
819+
#country: { type: 'string', boost: 50, analyzer: 'snowball' },
820+
#url: { type: 'string', index: 'not_analyzed' },
821+
#follow_path: { type: 'string', index: 'not_analyzed' },
822+
#hiring: { type: 'boolean', index: 'not_analyzed' },
823+
#total_member_count: { type: 'integer', index: 'not_analyzed' },
824+
#completed_sections: { type: 'integer', index: 'not_analyzed' },
825+
#team_members: { type: 'multi_field', fields: {
826+
#username: { type: 'string', index: 'not_analyzed' },
827+
#profile_url: { type: 'string', index: 'not_analyzed' },
828+
#avatar: { type: 'string', index: 'not_analyzed' }
829+
#} }
830+
#}
831+
#}
832+
833+
settings do
834+
mappings dynamic: 'false' do
835+
indexes :id, type: 'string', index: 'not_analyzed'
836+
indexes :slug, type: 'string', index: 'not_analyzed'
837+
indexes :name, type: 'string', boost: 100, analyzer: 'snowball'
838+
indexes :core, type: 'float', analyzer: 'snowball'
839+
indexes :size, type: 'integer', analyzer: 'snowball'
840+
indexes :avatar, type: 'string', analyzer: 'snowball'
841+
indexes :country, type: 'string', boost: 50, analyzer: 'snowball'
842+
indexes :url, type: 'string', analyzer: 'snowball'
843+
indexes :follow_path, type: 'string', analyzer: 'snowball'
844+
indexes :hiring, type: 'boolean', analyzer: 'snowball'
845+
indexes :total_member_count, type: 'integer', analyzer: 'snowball'
846+
indexes :completed_sections, type: 'integer', analyzer: 'snowball'
847+
#indexes :team_members do #, type: 'multi_field', fields:
848+
#indexes :username, type: 'string', analyzer: 'snowball'
849+
#indexes :profile_url, type: 'string', analyzer: 'snowball'
850+
#indexes :avatar, type: 'string', analyzer: 'snowball'
851+
#end
852+
end
853+
end
854+
830855
private
831856

832857
def identify_visitor(visitor_name)
@@ -858,6 +883,7 @@ def update_team_size!
858883
end
859884

860885
def clear_cache_if_premium_team
886+
Team
861887
Rails.cache.delete(Team::FEATURED_TEAMS_CACHE_KEY) if premium?
862888
end
863889

‎config/environments/development.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,6 @@
3434
Rails.application.config.sass.cache_location = "/tmp/codewall-cache/sass/"
3535

3636
BetterErrors::Middleware.allow_ip! ENV['TRUSTED_IP'] if ENV['TRUSTED_IP']
37-
Rails.logger = Logger.new(STDOUT)
38-
Rails.logger.level = Logger::DEBUG
37+
#Rails.logger = Logger.new(STDOUT)
38+
#Rails.logger.level = Logger::DEBUG
3939
end

‎config/environments/test.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@
1515
config.assets.allow_debugging = true
1616
config.middleware.use RackSessionAccess::Middleware # allows to set session from within Capybara
1717

18-
Rails.logger = Logger.new(STDOUT) # provides more verbose output when testing with headless browsers in case of errors
19-
Rails.logger.level = Logger::DEBUG
18+
#Rails.logger = Logger.new(STDOUT) # provides more verbose output when testing with headless browsers in case of errors
19+
#Rails.logger.level = Logger::DEBUG
2020
end

‎config/initializers/elasticsearch.rb

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,35 @@
1+
require 'elasticsearch/rails/instrumentation'
2+
# require 'elasticsearch/rails/lograge'
3+
14
Tire.configure do
25
url ENV['ELASTICSEARCH_URL']
36
end
47

5-
Elasticsearch::Model.client = Elasticsearch::Client.new url: ENV['ELASTICSEARCH_URL']
8+
Elasticsearch::Model.client = Elasticsearch::Client.new url: ENV['ELASTICSEARCH_URL'], log: true
9+
10+
#if ENV['ELASTICSEARCH_URL']
11+
#Elasticsearch::Model.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'], log: false
12+
#end
13+
14+
#Kaminari::Hooks.init
15+
#Elasticsearch::Model::Response::Response.__send__ :include, Elasticsearch::Model::Response::Pagination::Kaminari
16+
17+
NGRAM_ANALYZER = {
18+
analysis: {
19+
filter: {
20+
ngram_filter: {
21+
type: 'nGram',
22+
min_gram: 4,
23+
max_gram: 15
24+
}
25+
},
26+
analyzer: {
27+
ngram_analyzer: {
28+
tokenizer: 'lowercase',
29+
filter: ['ngram_filter'],
30+
type: 'custom'
31+
}
32+
}
33+
}
34+
}
35+

‎deploy

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)