Skip to content

Commit

Permalink
remove use of search_type scan
Browse files Browse the repository at this point in the history 
search_type: scan has been removed from newer Elasticsearch versions,
replaced with sort: _doc.
  • Loading branch information
@haarg
haarg committed Sep 24, 2024
1 parent 23af37b commit 41d4fa9
Show file tree
Hide file tree
Showing 9 changed files with 112 additions and 100 deletions.
10 changes: 5 additions & 5 deletions lib/MetaCPAN/Script/Author.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,10 @@ sub index_authors {
);

my $scroll = $self->es->scroll_helper(
index => $self->index->name,
type => 'author',
search_type => 'scan',
size => 500,
body => {
index => $self->index->name,
type => 'author',
size => 500,
body => {
query => {
$self->pauseid
? (
Expand All @@ -150,6 +149,7 @@ sub index_authors {
: ( match_all => {} ),
},
_source => [@compare_fields],
sort => '_doc',
},
);

Expand Down
10 changes: 6 additions & 4 deletions lib/MetaCPAN/Script/Backup.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,12 @@ sub run {
my $scroll = $es->scroll_helper(
index => $self->index->name,
$self->type ? ( type => $self->type ) : (),
size => $self->size,
search_type => 'scan',
fields => [qw(_parent _source)],
scroll => '1m',
size => $self->size,
fields => [qw(_parent _source)],
scroll => '1m',
body => {
sort => '_doc',
},
);

log_info { 'Backing up ', $scroll->total, ' documents' };
Expand Down
10 changes: 6 additions & 4 deletions lib/MetaCPAN/Script/CPANTesters.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,12 @@ sub index_reports {
bunzip2 "$db.bz2" => "$db", AutoClose => 1 if -e "$db.bz2";

my $scroll = $es->scroll_helper(
index => $self->index->name,
search_type => 'scan',
size => '500',
type => 'release',
index => $self->index->name,
size => '500',
type => 'release',
body => {
sort => '_doc',
},
);

my %releases;
Expand Down
10 changes: 6 additions & 4 deletions lib/MetaCPAN/Script/CPANTestersAPI.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,12 @@ sub index_reports {
my $data = decode_json $json;

my $scroll = $es->scroll_helper(
index => $self->index->name,
search_type => 'scan',
size => '500',
type => 'release',
index => $self->index->name,
size => '500',
type => 'release',
body => {
sort => '_doc',
},
);

# Create a cache of all releases (dist + version combos)
Expand Down
78 changes: 36 additions & 42 deletions lib/MetaCPAN/Script/Favorite.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ sub run {
sub index_favorites {
my $self = shift;

my $body;
my $query = { match_all => {} };
my $age_filter;
if ( $self->age ) {
$age_filter = {
Expand All @@ -86,24 +86,20 @@ sub index_favorites {
}

if ( $self->distribution ) {
$body = {
query => {
term => { distribution => $self->distribution }
}
};
$query = { term => { distribution => $self->distribution } };

}
elsif ( $self->age ) {
my $favs = $self->es->scroll_helper(
index => $self->index->name,
type => 'favorite',
search_type => 'scan',
scroll => '5m',
fields => [qw< distribution >],
size => 500,
body => {
index => $self->index->name,
type => 'favorite',
scroll => '5m',
fields => [qw< distribution >],
size => 500,
body => {
query => $age_filter,
( $self->limit ? ( size => $self->limit ) : () )
( $self->limit ? ( size => $self->limit ) : () ),
sort => '_doc',
}
);

Expand All @@ -116,11 +112,7 @@ sub index_favorites {

my @keys = keys %recent_dists;
if (@keys) {
$body = {
query => {
terms => { distribution => \@keys }
}
};
$query = { terms => { distribution => \@keys } };
}
}

Expand All @@ -133,13 +125,15 @@ sub index_favorites {
}
else {
my $favs = $self->es->scroll_helper(
index => $self->index->name,
type => 'favorite',
search_type => 'scan',
scroll => '30s',
fields => [qw< distribution >],
size => 500,
( $body ? ( body => $body ) : () ),
index => $self->index->name,
type => 'favorite',
scroll => '30s',
fields => [qw< distribution >],
size => 500,
body => {
query => $query,
sort => '_doc',
},
);

while ( my $fav = $favs->next ) {
Expand All @@ -160,21 +154,21 @@ sub index_favorites {
}

my $files = $self->es->scroll_helper(
index => $self->index->name,
type => 'file',
search_type => 'scan',
scroll => '15m',
fields => [qw< id distribution >],
size => 500,
body => {
index => $self->index->name,
type => 'file',
scroll => '15m',
fields => [qw< id distribution >],
size => 500,
body => {
query => {
bool => {
must_not => [
{ range => { dist_fav_count => { gte => 1 } } }
],
@age_filter,
}
}
},
sort => '_doc',
},
);

Expand Down Expand Up @@ -236,14 +230,14 @@ sub index_favorites {
);

my $files = $self->es->scroll_helper(
index => $self->index->name,
type => 'file',
search_type => 'scan',
scroll => '15s',
fields => [qw< id >],
size => 500,
body => {
query => { term => { distribution => $dist } }
index => $self->index->name,
type => 'file',
scroll => '15s',
fields => [qw< id >],
size => 500,
body => {
query => { term => { distribution => $dist } },
sort => '_doc',
},
);

Expand Down
25 changes: 17 additions & 8 deletions lib/MetaCPAN/Script/Latest.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,14 +265,23 @@ sub reindex {
$release->put unless ( $self->dry_run );

# Get all the files for the release.
my $scroll = $self->index->type("file")->search_type('scan')->filter( {
bool => {
must => [
{ term => { 'release' => $source->{release} } },
{ term => { 'author' => $source->{author} } }
]
}
} )->size(100)->source( [ 'status', 'file' ] )->raw->scroll;
my $scroll = $self->es->scroll_helper(
index => $self->index->name,
type => 'file',
size => 100,
body => {
query => {
bool => {
must => [
{ term => { 'release' => $source->{release} } },
{ term => { 'author' => $source->{author} } },
],
},
},
_source => [ 'status', 'file' ],
sort => '_doc',
},
);

while ( my $row = $scroll->next ) {
my $source = $row->{_source};
Expand Down
28 changes: 15 additions & 13 deletions lib/MetaCPAN/Script/Mapping.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,17 +411,17 @@ sub _copy_slice {
my ( $self, $query, $index, $type ) = @_;

my $scroll = $self->es->scroll_helper(
search_type => 'scan',
size => 250,
scroll => '10m',
index => $self->index->name,
type => $type,
body => {
size => 250,
scroll => '10m',
index => $self->index->name,
type => $type,
body => {
query => {
filtered => {
query => $query
}
}
},
sort => '_doc',
},
);

Expand Down Expand Up @@ -453,12 +453,14 @@ sub empty_type {
);

my $scroll = $self->es->scroll_helper(
search_type => 'scan',
size => 250,
scroll => '10m',
index => $self->index->name,
type => $type,
body => { query => { match_all => {} } },
size => 250,
scroll => '10m',
index => $self->index->name,
type => $type,
body => {
query => { match_all => {} },
sort => '_doc',
},
);

my @ids;
Expand Down
18 changes: 9 additions & 9 deletions lib/MetaCPAN/Script/Suggest.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,20 +63,20 @@ sub _update_slice {
my ( $self, $range ) = @_;

my $files = $self->es->scroll_helper(
index => $self->index->name,
type => 'file',
search_type => 'scan',
scroll => '5m',
fields => [qw< id documentation >],
size => 500,
body => {
index => $self->index->name,
type => 'file',
scroll => '5m',
fields => [qw< id documentation >],
size => 500,
body => {
query => {
bool => {
must => [
{ exists => { field => "documentation" } }, $range
],
}
}
},
},
sort => '_doc',
},
);

Expand Down
23 changes: 12 additions & 11 deletions lib/MetaCPAN/Script/Watcher.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ sub backpan_changes {
filter => { term => { status => 'backpan' } }
}
},
}
}
},
},
sort => '_doc',
}
} );
my @changes;
Expand Down Expand Up @@ -183,13 +184,12 @@ sub reindex_release {

my $es = $self->es;
my $scroll = $es->scroll_helper( {
index => $self->index->name,
type => 'file',
scroll => '1m',
size => 1000,
search_type => 'scan',
fields => [ '_parent', '_source' ],
body => {
index => $self->index->name,
type => 'file',
scroll => '1m',
size => 1000,
fields => [ '_parent', '_source' ],
body => {
query => {
filtered => {
query => { match_all => {} },
Expand All @@ -208,8 +208,9 @@ sub reindex_release {
]
}
}
}
}
},
sort => '_doc',
},
} );
return if ( $self->dry_run );

Expand Down

0 comments on commit 41d4fa9

Please sign in to comment.