Skip to content

Commit

Permalink
limit all pagination to 10000 results
Browse files Browse the repository at this point in the history
Paged results are expensive to compute on Elasticsearch. It will refuse
to give results if paged forward far enough.

Limit our pagination and give empty results if paged too deep.
  • Loading branch information
haarg committed Nov 5, 2024
1 parent fbe6f1a commit 4814f61
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 9 deletions.
10 changes: 9 additions & 1 deletion lib/MetaCPAN/Query/Favorite.pm
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package MetaCPAN::Query::Favorite;
use MetaCPAN::Moose;

use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Util qw(hit_total);
use MetaCPAN::Util qw( MAX_RESULT_WINDOW hit_total );

with 'MetaCPAN::Query::Role::Common';

Expand Down Expand Up @@ -148,6 +148,14 @@ sub recent {
$page //= 1;
$size //= 100;

if ( $page * $size >= MAX_RESULT_WINDOW ) {
return +{
favorites => [],
took => 0,
total => 0,
};
}

my $favs = $self->es->search(
es_doc_path('favorite'),
body => {
Expand Down
35 changes: 28 additions & 7 deletions lib/MetaCPAN/Query/Release.pm
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use MetaCPAN::Moose;

use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Util
qw( hit_total single_valued_arrayref_to_scalar true false );
qw( MAX_RESULT_WINDOW hit_total single_valued_arrayref_to_scalar true false );

with 'MetaCPAN::Query::Role::Common';

Expand Down Expand Up @@ -399,6 +399,14 @@ sub by_author {
$size //= 1000;
$page //= 1;

if ( $page * $size >= MAX_RESULT_WINDOW ) {
return {
releases => [],
took => 0,
total => 0,
};
}

my $body = {
query => {
bool => {
Expand Down Expand Up @@ -491,6 +499,14 @@ sub all_by_author {
$size //= 100;
$page //= 1;

if ( $page * $size >= MAX_RESULT_WINDOW ) {
return {
releases => [],
took => 0,
total => 0,
};
}

my $body = {
query => { term => { author => uc($author) } },
sort => [ { date => 'desc' } ],
Expand Down Expand Up @@ -713,6 +729,14 @@ sub _get_depended_releases {
$page //= 1;
$page_size //= 50;

if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
return +{
data => [],
took => 0,
total => 0,
};
}

$sort = _fix_sort_value($sort);

my $dependency_filter = {
Expand Down Expand Up @@ -773,14 +797,11 @@ sub recent {
$page_size //= 10000;
$type //= '';

my $query;
my $from = ( $page - 1 ) * $page_size;

if ( $from + $page_size > 10000 ) {
return {
if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
return +{
releases => [],
total => 0,
took => 0,
total => 0,
};
}

Expand Down
11 changes: 10 additions & 1 deletion lib/MetaCPAN/Query/Search.pm
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use List::Util qw( min uniq );
use Log::Contextual qw( :log :dlog );
use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Types::TypeTiny qw( Object Str );
use MetaCPAN::Util qw( hit_total true false );
use MetaCPAN::Util qw( MAX_RESULT_WINDOW hit_total true false );
use MooseX::StrictConstructor;

with 'MetaCPAN::Query::Role::Common';
Expand Down Expand Up @@ -60,6 +60,15 @@ sub search_web {
$page_size //= 20;
$page //= 1;

if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
return {
results => [],
total => 0,
tool => 0,
colapsed => $collapsed ? true : false,
};
}

$search_term =~ s{([+=><!&|\(\)\{\}[\]\^"~*?\\/])}{\\$1}g;

# munge the search_term
Expand Down
3 changes: 3 additions & 0 deletions lib/MetaCPAN/Util.pm
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,12 @@ use Sub::Exporter -setup => {
true
false
is_bool
MAX_RESULT_WINDOW
) ]
};

use constant MAX_RESULT_WINDOW => 10000;

*true = \&Cpanel::JSON::XS::true;
*false = \&Cpanel::JSON::XS::false;
*is_bool = \&Cpanel::JSON::XS::is_bool;
Expand Down

0 comments on commit 4814f61

Please sign in to comment.