Merge pull request #1202 from metacpan/oalders/cloud-config

Move Elasticsearch server setup to config files
metacpan · Sep 17, 2024 · 9d64476 · 9d64476
2 parents fe5bdc4 + 3423628
commit 9d64476
Show file tree

Hide file tree

Showing 26 changed files with 333 additions and 407 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,6 +21,7 @@ jobs:
           command: |
             git clone https://github.com/metacpan/metacpan-docker.git
             cd metacpan-docker
+            git switch oalders/cloud-config
           name: metacpan-docker checkout
       - checkout:
           path: metacpan-docker/src/metacpan-api

diff --git a/etc/metacpan.pl b/etc/metacpan.pl
diff --git a/etc/metacpan_testing.pl b/etc/metacpan_testing.pl
diff --git a/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm b/lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm
@@ -6,12 +6,15 @@ use Moose;
 extends 'Catalyst::Plugin::Session::Store';
 use MooseX::Types::ElasticSearch qw( ES );
 
+use MetaCPAN::Server::Config ();
+
 has _session_es => (
-    required => 1,
-    is       => 'ro',
-    coerce   => 1,
-    isa      => ES,
-    default  => sub { shift->_session_plugin_config->{servers} || ':9200' }
+    is      => 'ro',
+    lazy    => 1,
+    coerce  => 1,
+    isa     => ES,
+    default =>
+        sub { MetaCPAN::Server::Config::config()->{elasticsearch_servers} },
 );
 has _session_es_index => (
     required => 1,

diff --git a/lib/MetaCPAN/Role/HasConfig.pm b/lib/MetaCPAN/Role/HasConfig.pm
@@ -2,8 +2,7 @@ package MetaCPAN::Role::HasConfig;
 
 use Moose::Role;
 
-use Config::ZOMG              ();
-use FindBin                   ();
+use MetaCPAN::Server::Config  ();
 use MetaCPAN::Types::TypeTiny qw( HashRef );
 use MetaCPAN::Util            qw( checkout_root );
 
@@ -20,29 +19,8 @@ has _config => (
 );
 
 sub _build_config {
-    my $self   = shift;
-    my $config = $self->_zomg("$FindBin::RealBin/..");
-    return $config if $config;
-
-    $config = $self->_zomg( checkout_root() );
-
-    return $config if $config;
-
-    die "Couldn't find config file in $FindBin::RealBin/.. or "
-        . checkout_root();
-}
-
-sub _zomg {
     my $self = shift;
-    my $path = shift;
-
-    my $config = Config::ZOMG->new(
-        local_suffix => $ENV{HARNESS_ACTIVE} ? 'testing' : 'local',
-        name         => 'metacpan_server',
-        path         => $path,
-    );
-
-    return $config->open;
+    return MetaCPAN::Server::Config::config();
 }
 
 1;
diff --git a/lib/MetaCPAN/Role/Logger.pm b/lib/MetaCPAN/Role/Logger.pm
@@ -2,6 +2,7 @@ package MetaCPAN::Role::Logger;
 
 use v5.10;
 use Moose::Role;
+
 use Log::Contextual qw( set_logger );
 use Log::Log4perl ':easy';
 use MetaCPAN::Types::TypeTiny qw( Logger Str );
@@ -45,8 +46,7 @@ sub set_logger_once {
     return;
 }
 
-# XXX NOT A MOOSE BUILDER
-# XXX This doesn't belong here.
+# Not actually a Moose builder, so we should probably rename it.
 sub _build_logger {
     my ($config) = @_;
     my $log = Log::Log4perl->get_logger( $ARGV[0]

diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm
@@ -2,6 +2,7 @@ package MetaCPAN::Role::Script;
 
 use Moose::Role;
 
+use Carp                                   ();
 use ElasticSearchX::Model::Document::Types qw( ES );
 use File::Path                             ();
 use IO::Prompt::Tiny                       qw( prompt );
@@ -12,8 +13,6 @@ use MetaCPAN::Util                         qw( checkout_root );
 use Mojo::Server                           ();
 use Term::ANSIColor                        qw( colored );
 
-use Carp ();
-
 with( 'MetaCPAN::Role::HasConfig', 'MetaCPAN::Role::Fastly',
     'MetaCPAN::Role::Logger' );
 
@@ -74,21 +73,24 @@ has es => (
     is            => 'ro',
     isa           => ES,
     required      => 1,
+    init_arg      => 'elasticsearch_servers',
     coerce        => 1,
     documentation => 'Elasticsearch http connection string',
 );
 
 has model => (
-    is      => 'ro',
-    lazy    => 1,
-    builder => '_build_model',
-    traits  => ['NoGetopt'],
+    is       => 'ro',
+    init_arg => undef,
+    lazy     => 1,
+    builder  => '_build_model',
+    traits   => ['NoGetopt'],
 );
 
 has index => (
     reader        => '_index',
     is            => 'ro',
     isa           => Str,
+    lazy          => 1,
     default       => 'cpan',
     documentation =>
         'Index to use, defaults to "cpan" (when used: also export ES_SCRIPT_INDEX)',
@@ -98,13 +100,15 @@ has cluster_info => (
     isa     => HashRef,
     traits  => ['Hash'],
     is      => 'rw',
+    lazy    => 1,
     default => sub { {} },
 );
 
 has indices_info => (
     isa     => HashRef,
     traits  => ['Hash'],
     is      => 'rw',
+    lazy    => 1,
     default => sub { {} },
 );
 
@@ -118,7 +122,9 @@ has aliases_info => (
 has port => (
     isa           => Int,
     is            => 'ro',
-    required      => 1,
+    required      => 0,
+    lazy          => 1,
+    default       => sub {5000},
     documentation => 'Port for the proxy, defaults to 5000',
 );
 
@@ -156,11 +162,11 @@ sub BUILDARGS {
     my ( $self, @args ) = @_;
     my %args = @args == 1 ? %{ $args[0] } : @args;
 
-    if ( exists $args{'index'} ) {
+    if ( exists $args{index} ) {
         die
             "when setting --index, please export ES_SCRIPT_INDEX to the same value\n"
-            unless $ENV{'ES_SCRIPT_INDEX'}
-            and $args{'index'} eq $ENV{'ES_SCRIPT_INDEX'};
+            unless $ENV{ES_SCRIPT_INDEX}
+            and $args{index} eq $ENV{ES_SCRIPT_INDEX};
     }
 
     return \%args;
@@ -183,11 +189,7 @@ sub handle_error {
 sub print_error {
     my ( $self, $error ) = @_;
 
-    # Always log.
     log_error {$error};
-
-    # Display Error in red
-    print colored( ['bold red'], "*** ERROR ***: $error" ), "\n";
 }
 
 sub index {
@@ -265,10 +267,7 @@ sub remote {
 sub run { }
 before run => sub {
     my $self = shift;
-
     $self->set_logger_once;
-
-    #Dlog_debug {"Connected to $_"} $self->remote;
 };
 
 sub _get_indices_info {
@@ -410,7 +409,7 @@ sub are_you_sure {
         }
     }
     else {
-        print colored( ['bold yellow'], "*** Warning ***: $msg" ) . "\n";
+        log_info {"*** Warning ***: $msg"};
         $iconfirmed = 1;
     }
 

diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm
@@ -254,8 +254,6 @@ sub delete_all {
         || $runtime_environment eq 'testing';
 
     if ($is_development) {
-        $self->are_you_sure("ALL Indices will be deleted !!!");
-
         foreach my $name ( keys %{ $self->indices_info } ) {
             $self->_delete_index($name);
         }
@@ -489,7 +487,7 @@ sub show_info {
         'indices_info' => \%{ $self->indices_info },
         'aliases_info' => \%{ $self->aliases_info }
     };
-    print JSON->new->utf8->pretty->encode($info_rs);
+    log_info { JSON->new->utf8->pretty->encode($info_rs) };
 }
 
 sub _build_mapping {
@@ -538,7 +536,6 @@ sub _build_mapping {
 sub _build_aliases {
     my $self = $_[0];
     return { 'cpan' => $self->cpan_index };
-
 }
 
 sub deploy_mapping {

diff --git a/lib/MetaCPAN/Script/Runner.pm b/lib/MetaCPAN/Script/Runner.pm
@@ -3,10 +3,9 @@ package MetaCPAN::Script::Runner;
 use strict;
 use warnings;
 
-use Config::ZOMG        ();
-use File::Path          ();
-use Hash::Merge::Simple qw( merge );
-use Module::Pluggable search_path => ['MetaCPAN::Script'];
+use File::Path               ();
+use MetaCPAN::Server::Config ();
+use Module::Pluggable search_path => ['MetaCPAN::Script'];    # plugins()
 use Module::Runtime ();
 use Term::ANSIColor qw( colored );
 use Try::Tiny       qw( catch try );
@@ -21,7 +20,7 @@ sub run {
     die "Usage: metacpan [command] [args]" unless ($class);
     Module::Runtime::require_module( $plugins{$class} );
 
-    my $config = build_config();
+    my $config = MetaCPAN::Server::Config::config();
 
     foreach my $logger ( @{ $config->{logger} || [] } ) {
         my $path = $logger->{filename} or next;
@@ -61,9 +60,11 @@ sub run {
         }
 
         # Display Exception Message in red
-        print colored( ['bold red'],
-            "*** EXCEPTION [ $EXIT_CODE ] ***: " . $ex->{'message'} ),
-            "\n";
+        unless ( $ENV{HARNESS_ACTIVE} ) {
+            print colored( ['bold red'],
+                "*** exception [ $EXIT_CODE ] ***: " . $ex->{'message'} ),
+                "\n";
+        }
     };
 
     unless ( defined $ex ) {
@@ -75,21 +76,6 @@ sub run {
     return ( $EXIT_CODE == 0 );
 }
 
-sub build_config {
-    my $config = Config::ZOMG->new(
-        name => 'metacpan',
-        path => 'etc'
-    )->load;
-    if ( $ENV{HARNESS_ACTIVE} ) {
-        my $tconf = Config::ZOMG->new(
-            name => 'metacpan',
-            file => 'etc/metacpan_testing.pl'
-        )->load;
-        $config = merge $config, $tconf;
-    }
-    return $config;
-}
-
 # AnyEvent::Run calls the main method
 *main = \&run;
 

diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm
@@ -7,7 +7,8 @@ use Cpanel::JSON::XS          qw( decode_json encode_json );
 use DateTime                  ();
 use DateTime::Format::ISO8601 ();
 use HTTP::Tiny                ();
-use Log::Contextual           qw( :log :dlog );
+use Log::Contextual           qw( :log );
+use MetaCPAN::Server::Config  ();
 use MetaCPAN::Types::TypeTiny qw( ArrayRef Bool Str );
 use Moose;
 use Sys::Hostname qw( hostname );
@@ -79,9 +80,10 @@ has snap_name => (
 );
 
 has host => (
-    is            => 'ro',
-    isa           => Str,
-    default       => 'http://localhost:9200',
+    is      => 'ro',
+    isa     => Str,
+    default =>
+        sub { MetaCPAN::Server::Config::config()->{elasticsearch_servers} },
     documentation => 'ES host, defaults to: http://localhost:9200',
 );