Merge pull request #1245 from travis-ci/logs_changes_am

Disable job logs PRD
travis-ci · Oct 5, 2022 · 9db43ae · 9db43ae
2 parents f5691fc + b3ad0fe
commit 9db43ae
Show file tree

Hide file tree

Showing 22 changed files with 529 additions and 119 deletions.
diff --git a/lib/travis.rb b/lib/travis.rb
@@ -31,6 +31,8 @@ class GithubApiError    < StandardError; end
   class AdminMissing      < StandardError; end
   class RepositoryMissing < StandardError; end
   class LogAlreadyRemoved < StandardError; end
+  class LogExpired        < StandardError; end
+  class LogAccessDenied   < StandardError; end
   class AuthorizationDenied < StandardError; end
   class JobUnfinished     < StandardError; end
 

diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb
@@ -93,16 +93,7 @@ class Jobs < Endpoint
           # the way we use responders makes it hard to validate proper format
           # automatically here, so we need to check it explicitly
           if accepts?('text/plain')
-            archived_log_path = resource.archived_url
-
-            if params[:cors_hax]
-              status 204
-              headers['Access-Control-Expose-Headers'] = 'Location'
-              headers['Location'] = archived_log_path
-              attach_log_token if job.try(:private?)
-            else
-              redirect archived_log_path, 307
-            end
+            respond_with resource.archived_log_content
           elsif accepts?('application/json')
             attach_log_token if job.try(:private?)
             respond_with resource.as_json

diff --git a/lib/travis/api/app/endpoint/logs.rb b/lib/travis/api/app/endpoint/logs.rb
@@ -9,6 +9,14 @@ class Logs < Endpoint
         resource = service(:find_log, id: params[:id]).run
         job = resource ? Job.find(resource.job_id) : nil
 
+        halt 404 unless job
+
+        repo = Travis::API::V3::Models::Repository.find(job.repository.id)
+        repo_can_write = current_user ? !!repo.users.where(id: current_user.id, permissions: { push: true }).first : false
+
+        raise LogExpired if repo.user_settings.job_log_time_based_limit && job.started_at && job.started_at < Time.now - repo.user_settings.job_log_access_older_than_days.days
+        raise LogAccessDenied if repo.user_settings.job_log_access_based_limit && !repo_can_write
+
         if !resource || ((job.try(:private?) || !allow_public?) && !has_permission?(job))
           halt 404
         elsif resource.removed_at && accepts?('application/json')
@@ -17,7 +25,7 @@ class Logs < Endpoint
           # the way we use responders makes it hard to validate proper format
           # automatically here, so we need to check it explicitly
           if accepts?('text/plain')
-            redirect resource.archived_url, 307
+            respond_with resource.archived_log_content
           elsif accepts?('application/json')
             respond_with resource.as_json
           else

diff --git a/lib/travis/api/v3.rb b/lib/travis/api/v3.rb
@@ -39,6 +39,8 @@ def location(env)
       JobNotCancelable    = ClientError        .create('job is not running, cannot cancel', status: 409)
       JobUnfinished       = ClientError        .create('job still running, cannot remove log yet', status: 409)
       LogAlreadyRemoved   = ClientError        .create('log has already been removed', status: 409)
+      LogExpired          = ClientError        .create("We're sorry, but this data is not available anymore. Please check the repository settings in Travis CI.", status: 403)
+      LogAccessDenied     = ClientError        .create("We're sorry, but this data is not available. Please check the repository settings in Travis CI.", status: 403)
       LoginRequired       = ClientError        .create('login required', status: 403)
       MethodNotAllowed    = ClientError        .create('method not allowed', status: 405)
       NotImplemented      = ServerError        .create('request not (yet) implemented', status: 501)

diff --git a/lib/travis/api/v3/models/audit.rb b/lib/travis/api/v3/models/audit.rb
@@ -0,0 +1,6 @@
+module Travis::API::V3
+  class Models::Audit < Model
+    belongs_to :owner, polymorphic: true
+    belongs_to :source, polymorphic: true
+  end
+end
diff --git a/lib/travis/api/v3/models/json_slice.rb b/lib/travis/api/v3/models/json_slice.rb
@@ -2,7 +2,9 @@
 
 module Travis::API::V3
   class Models::JsonSlice
-    include Virtus.model, Enumerable, Models::JsonSync, ActiveModel::Validations
+    include Virtus.model, Enumerable, Models::JsonSync, ActiveModel::Validations, ActiveSupport::Callbacks, ActiveModel::Dirty
+    extend ActiveSupport::Concern
+    define_callbacks :after_save
 
     class << self
       attr_accessor :child_klass
@@ -30,14 +32,21 @@ def read(name)
 
     def update(name, value)
       raise NotFound unless respond_to?(:"#{name}=")
+      @changes = { :"#{name}" => { before: send(name), after: value } } unless value == send(name)
       send(:"#{name}=", value)
       raise UnprocessableEntity, errors.full_messages.to_sentence unless valid?
       sync!
+      run_callbacks :after_save
+      @changes = {}
       read(name)
     end
 
     def to_h
       Hash[map { |x| [x.name, x.value] }]
     end
+
+    def changes
+      @changes
+    end
   end
 end
diff --git a/lib/travis/api/v3/models/user_settings.rb b/lib/travis/api/v3/models/user_settings.rb
@@ -12,9 +12,20 @@ class Models::UserSettings < Models::JsonSlice
     attribute :config_validation, Boolean, default: lambda { |us, _| us.config_validation? }
     attribute :share_encrypted_env_with_forks, Boolean, default: false
     attribute :share_ssh_keys_with_forks, Boolean, default: lambda { |us, _| us.share_ssh_keys_with_forks? }
+    attribute :job_log_time_based_limit, Boolean, default: lambda { |s, _| s.job_log_access_permissions[:time_based_limit] }
+    attribute :job_log_access_based_limit, Boolean, default: lambda { |s, _| s.job_log_access_permissions[:access_based_limit] }
+    attribute :job_log_access_older_than_days, Integer, default: lambda { |s, _| s.job_log_access_permissions[:older_than_days] }
+
+    validates :job_log_access_older_than_days, numericality: true
+
+    validate :job_log_access_older_than_days_restriction
+
+    set_callback :after_save, :after, :save_audit
 
     attr_reader :repo
 
+    attr_accessor :user, :change_source
+
     def initialize(repo, data)
       @repo = repo
       super(data)
@@ -57,5 +68,24 @@ def cutoff_date
     def days_since_jan_15
       Date.today.mjd - JAN_15.mjd + 1
     end
+
+    def job_log_access_permissions
+      Travis.config.to_h.fetch(:job_log_access_permissions) { {} }
+    end
+
+    def job_log_access_older_than_days_restriction
+      if job_log_access_older_than_days.to_i > job_log_access_permissions[:max_days_value] ||
+        job_log_access_older_than_days.to_i < job_log_access_permissions[:min_days_value]
+        errors.add(:job_log_access_older_than_days, "is outside the bounds")
+      end
+    end
+
+    private
+
+    def save_audit
+      if self.change_source
+        Travis::API::V3::Models::Audit.create!(owner: self.user, change_source: self.change_source, source: self.repo, source_changes: { settings: self.changes })
+      end
+    end
   end
 end
diff --git a/lib/travis/api/v3/queries/log.rb b/lib/travis/api/v3/queries/log.rb
@@ -8,6 +8,7 @@ def find(job)
       @job = job
       remote_log = Travis::RemoteLog::Remote.new(platform: platform).find_by_job_id(platform_job_id)
       raise EntityMissing, 'log not found'.freeze if remote_log.nil?
+
       log = Travis::API::V3::Models::Log.new(remote_log: remote_log, job: job)
       # if the log has been archived, go to s3
       if log.archived?

diff --git a/lib/travis/api/v3/queries/user_setting.rb b/lib/travis/api/v3/queries/user_setting.rb
@@ -6,8 +6,11 @@ def find(repository)
       repository.user_settings.read(_name)
     end
 
-    def update(repository)
-      repository.user_settings.update(_name, _value)
+    def update(repository, user, from_admin)
+      user_settings = repository.user_settings
+      user_settings.user = user
+      user_settings.change_source = 'travis-api' unless from_admin
+      user_settings.update(_name, _value)
     end
 
     private

diff --git a/lib/travis/api/v3/services/log/find.rb b/lib/travis/api/v3/services/log/find.rb
@@ -3,8 +3,14 @@ class Services::Log::Find < Service
     params 'log.token'
 
     def run!
-      log = query.find_by_job_id(params['job.id'])
+      job = Models::Job.find(params['job.id'])
+      repo_can_write = access_control.user ? !!job.repository.users.where(id: access_control.user.id, permissions: { push: true }).first : false
+
+      log = query.find(job)
       raise(NotFound, :log) unless access_control.visible? log
+      raise LogExpired if job.repository.user_settings.job_log_time_based_limit && job.started_at && job.started_at < Time.now - job.repository.user_settings.job_log_access_older_than_days.days
+      raise LogAccessDenied if job.repository.user_settings.job_log_access_based_limit && !repo_can_write
+
       result log
     end
   end

diff --git a/lib/travis/api/v3/services/user_setting/update.rb b/lib/travis/api/v3/services/user_setting/update.rb
@@ -9,8 +9,9 @@ def run!
 
       user_setting = query.find(repository)
       access_control.permissions(user_setting).write!
+      app_id = Travis::Api::App::AccessToken.find_by_token(access_control.token).app_id
 
-      user_setting = query.update(repository)      
+      user_setting = query.update(repository, access_control.user, app_id == 2)
       result user_setting
     end
   end

diff --git a/lib/travis/config/defaults.rb b/lib/travis/config/defaults.rb
@@ -88,7 +88,8 @@ def fallback_logs_api_auth_token
             force_authentication: false,
             yml:                  { url: 'https://yml.travis-ci.org', token: 'secret', auth_key: 'abc123' },
             read_only:            ENV['READ_ONLY'] || false,
-            vcs: {}
+            vcs: {},
+            job_log_access_permissions: { time_based_limit: false, access_based_limit: false, older_than_days: 365, max_days_value: 730, min_days_value: 30 }
 
     default :_access => [:key]
 

diff --git a/lib/travis/model/repository/settings.rb b/lib/travis/model/repository/settings.rb
@@ -101,13 +101,20 @@ def custom_timeouts?(settings)
   attribute :allow_config_imports, Boolean, default: false
   attribute :share_encrypted_env_with_forks, Boolean, default: false
   attribute :share_ssh_keys_with_forks, Boolean, default: nil
+  attribute :job_log_time_based_limit, Boolean, default: lambda { |s, _| s.job_log_access_permissions[:time_based_limit] }
+  attribute :job_log_access_based_limit, Boolean, default: lambda { |s, _| s.job_log_access_permissions[:access_based_limit] }
+  attribute :job_log_access_older_than_days, Integer, default: lambda { |s, _| s.job_log_access_permissions[:older_than_days] }
 
   validates :maximum_number_of_builds, numericality: true
 
   validate :api_builds_rate_limit_restriction
 
   validates_with TimeoutsValidator
 
+  def job_log_access_permissions
+    Travis.config.to_h.fetch(:job_log_access_permissions) { {} }
+  end
+
   def auto_cancel_default?
     ENV.fetch('AUTO_CANCEL_DEFAULT', 'false') == 'true'
   end

diff --git a/lib/travis/remote_log.rb b/lib/travis/remote_log.rb
@@ -224,14 +224,15 @@ def write_content_for_job_id(job_id, content: '', removed_by: nil)
     end
 
     class ArchiveClient
-      def initialize(access_key_id: nil, secret_access_key: nil, bucket_name: nil)
+      def initialize(access_key_id: nil, secret_access_key: nil, bucket_name: nil, region: nil)
         @bucket_name = bucket_name
         @s3 = Fog::Storage.new(
           aws_access_key_id: access_key_id,
           aws_secret_access_key: secret_access_key,
           provider: 'AWS',
           instrumentor: ActiveSupport::Notifications,
-          connection_options: { instrumentor: ActiveSupport::Notifications }
+          connection_options: { instrumentor: ActiveSupport::Notifications },
+          region: region
         )
       end
 
@@ -318,7 +319,8 @@ def initialize(platform: :default)
         ArchiveClient.new(
           access_key_id: archive_s3_config[:access_key_id],
           secret_access_key: archive_s3_config[:secret_access_key],
-          bucket_name: archive_s3_bucket
+          bucket_name: archive_s3_config[:bucket] || archive_s3_bucket,
+          region: archive_s3_config[:region] || 'us-east-2'
         )
       end
 

diff --git a/spec/auth/v2.1/jobs_spec.rb b/spec/auth/v2.1/jobs_spec.rb
@@ -30,7 +30,7 @@
     allow(Travis::RemoteLog::Remote).to receive(:new).and_return(remote)
     allow(remote).to receive(:find_by_job_id).and_return(Travis::RemoteLog.new(log_from_api))
     allow(remote).to receive(:find_by_id).and_return(Travis::RemoteLog.new(log_from_api))
-    allow(remote).to receive(:fetch_archived_url).and_return('https://s3.amazonaws.com/STUFFS')
+    allow(remote).to receive(:fetch_archived_log_content).and_return(archived_content)
   end
 
   before { Job.update_all(state: :started) }
@@ -56,7 +56,7 @@
     end
 
     describe 'GET /jobs/%{job.id}/log' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 404 }
@@ -79,10 +79,10 @@
     end
 
     describe 'GET /jobs/%{job.id}/log' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
-      it(:without_permission) { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
+      it(:without_permission) { should auth status: 200, type: [:json, :text], empty: false }
       it(:invalid_token)      { should auth status: 403 }
-      it(:unauthenticated)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:unauthenticated)    { should auth status: 200, type: [:json, :text], empty: false }
     end
   end
 
@@ -102,7 +102,7 @@
     end
 
     describe 'GET /jobs/%{job.id}/log' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 401 }
@@ -131,7 +131,7 @@
     end
 
     describe 'GET /jobs/%{job.id}/log' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 401 }
@@ -154,10 +154,10 @@
     end
 
     describe 'GET /jobs/%{job.id}/log' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
-      it(:without_permission) { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
+      it(:without_permission) { should auth status: 200, type: [:json, :text], empty: false }
       it(:invalid_token)      { should auth status: 403 }
-      it(:unauthenticated)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:unauthenticated)    { should auth status: 200, type: [:json, :text], empty: false }
     end
   end
 end
diff --git a/spec/auth/v2.1/logs_spec.rb b/spec/auth/v2.1/logs_spec.rb
@@ -31,12 +31,12 @@
     allow(Travis::RemoteLog::Remote).to receive(:new).and_return(remote)
     allow(remote).to receive(:find_by_job_id).and_return(Travis::RemoteLog.new(log_from_api))
     allow(remote).to receive(:find_by_id).and_return(Travis::RemoteLog.new(log_from_api))
-    allow(remote).to receive(:fetch_archived_url).and_return('https://s3.amazonaws.com/STUFFS')
+    allow(remote).to receive(:fetch_archived_log_content).and_return(archived_content)
   end
 
   describe 'in public mode, with a private repo', mode: :public, repo: :private do
     describe 'GET /logs/%{log.id}' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 404 }
@@ -45,16 +45,16 @@
 
   describe 'in public mode, with a public repo', mode: :public, repo: :public do
     describe 'GET /logs/%{log.id}' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
-      it(:without_permission) { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
+      it(:without_permission) { should auth status: 200, type: [:json, :text], empty: false }
       it(:invalid_token)      { should auth status: 403 }
-      it(:unauthenticated)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:unauthenticated)    { should auth status: 200, type: [:json, :text], empty: false }
     end
   end
 
   describe 'in private mode, with a public repo', mode: :private, repo: :public do
     describe 'GET /logs/%{log.id}' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 401 }
@@ -69,7 +69,7 @@
 
   describe 'in private mode, with a private repo', mode: :private, repo: :private do
     describe 'GET /logs/%{log.id}' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
       it(:without_permission) { should auth status: 404 }
       it(:invalid_token)      { should auth status: 403 }
       it(:unauthenticated)    { should auth status: 401 }
@@ -78,10 +78,10 @@
 
   describe 'in org mode, with a public repo', mode: :org, repo: :public do
     describe 'GET /logs/%{log.id}' do
-      it(:with_permission)    { should auth status: [200, 307], type: :json, empty: false }
-      it(:without_permission) { should auth status: [200, 307], type: :json, empty: false }
+      it(:with_permission)    { should auth status: 200, type: [:json, :text], empty: false }
+      it(:without_permission) { should auth status: 200, type: [:json, :text], empty: false }
       it(:invalid_token)      { should auth status: 403 }
-      it(:unauthenticated)    { should auth status: [200, 307], type: :json, empty: false }
+      it(:unauthenticated)    { should auth status: 200, type: [:json, :text], empty: false }
     end
   end
 end