diff --git a/continuous_reporting/benchmark_and_update.rb b/continuous_reporting/benchmark_and_update.rb index 03a830231..f4e2ac985 100755 --- a/continuous_reporting/benchmark_and_update.rb +++ b/continuous_reporting/benchmark_and_update.rb @@ -36,10 +36,10 @@ # tolerance needs to be significantly larger to avoid frequent false positives. MICRO_DROP_TOLERANCE = 0.20 -YM_PAGES_DIR = File.expand_path "#{__dir__}/../../yjit-metrics-pages" -YM_RAW_DATA_DIR = "#{YM_PAGES_DIR}/raw_benchmark_data" +BUILT_REPORTS_ROOT = YJITMetrics::ContinuousReporting::BUILT_REPORTS_ROOT +RAW_BENCHMARK_ROOT = YJITMetrics::ContinuousReporting::RAW_BENCHMARK_ROOT -YM_REPORT_DIR = File.expand_path "#{YM_PAGES_DIR}/_includes/reports/" +YM_REPORT_DIR = File.expand_path "#{BUILT_REPORTS_ROOT}/_includes/reports/" if File.exist?(YM_REPORT_DIR) var_warmup_reports = Dir.glob(YM_REPORT_DIR + "/variable_warmup_*.warmup_settings.json").to_a if var_warmup_reports.empty? @@ -204,7 +204,7 @@ def ghapi_post(api_uri, params, verb: :post) class BenchmarkDetails def initialize(timestamp) @timestamp = timestamp - benchmark_details_file = File.join(YM_PAGES_DIR, "_benchmarks", "bench_#{timestamp}.md") + benchmark_details_file = File.join(BUILT_REPORTS_ROOT, "_benchmarks", "bench_#{timestamp}.md") @data = YAML.load File.read(benchmark_details_file) end @@ -215,7 +215,7 @@ def raw_data def yjit_test_result yjit_file = @data["test_results"]["prod_ruby_with_yjit"] raise("Cannot locate latest YJIT data file for timestamp #{@timestamp}") unless yjit_file - File.join YM_PAGES_DIR, yjit_file + File.join RAW_BENCHMARK_ROOT, yjit_file end def yjit_permalink @@ -329,7 +329,7 @@ def timestr_from_ts(ts) # If benchmark results drop noticeably, file a Github issue def check_perf_tripwires - Dir.chdir(__dir__ + "/../../yjit-metrics-pages/_includes/reports") do + Dir.chdir(YM_REPORT_DIR) do # Grab only non-platform-specific tripwire files that do *not* have a platform name in them, # but instead end in a six-digit timestamp. tripwire_files = Dir["*.tripwires.json"].to_a.select {|f| f =~ /\d{6}\.tripwires\.json\Z/}.sort diff --git a/continuous_reporting/create_json_params_file.rb b/continuous_reporting/create_json_params_file.rb index 09d2bea6b..a7bad1d55 100644 --- a/continuous_reporting/create_json_params_file.rb +++ b/continuous_reporting/create_json_params_file.rb @@ -57,7 +57,7 @@ yjit_metrics_repo = "" yjit_bench_name = "main" yjit_bench_repo = "https://github.com/Shopify/yjit-bench.git" -benchmark_data_dir = File.join(YJIT_METRICS_DIR, "raw-benchmark-data/raw_benchmark_data") +benchmark_data_dir = nil # TODO: try looking up the given yjit_metrics and/or yjit_bench and/or CRuby revisions in the local repos to see if they exist? @@ -108,10 +108,13 @@ end opts.on("-bd PATH", "--benchmark-data-dir PATH") do |dir| + raise "--benchmark-data-dir must specify a directory" if dir.to_s.empty? benchmark_data_dir = File.expand_path(dir) end end.parse! +raise "--benchmark-data-dir is required!" unless benchmark_data_dir + def sha_for_name_in_dir(name:, dir:, repo:, desc:) Dir.chdir(dir) do system("git remote remove current_repo") # Don't care if this succeeds or not diff --git a/continuous_reporting/file_benchmark_data_into_raw.rb b/continuous_reporting/file_benchmark_data_into_raw.rb index d9c7963de..e53e8d4dc 100755 --- a/continuous_reporting/file_benchmark_data_into_raw.rb +++ b/continuous_reporting/file_benchmark_data_into_raw.rb @@ -7,15 +7,9 @@ require_relative "../lib/yjit-metrics" -YJIT_METRICS_PAGES_DIR = File.expand_path File.join(__dir__, "../../yjit-metrics-pages/raw_benchmark_data") -unless File.exist?(YJIT_METRICS_PAGES_DIR) - raise "This script expects to be cloned in a repo right next to a \"yjit-metrics-pages\" repo of the `pages` branch of yjit-metrics" -end - -YJIT_RAW_DATA_REPO = File.expand_path File.join(__dir__, "../../raw-benchmark-data/raw_benchmark_data") +YJIT_RAW_DATA_REPO = File.join(YJITMetrics::ContinuousReporting::RAW_BENCHMARK_ROOT, "raw_benchmark_data") DESTINATIONS = [ - #YJIT_METRICS_PAGES_DIR, YJIT_RAW_DATA_REPO, ] diff --git a/continuous_reporting/generate_and_upload_reports.rb b/continuous_reporting/generate_and_upload_reports.rb index 6b2af087b..fdf0e9969 100755 --- a/continuous_reporting/generate_and_upload_reports.rb +++ b/continuous_reporting/generate_and_upload_reports.rb @@ -9,25 +9,13 @@ ### Required repos, etc, to build from -# Dir in which yjit-metrics, yjit-bench, etc are cloned -YM_ROOT_DIR = File.expand_path(File.join(__dir__, "../..")) +YM_REPO = YJITMetrics::ContinuousReporting::YM_REPO +RAW_BENCHMARK_ROOT = YJITMetrics::ContinuousReporting::RAW_BENCHMARK_ROOT +RAW_REPORTS_ROOT = YJITMetrics::ContinuousReporting::RAW_REPORTS_ROOT +BUILT_REPORTS_ROOT = YJITMetrics::ContinuousReporting::BUILT_REPORTS_ROOT +GHPAGES_REPO = YJITMetrics::ContinuousReporting::GHPAGES_REPO -# Clone of yjit-metrics repo, pages branch -YJIT_METRICS_PAGES_DIR = File.expand_path File.join(YM_ROOT_DIR, "yjit-metrics-pages") - -# Raw benchmark data gets written to a platform- and date-specific subdirectory, but will often be read from multiple subdirectories -RAW_BENCHMARK_ROOT = File.join(YM_ROOT_DIR, "raw-benchmark-data") - -# This contains Jekyll source files of various kinds - everything but the built reports -RAW_REPORTS_ROOT = File.join(YM_ROOT_DIR, "raw-yjit-reports") - -# We cache all the built per-run reports, which can take a long time to rebuild -BUILT_REPORTS_ROOT = File.join(YM_ROOT_DIR, "built-yjit-reports") - -# We have a separate repo for the final HTML, because creating the new orphan branch is fiddly -GHPAGES_REPO = File.join(YM_ROOT_DIR, "ghpages-yjit-metrics") - -[YJIT_METRICS_PAGES_DIR, RAW_BENCHMARK_ROOT, RAW_REPORTS_ROOT, BUILT_REPORTS_ROOT].each do |dir| +[RAW_BENCHMARK_ROOT, RAW_REPORTS_ROOT, BUILT_REPORTS_ROOT].each do |dir| unless File.exist?(dir) raise "We expected directory #{dir.inspect} to exist in order to generate reports!" end @@ -137,10 +125,9 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl REPORTS_AND_FILES.select! { |k, _| only_reports.include?(k) } end -# From here on out, we're just in the yjit-metrics checkout of "pages" -- until we can stop relying on it. -Dir.chdir(YJIT_METRICS_PAGES_DIR) +# This is probably unnecessary now. +Dir.chdir(YM_REPO) puts "Switched to #{Dir.pwd}" -YJITMetrics.check_call("git checkout pages") # Turn JSON files into reports where outdated - first, find out what test results we have. # json_timestamps maps timestamps to file paths relative to the RAW_BENCHMARK_ROOT @@ -214,7 +201,7 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl reason ||= "we're missing files: #{missing_files.inspect}" puts "Running basic_report for timestamp #{ts} because #{reason} with data files #{test_files.inspect}" - YJITMetrics.check_call("ruby ../yjit-metrics/basic_report.rb -d #{RAW_BENCHMARK_ROOT} --report=#{report_name} -o #{BUILT_REPORTS_ROOT}/_includes/reports -w #{test_files.join(" ")}") + YJITMetrics.check_call("ruby #{YM_REPO}/basic_report.rb -d #{RAW_BENCHMARK_ROOT} --report=#{report_name} -o #{BUILT_REPORTS_ROOT}/_includes/reports -w #{test_files.join(" ")}") rf = basic_report_filenames(report_name, ts) files_not_found = rf.select { |f| !File.exist? f } @@ -252,7 +239,7 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl # Add a field like blog_speed_details_x86_64_svg YJITMetrics::PLATFORMS.each do |platform| report_filename = "reports/#{report_name}_#{ts}.#{platform}.#{ext}" - if File.exist?("_includes/#{report_filename}") + if File.exist?("#{BUILT_REPORTS_ROOT}/_includes/#{report_filename}") platforms[platform] = true generated_reports[report_name + "_" + platform + "_" + ext.gsub(".", "_")] = report_filename end @@ -292,7 +279,7 @@ def basic_report_filenames(report_name, ts, prefix: "#{BUILT_REPORTS_ROOT}/_incl # It's possible to run only specific non-timeline reports -- then this would be empty. unless timeline_reports.empty? - YJITMetrics.check_call("ruby ../yjit-metrics/timeline_report.rb -d #{RAW_BENCHMARK_ROOT} --report='#{timeline_reports.keys.join(",")}' -o #{BUILT_REPORTS_ROOT}") + YJITMetrics.check_call("ruby #{YM_REPO}/timeline_report.rb -d #{RAW_BENCHMARK_ROOT} --report='#{timeline_reports.keys.join(",")}' -o #{BUILT_REPORTS_ROOT}") end # TODO: figure out a new way to verify that appropriate files were written. With various subdirs, the old way won't cut it. diff --git a/continuous_reporting/gh_tasks/check_reporting.sh b/continuous_reporting/gh_tasks/check_reporting.sh index b71694336..e863314b5 100755 --- a/continuous_reporting/gh_tasks/check_reporting.sh +++ b/continuous_reporting/gh_tasks/check_reporting.sh @@ -9,13 +9,10 @@ chruby 3.0.2 cd ~/ym/yjit-metrics git pull -# Copy benchmark raw data into yjit-metrics-pages repo +# Copy benchmark raw data into destination repo #ruby continuous_reporting/file_benchmark_data_into_raw.rb -d continuous_reporting/data -# Copy benchmark raw data into yjit-metrics-pages repo, generate reports, commit changes to Git. +# Copy benchmark raw data into destination repo, generate reports, commit changes to Git. ruby continuous_reporting/generate_and_upload_reports.rb --no-push -# Don't keep the provisional report files around -cd ../yjit-metrics-pages && git checkout reports _includes - echo "Reporting check completed successfully." diff --git a/continuous_reporting/gh_tasks/check_reporting_minimal.sh b/continuous_reporting/gh_tasks/check_reporting_minimal.sh index 77177b802..72248b751 100755 --- a/continuous_reporting/gh_tasks/check_reporting_minimal.sh +++ b/continuous_reporting/gh_tasks/check_reporting_minimal.sh @@ -11,13 +11,10 @@ chruby 3.0.2 cd ~/ym/yjit-metrics -# Copy benchmark raw data into yjit-metrics-pages repo +# Copy benchmark raw data into destination repo #ruby continuous_reporting/file_benchmark_data_into_raw.rb -d continuous_reporting/data # Generate reports, commit changes to Git. ruby continuous_reporting/generate_and_upload_reports.rb --no-push --only-reports blog_timeline -# Don't keep the provisional report files around -cd ../yjit-metrics-pages && git checkout reports _includes && git clean -d -f reports _includes - echo "Minimal reporting check completed successfully." diff --git a/continuous_reporting/gh_tasks/report_and_upload.sh b/continuous_reporting/gh_tasks/report_and_upload.sh index 656820343..8960d1503 100755 --- a/continuous_reporting/gh_tasks/report_and_upload.sh +++ b/continuous_reporting/gh_tasks/report_and_upload.sh @@ -10,7 +10,7 @@ chruby 3.0.2 cd ~/ym/yjit-metrics -# Copy benchmark raw data into yjit-metrics-pages repo, generate reports, commit changes to Git. +# Copy benchmark raw data into destination repo, generate reports, commit changes to Git. ruby continuous_reporting/generate_and_upload_reports.rb # Now we'll verify that we're not regenerating results when we shouldn't. diff --git a/continuous_reporting/gh_tasks/report_full_rebuild.sh b/continuous_reporting/gh_tasks/report_full_rebuild.sh index 1f0dba279..b06d2bfae 100755 --- a/continuous_reporting/gh_tasks/report_full_rebuild.sh +++ b/continuous_reporting/gh_tasks/report_full_rebuild.sh @@ -10,7 +10,7 @@ chruby 3.0.2 cd ~/ym/yjit-metrics -# Copy benchmark raw data into yjit-metrics-pages repo, generate reports, commit changes to Git. +# Copy benchmark raw data into destination repo, generate reports, commit changes to Git. # The --regenerate-reports argument will regenerate ***all*** reports, which can take quite a # long time. It will also occasionally hit a Ruby error, so we should update from 3.0.2 when # we can for "system" Ruby here. diff --git a/continuous_reporting/jenkins/Jenkinsfile_data_checkin_and_upload b/continuous_reporting/jenkins/Jenkinsfile_data_checkin_and_upload index 9f9b0d04b..171d3bb2a 100644 --- a/continuous_reporting/jenkins/Jenkinsfile_data_checkin_and_upload +++ b/continuous_reporting/jenkins/Jenkinsfile_data_checkin_and_upload @@ -19,7 +19,7 @@ pipeline { } stages { - // This stage commits and pushes the benchmark data from the yjit-metrics-pages directory, but doesn't get it there from yjit-metrics/continuous_reporting/data + // This stage commits and pushes the benchmark data from its directory, but doesn't get it there from yjit-metrics/continuous_reporting/data stage('data checkin') { parallel { stage('data checkin x86_64') { diff --git a/lib/yjit-metrics.rb b/lib/yjit-metrics.rb index 461d5bf79..4fac2e215 100644 --- a/lib/yjit-metrics.rb +++ b/lib/yjit-metrics.rb @@ -9,6 +9,7 @@ require_relative "./yjit-metrics/bench-results" require_relative "./yjit-metrics/repo-management" require_relative "./yjit-metrics/cli-interface" +require_relative "./yjit-metrics/continuous_reporting" # Require all source files in yjit-metrics/report_types/*.rb Dir.glob("yjit-metrics/report_types/*.rb", base: __dir__).each do |report_type_file| diff --git a/lib/yjit-metrics/continuous_reporting.rb b/lib/yjit-metrics/continuous_reporting.rb new file mode 100644 index 000000000..b4835673e --- /dev/null +++ b/lib/yjit-metrics/continuous_reporting.rb @@ -0,0 +1,21 @@ +module YJITMetrics + module ContinuousReporting + # Dir in which yjit-metrics, yjit-bench, etc are cloned + YM_ROOT_DIR = File.expand_path(File.join(__dir__, "../../..")) + + # This repo. + YM_REPO = File.join(YM_ROOT_DIR, "yjit-metrics") + + # Raw benchmark data gets written to a platform- and date-specific subdirectory, but will often be read from multiple subdirectories + RAW_BENCHMARK_ROOT = File.join(YM_ROOT_DIR, "raw-benchmark-data") + + # This contains Jekyll source files of various kinds - everything but the built reports + RAW_REPORTS_ROOT = File.join(YM_ROOT_DIR, "raw-yjit-reports") + + # We cache all the built per-run reports, which can take a long time to rebuild + BUILT_REPORTS_ROOT = File.join(YM_ROOT_DIR, "built-yjit-reports") + + # We have a separate repo for the final HTML, because creating the new orphan branch is fiddly + GHPAGES_REPO = File.join(YM_ROOT_DIR, "ghpages-yjit-metrics") + end +end