From 6b49645c5739a35ca37881d343f5dab56f39062d Mon Sep 17 00:00:00 2001 From: Omar Tawfik <15987992+OmarTawfik@users.noreply.github.com> Date: Mon, 4 Dec 2023 04:05:46 -0800 Subject: [PATCH 1/3] download a fresh list of compiler binaries if out of date Makes sure we fetch new versions when they are released. --- .../testing/solc/src/utils/binaries.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/solidity/testing/solc/src/utils/binaries.rs b/crates/solidity/testing/solc/src/utils/binaries.rs index b578174a98..4ccc009464 100644 --- a/crates/solidity/testing/solc/src/utils/binaries.rs +++ b/crates/solidity/testing/solc/src/utils/binaries.rs @@ -6,7 +6,10 @@ use infra_utils::{cargo::CargoWorkspace, commands::Command}; use rayon::prelude::{ParallelBridge, ParallelIterator}; use semver::Version; use serde::Deserialize; -use std::{collections::HashMap, os::unix::prelude::PermissionsExt, path::Path, path::PathBuf}; +use std::{ + collections::HashMap, os::unix::prelude::PermissionsExt, path::Path, path::PathBuf, + time::Duration, +}; use url::Url; #[derive(Debug)] @@ -34,7 +37,11 @@ impl Binary { .map(|version| { let local_path = binaries_dir.join(version.to_string()); if !local_path.exists() { - let remote_url = mirror_url.join(&releases[version]).unwrap(); + let release = releases.get(version).unwrap_or_else(|| { + panic!("Expected release '{version}' to exist at: {mirror_url}") + }); + + let remote_url = mirror_url.join(release).unwrap(); download_file(remote_url, &local_path); make_file_executable(&local_path); } @@ -74,7 +81,19 @@ fn fetch_releases(mirror_url: &Url, binaries_dir: &Path) -> HashMap one_day + } else { + // Doesn't exist. Download it: + true + }; + + if should_download_list { let list_url = mirror_url.join("list.json").unwrap(); download_file(list_url, &list_path); } From 41474e0d781bf81a48d1a31d53ee59e8af993fe7 Mon Sep 17 00:00:00 2001 From: OmarTawfik <15987992+OmarTawfik@users.noreply.github.com> Date: Mon, 4 Dec 2023 04:05:46 -0800 Subject: [PATCH 2/3] replace panics --- .../solidity/testing/solc/src/keywords/mod.rs | 9 +- crates/solidity/testing/solc/src/main.rs | 9 +- .../testing/solc/src/utils/binaries.rs | 89 +++++++++---------- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/crates/solidity/testing/solc/src/keywords/mod.rs b/crates/solidity/testing/solc/src/keywords/mod.rs index 3e17cb9735..5a07811f41 100644 --- a/crates/solidity/testing/solc/src/keywords/mod.rs +++ b/crates/solidity/testing/solc/src/keywords/mod.rs @@ -1,4 +1,5 @@ use crate::utils::{ApiInput, Binary, InputSource}; +use anyhow::Result; use codegen_language_definition::model::{Item, KeywordDefinition, KeywordItem, Language}; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; @@ -6,7 +7,7 @@ use semver::Version; use solidity_language::SolidityDefinition; use std::collections::HashSet; -pub fn check_solidity_keywords() { +pub fn check_solidity_keywords() -> Result<()> { println!(); println!(" > Checking Solidity keywords:"); println!(); @@ -18,7 +19,7 @@ pub fn check_solidity_keywords() { println!(" > Downloading solc binaries:"); println!(); - let binaries = Binary::fetch_all(&language); + let binaries = Binary::fetch_all(&language)?; println!(); println!(" > Testing all variations:"); @@ -27,7 +28,7 @@ pub fn check_solidity_keywords() { let progress_bar = ProgressBar::new(test_cases.len() as u64); let style = "[{elapsed_precise}] [{bar:80.cyan/blue}] {pos}/{len} │ ETA: {eta_precise}"; - progress_bar.set_style(ProgressStyle::with_template(style).unwrap()); + progress_bar.set_style(ProgressStyle::with_template(style)?); test_cases.par_iter().for_each(|test_case| { let result = test_case.execute(&binaries); @@ -55,6 +56,8 @@ pub fn check_solidity_keywords() { progress_bar.finish(); println!(); + + Ok(()) } fn generate_test_cases(language: &Language) -> Vec { diff --git a/crates/solidity/testing/solc/src/main.rs b/crates/solidity/testing/solc/src/main.rs index 91646a97d3..30b06222a6 100644 --- a/crates/solidity/testing/solc/src/main.rs +++ b/crates/solidity/testing/solc/src/main.rs @@ -2,6 +2,7 @@ mod keywords; mod utils; use crate::keywords::check_solidity_keywords; +use anyhow::Result; use clap::{Parser, Subcommand}; #[derive(Debug, Parser)] @@ -16,12 +17,10 @@ enum AppCommand { CheckSolidityKeywords, } -fn main() { +fn main() -> Result<()> { match CLI::parse().command { - AppCommand::CheckSolidityKeywords => { - check_solidity_keywords(); - } - }; + AppCommand::CheckSolidityKeywords => check_solidity_keywords(), + } } #[test] diff --git a/crates/solidity/testing/solc/src/utils/binaries.rs b/crates/solidity/testing/solc/src/utils/binaries.rs index 4ccc009464..aa88d72282 100644 --- a/crates/solidity/testing/solc/src/utils/binaries.rs +++ b/crates/solidity/testing/solc/src/utils/binaries.rs @@ -2,7 +2,7 @@ use crate::utils::{ApiInput, ApiOutput}; use anyhow::Result; use codegen_language_definition::model::Language; use indicatif::{ProgressBar, ProgressStyle}; -use infra_utils::{cargo::CargoWorkspace, commands::Command}; +use infra_utils::{cargo::CargoWorkspace, commands::Command, paths::PathExtensions}; use rayon::prelude::{ParallelBridge, ParallelIterator}; use semver::Version; use serde::Deserialize; @@ -20,17 +20,17 @@ pub struct Binary { } impl Binary { - pub fn fetch_all(language: &Language) -> Vec { - let binaries_dir = get_binaries_dir(); - let mirror_url = get_mirror_url(); - let releases = fetch_releases(&mirror_url, &binaries_dir); + pub fn fetch_all(language: &Language) -> Result> { + let binaries_dir = get_binaries_dir()?; + let mirror_url = get_mirror_url()?; + let releases = fetch_releases(&mirror_url, &binaries_dir)?; let progress_bar = ProgressBar::new(language.versions.len() as u64); let style = "[{elapsed_precise}] [{bar:80.cyan/blue}] {pos}/{len} │ ETA: {eta_precise}"; - progress_bar.set_style(ProgressStyle::with_template(style).unwrap()); + progress_bar.set_style(ProgressStyle::with_template(style)?); - let mut binaries: Vec<_> = language + let mut binaries = language .versions .iter() .par_bridge() @@ -41,40 +41,40 @@ impl Binary { panic!("Expected release '{version}' to exist at: {mirror_url}") }); - let remote_url = mirror_url.join(release).unwrap(); - download_file(remote_url, &local_path); - make_file_executable(&local_path); + let remote_url = mirror_url.join(release)?; + download_file(remote_url, &local_path)?; + make_file_executable(&local_path)?; } progress_bar.inc(1); - Self { + Ok(Self { version: version.to_owned(), local_path, - } + }) }) - .collect(); + .collect::>>()?; progress_bar.finish(); println!(); binaries.sort_by_key(|binary| binary.version.to_owned()); - binaries + Ok(binaries) } pub fn run(&self, input: &ApiInput) -> Result { - let input = serde_json::to_string(input).unwrap(); + let input = serde_json::to_string(input)?; - let output = Command::new(self.local_path.to_str().unwrap()) + let output = Command::new(self.local_path.unwrap_str()) .flag("--standard-json") .evaluate_with_input(input)?; - Ok(serde_json::from_str(&output).unwrap()) + Ok(serde_json::from_str(&output)?) } } -fn fetch_releases(mirror_url: &Url, binaries_dir: &Path) -> HashMap { +fn fetch_releases(mirror_url: &Url, binaries_dir: &Path) -> Result> { #[derive(Deserialize)] struct MirrorList { releases: HashMap, @@ -82,47 +82,40 @@ fn fetch_releases(mirror_url: &Url, binaries_dir: &Path) -> HashMap one_day - } else { - // Doesn't exist. Download it: - true + let download_fresh_list = match list_path.metadata() { + Err(_) => true, + Ok(metadata) => metadata.created()?.elapsed()? > Duration::from_secs(60 * 60 * 24), }; - if should_download_list { - let list_url = mirror_url.join("list.json").unwrap(); - download_file(list_url, &list_path); + if download_fresh_list { + let list_url = mirror_url.join("list.json")?; + download_file(list_url, &list_path)?; } - let list_file = std::fs::read_to_string(list_path).unwrap(); - let list: MirrorList = serde_json::from_str(&list_file).unwrap(); - list.releases + let list_file = std::fs::read_to_string(list_path)?; + let list: MirrorList = serde_json::from_str(&list_file)?; + Ok(list.releases) } -fn download_file(url: Url, path: &Path) { - let bytes = reqwest::blocking::get(url).unwrap().bytes().unwrap(); - - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, bytes).unwrap(); +fn download_file(url: Url, path: &Path) -> Result<()> { + let bytes = reqwest::blocking::get(url)?.bytes()?; + std::fs::create_dir_all(path.unwrap_parent())?; + std::fs::write(path, bytes)?; + Ok(()) } -fn make_file_executable(local_path: &PathBuf) { - let mut permissions = local_path.metadata().unwrap().permissions(); +fn make_file_executable(local_path: &PathBuf) -> Result<()> { + let mut permissions = local_path.metadata()?.permissions(); permissions.set_mode(0o744); - std::fs::set_permissions(local_path, permissions).unwrap(); + std::fs::set_permissions(local_path, permissions)?; + Ok(()) } -fn get_binaries_dir() -> PathBuf { - CargoWorkspace::locate_source_crate("solidity_testing_solc") - .unwrap() - .join("target/binaries") +fn get_binaries_dir() -> Result { + Ok(CargoWorkspace::locate_source_crate("solidity_testing_solc")?.join("target/binaries")) } -fn get_mirror_url() -> Url { +fn get_mirror_url() -> Result { let platform_dir = if cfg!(target_os = "macos") { "macosx-amd64" } else { @@ -132,7 +125,5 @@ fn get_mirror_url() -> Url { ); }; - format!("https://binaries.soliditylang.org/{platform_dir}/") - .parse() - .unwrap() + Ok(format!("https://binaries.soliditylang.org/{platform_dir}/").parse()?) } From b7161a59e92799e5177b1abee855978298684e80 Mon Sep 17 00:00:00 2001 From: OmarTawfik <15987992+OmarTawfik@users.noreply.github.com> Date: Mon, 4 Dec 2023 04:11:32 -0800 Subject: [PATCH 3/3] pr feedback --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/solidity/testing/solc/src/utils/binaries.rs | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04e05ae4c3..87dfd0bfc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,9 +113,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" dependencies = [ "backtrace", ] diff --git a/Cargo.toml b/Cargo.toml index bb1d16c5fb..ac80005268 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ solidity_testing_utils = { path = "crates/solidity/testing/utils" } # # External # -anyhow = { version = "1.0.70", features = ["backtrace", "std"] } +anyhow = { version = "1.0.75", features = ["backtrace", "std"] } ariadne = { version = "0.2.0" } bson = { version = "2.6.1" } cargo-emit = { version = "0.2.1" } diff --git a/crates/solidity/testing/solc/src/utils/binaries.rs b/crates/solidity/testing/solc/src/utils/binaries.rs index aa88d72282..8de75a52b3 100644 --- a/crates/solidity/testing/solc/src/utils/binaries.rs +++ b/crates/solidity/testing/solc/src/utils/binaries.rs @@ -82,12 +82,13 @@ fn fetch_releases(mirror_url: &Url, binaries_dir: &Path) -> Result true, + let should_download_list = match list_path.metadata() { + Err(err) if err.kind() == std::io::ErrorKind::NotFound => true, + Err(err) => return Err(err)?, Ok(metadata) => metadata.created()?.elapsed()? > Duration::from_secs(60 * 60 * 24), }; - if download_fresh_list { + if should_download_list { let list_url = mirror_url.join("list.json")?; download_file(list_url, &list_path)?; }