diff --git a/CHANGELOG.md b/CHANGELOG.md index e61480927..fda6cc40f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Changed +- Include lockfile paths when analyzing projects + ## [5.5.0] - 2023-07-18 ### Added diff --git a/Cargo.lock b/Cargo.lock index 53d9cbed7..bae92379d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1111,7 +1111,7 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "phf", + "phf 0.10.1", "pin-project", "ring", "serde", @@ -3521,18 +3521,38 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.10.0", + "phf_shared 0.10.0", "proc-macro-hack", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros 0.11.2", + "phf_shared 0.11.2", +] + [[package]] name = "phf_generator" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ - "phf_shared", + "phf_shared 0.10.0", + "rand 0.8.5", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", "rand 0.8.5", ] @@ -3542,14 +3562,28 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.10.0", + "phf_shared 0.10.0", "proc-macro-hack", "proc-macro2 1.0.66", "quote 1.0.32", "syn 1.0.109", ] +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", + "proc-macro2 1.0.66", + "quote 1.0.32", + "syn 2.0.18", + "unicase", +] + [[package]] name = "phf_shared" version = "0.10.0" @@ -3559,6 +3593,16 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", + "unicase", +] + [[package]] name = "phylum-cli" version = "5.5.0" @@ -3655,9 +3699,10 @@ dependencies = [ [[package]] name = "phylum_types" version = "0.1.0" -source = "git+https://github.com/phylum-dev/phylum-types?branch=development#1706737e28262bb92d00c41b7c6cd67a9e4ce506" +source = "git+https://github.com/phylum-dev/phylum-types?branch=development#4e8fa6a63fa29a44b2061d6919b467755d8c212a" dependencies = [ "chrono", + "purl", "schemars", "serde", "serde_derive", @@ -3904,6 +3949,20 @@ dependencies = [ "cc", ] +[[package]] +name = "purl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9970524438ae20d0f91d587f80f930dc2a1264d8a4904a2f1c0d099b5c9a746" +dependencies = [ + "hex", + "percent-encoding", + "phf 0.11.2", + "smartstring", + "thiserror", + "unicase", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4925,7 +4984,7 @@ dependencies = [ "new_debug_unreachable", "once_cell", "parking_lot", - "phf_shared", + "phf_shared 0.10.0", "precomputed-hash", "serde", ] @@ -4936,8 +4995,8 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.10.0", + "phf_shared 0.10.0", "proc-macro2 1.0.66", "quote 1.0.32", ] @@ -5149,7 +5208,7 @@ dependencies = [ "bitflags 2.3.3", "indexmap 1.9.3", "once_cell", - "phf", + "phf 0.10.1", "rustc-hash", "serde", "smallvec", @@ -5826,6 +5885,15 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/cli/src/api/mod.rs b/cli/src/api/mod.rs index ccdbce3fd..ef63e84fa 100644 --- a/cli/src/api/mod.rs +++ b/cli/src/api/mod.rs @@ -10,7 +10,9 @@ use phylum_types::types::group::{ use phylum_types::types::job::{ AllJobsStatusResponse, SubmitPackageRequest, SubmitPackageResponse, }; -use phylum_types::types::package::{PackageDescriptor, PackageSpecifier, PackageSubmitResponse}; +use phylum_types::types::package::{ + PackageDescriptor, PackageDescriptorAndLockfile, PackageSpecifier, PackageSubmitResponse, +}; use phylum_types::types::project::{ CreateProjectRequest, CreateProjectResponse, ProjectSummaryResponse, }; @@ -277,7 +279,7 @@ impl PhylumApi { /// Submit a new request to the system pub async fn submit_request( &self, - package_list: &[PackageDescriptor], + package_list: &[PackageDescriptorAndLockfile], project: ProjectId, label: Option, group_name: Option, @@ -500,11 +502,17 @@ mod tests { let client = build_phylum_api(&mock_server).await?; - let pkg = PackageDescriptor { + let package_descriptor = PackageDescriptor { name: "react".to_string(), version: "16.13.1".to_string(), package_type: PackageType::Npm, }; + + let pkg = PackageDescriptorAndLockfile { + package_descriptor, + lockfile: Some("package-lock.json".to_owned()), + }; + let project_id = ProjectId::new_v4(); let label = Some("mylabel".to_string()); client.submit_request(&[pkg], project_id, label, None).await?; @@ -530,11 +538,17 @@ mod tests { let client = build_phylum_api(&mock_server).await?; - let pkg = PackageDescriptor { + let package_descriptor = PackageDescriptor { name: "react".to_string(), version: "16.13.1".to_string(), package_type: PackageType::Npm, }; + + let pkg = PackageDescriptorAndLockfile { + package_descriptor, + lockfile: Some("package-lock.json".to_owned()), + }; + let project_id = ProjectId::new_v4(); let label = Some("mylabel".to_string()); client.submit_request(&[pkg], project_id, label, None).await?; diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index e7459f079..379dca557 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -22,7 +22,8 @@ use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; use phylum_types::types::group::ListUserGroupsResponse; use phylum_types::types::package::{ - Package, PackageDescriptor, PackageSpecifier as PTPackageSpecifier, PackageSubmitResponse, + Package, PackageDescriptor, PackageDescriptorAndLockfile, + PackageSpecifier as PTPackageSpecifier, PackageSubmitResponse, }; use phylum_types::types::project::ProjectSummaryResponse; use reqwest::StatusCode; @@ -137,7 +138,7 @@ struct ProcessOutput { #[op] async fn analyze( op_state: Rc>, - packages: Vec, + packages: Vec, project: Option, group: Option, label: Option, diff --git a/cli/src/commands/jobs.rs b/cli/src/commands/jobs.rs index fb52869a8..a92fe6406 100644 --- a/cli/src/commands/jobs.rs +++ b/cli/src/commands/jobs.rs @@ -116,7 +116,7 @@ pub async fn handle_submission(api: &mut PhylumApi, matches: &clap::ArgMatches) ); } - packages.extend(res.packages.into_iter()); + packages.extend(res.into_iter()); } if let Some(base) = matches.get_one::("base") { @@ -159,11 +159,14 @@ pub async fn handle_submission(api: &mut PhylumApi, matches: &clap::ArgMatches) let pkg_version = pkg_info.pop().unwrap(); let pkg_name = pkg_info.join(":"); - packages.push(PackageDescriptor { - name: pkg_name.to_owned(), - version: pkg_version.to_owned(), - package_type: request_type.to_owned(), - }); + packages.push( + PackageDescriptor { + name: pkg_name.to_owned(), + version: pkg_version.to_owned(), + package_type: request_type.to_owned(), + } + .into(), + ); line.clear(); }, Err(err) => { @@ -198,6 +201,8 @@ pub async fn handle_submission(api: &mut PhylumApi, matches: &clap::ArgMatches) if synch { if pretty_print { + #[cfg(feature = "vulnreach")] + let packages: Vec<_> = packages.into_iter().map(|pkg| pkg.package_descriptor).collect(); #[cfg(feature = "vulnreach")] if let Err(err) = vulnreach(api, matches, packages, job_id.to_string()).await { print_user_failure!("Reachability analysis failed: {err:?}"); diff --git a/cli/src/commands/parse.rs b/cli/src/commands/parse.rs index a5ba1561c..f3c3332c7 100644 --- a/cli/src/commands/parse.rs +++ b/cli/src/commands/parse.rs @@ -1,11 +1,12 @@ //! `phylum parse` command for lockfile parsing use std::path::{Path, PathBuf}; +use std::vec::IntoIter; use std::{fs, io}; use anyhow::{anyhow, Context, Result}; use phylum_lockfile::{LockfileFormat, Package, PackageVersion, Parse, ThirdPartyVersion}; -use phylum_types::types::package::PackageDescriptor; +use phylum_types::types::package::{PackageDescriptor, PackageDescriptorAndLockfile}; use walkdir::WalkDir; use crate::commands::{CommandResult, ExitCode}; @@ -17,6 +18,31 @@ pub struct ParsedLockfile { pub packages: Vec, } +pub struct ParsedLockfileIterator { + path: PathBuf, + packages: IntoIter, +} + +impl Iterator for ParsedLockfileIterator { + type Item = PackageDescriptorAndLockfile; + + fn next(&mut self) -> Option { + self.packages.next().map(|package_descriptor| PackageDescriptorAndLockfile { + package_descriptor, + lockfile: Some(self.path.to_string_lossy().into_owned()), + }) + } +} + +impl IntoIterator for ParsedLockfile { + type IntoIter = ParsedLockfileIterator; + type Item = PackageDescriptorAndLockfile; + + fn into_iter(self) -> Self::IntoIter { + ParsedLockfileIterator { path: self.path, packages: self.packages.into_iter() } + } +} + pub fn lockfile_types(add_auto: bool) -> Vec<&'static str> { let mut lockfile_types = LockfileFormat::iter().map(|format| format.name()).collect::>(); @@ -31,10 +57,11 @@ pub fn lockfile_types(add_auto: bool) -> Vec<&'static str> { pub fn handle_parse(matches: &clap::ArgMatches) -> CommandResult { let lockfiles = config::lockfiles(matches, phylum_project::get_current_project().as_ref())?; - let mut pkgs: Vec = Vec::new(); + let mut pkgs: Vec = Vec::new(); for lockfile in lockfiles { - pkgs.extend(parse_lockfile(lockfile.path, Some(&lockfile.lockfile_type))?.packages); + let parsed_lockfile = parse_lockfile(lockfile.path, Some(&lockfile.lockfile_type))?; + pkgs.extend(parsed_lockfile.into_iter()); } serde_json::to_writer_pretty(&mut io::stdout(), &pkgs)?; diff --git a/cli/tests/end_to_end/extension.rs b/cli/tests/end_to_end/extension.rs index a97f1e67c..3f37a6a0a 100644 --- a/cli/tests/end_to_end/extension.rs +++ b/cli/tests/end_to_end/extension.rs @@ -152,7 +152,8 @@ pub async fn get_job_status() { let project = create_project().await; let analyze = format!( " - const pkg = {{ name: 'typescript', version: '4.7.4', type: 'npm' }}; + const pkg = {{ name: 'typescript', version: '4.7.4', type: 'npm', lockfile: \ + 'package-lock.json' }}; const jobId = await PhylumApi.analyze([pkg], {project:?}); console.log(await PhylumApi.getJobStatus(jobId));" ); diff --git a/extensions/CHANGELOG.md b/extensions/CHANGELOG.md index 44581b15f..e34c19ee3 100644 --- a/extensions/CHANGELOG.md +++ b/extensions/CHANGELOG.md @@ -8,7 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Added + - Added `getJobStatusRaw` and `checkPackagesRaw` APIs for detailed analysis results +- Allow `lockfile` in packages passed to `PhylumApi.analyze()` ## 5.5.0 - 2023-07-18 diff --git a/extensions/phylum.ts b/extensions/phylum.ts index 2def05b1d..abeffd4fa 100644 --- a/extensions/phylum.ts +++ b/extensions/phylum.ts @@ -5,6 +5,7 @@ export type Package = { name: string; version: string; type: string; + lockfile?: string; }; export type Lockfile = { @@ -110,7 +111,7 @@ export class PhylumApi { * * ``` * [ - * { name: "accepts", version: "1.3.8", type: "npm" }, + * { name: "accepts", version: "1.3.8", type: "npm", lockfile: "/path/to/lockfile" }, * { name: "ms", version: "2.0.0", type: "npm" }, * { name: "negotiator", version: "0.6.3", type: "npm" }, * { name: "ms", version: "2.1.3", type: "npm" } @@ -118,6 +119,7 @@ export class PhylumApi { * ``` * * Accepted package types are "npm", "pypi", "maven", "rubygems", "nuget", "cargo", and "golang" + * The `lockfile` is an optional string used to represent the path to the lockfile. * * @param packages - List of packages to analyze * @param project - Project name. If undefined, the `.phylum_project` file will be used diff --git a/lockfile/src/javascript.rs b/lockfile/src/javascript.rs index d75e5fc24..c9d79f58f 100644 --- a/lockfile/src/javascript.rs +++ b/lockfile/src/javascript.rs @@ -528,16 +528,20 @@ mod tests { }, Package { name: "ethereumjs-abi".into(), - version: PackageVersion::Git("https://github.com/ethereumjs/ethereumjs-abi.git\ + version: PackageVersion::Git( + "https://github.com/ethereumjs/ethereumjs-abi.git\ #commit=ee3994657fa7a427238e6ba92a84d0b529bbcde0" - .into()), + .into(), + ), package_type: PackageType::Npm, }, Package { name: "@me/remote-patch".into(), - version: PackageVersion::Git("ssh://git@github.com:phylum/remote-patch\ + version: PackageVersion::Git( + "ssh://git@github.com:phylum/remote-patch\ #commit=d854c43ea177d1faeea56189249fff8c24a764bd" - .into()), + .into(), + ), package_type: PackageType::Npm, }, Package {