From 5b1320050fad07462c0a2ee6fcab2bf545b50159 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 20:46:12 +0000 Subject: [PATCH 01/43] Added sorbet typecheck under python module for setup_file_parser.rb , setup_file_sanitizer.rb, language_version_manager.rb,pipenv_runner.rb,index_finder.rb --- .../dependabot/python/file_parser/setup_file_parser.rb | 8 +++++++- .../python/file_updater/setup_file_sanitizer.rb | 3 +++ python/lib/dependabot/python/language_version_manager.rb | 6 ++++++ python/lib/dependabot/python/pipenv_runner.rb | 8 ++++++++ .../lib/dependabot/python/update_checker/index_finder.rb | 6 ++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index a5336e20c2..4413311026 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -8,11 +8,13 @@ require "dependabot/python/file_parser" require "dependabot/python/native_helpers" require "dependabot/python/name_normaliser" +require "sorbet-runtime" module Dependabot module Python class FileParser class SetupFileParser + extend T::Sig INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m @@ -98,6 +100,7 @@ def parsed_sanitized_setup_file [] end + sig { params(requirements: T.untyped).void } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -140,19 +143,21 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - + sig { params(regex: Regexp).returns(T.untyped) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end + sig { params(regex: Regexp).returns(T.untyped) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) "{#{mch.post_match[0..closing_bracket_index(mch.post_match, '{')]}" end + sig { params(string: String, bracket: String).returns(T.untyped) } def closing_bracket_index(string, bracket) closes_required = 1 @@ -165,6 +170,7 @@ def closing_bracket_index(string, bracket) 0 end + sig { params(name: String, extras: T::Array[String]).returns(T.untyped) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index de74f69fc9..75d1d1918c 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -3,6 +3,7 @@ require "dependabot/python/file_updater" require "dependabot/python/file_parser/setup_file_parser" +require "sorbet-runtime" module Dependabot module Python @@ -10,6 +11,7 @@ class FileUpdater # Take a setup.py, parses it (carefully!) and then create a new, clean # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer + extend T::Sig def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg @@ -86,6 +88,7 @@ def parsed_setup_file ).dependency_set end + sig{ returns(String) } def package_name content = setup_file.content match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 2bd4eb1e26..970c41731e 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -3,10 +3,12 @@ require "dependabot/logger" require "dependabot/python/version" +require "sorbet-runtime" module Dependabot module Python class LanguageVersionManager + extend T::Sig # This list must match the versions specified at the top of `python/Dockerfile` PRE_INSTALLED_PYTHON_VERSIONS = %w( 3.13.1 @@ -21,6 +23,7 @@ def initialize(python_requirement_parser:) @python_requirement_parser = python_requirement_parser end + sig { returns(T.nilable(String)) } def install_required_python # The leading space is important in the version check return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_major_minor}.") @@ -46,6 +49,7 @@ def python_version @python_version ||= python_version_from_supported_versions end + sig { returns(String) } def python_requirement_string if user_specified_python_version if user_specified_python_version.start_with?(/\d/) @@ -59,6 +63,7 @@ def python_requirement_string end end + sig { returns(String) } def python_version_from_supported_versions requirement_string = python_requirement_string @@ -88,6 +93,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end + sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index f59f0ec2bd..f242491a2a 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -4,16 +4,20 @@ require "dependabot/shared_helpers" require "dependabot/python/file_parser" require "json" +require "sorbet-runtime" module Dependabot module Python class PipenvRunner + extend T::Sig + def initialize(dependency:, lockfile:, language_version_manager:) @dependency = dependency @lockfile = lockfile @language_version_manager = language_version_manager end + sig { params(constraint: String).void } def run_upgrade(constraint) constraint = "" if constraint == "*" command = "pyenv exec pipenv upgrade --verbose #{dependency_name}#{constraint}" @@ -22,6 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end + sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -30,6 +35,7 @@ def run_upgrade_and_fetch_version(constraint) fetch_version_from_parsed_lockfile(updated_lockfile) end + sig { params(command: String, fingerprint: T.nilable(String)).void } def run(command, fingerprint: nil) run_command( "pyenv local #{language_version_manager.python_major_minor}", @@ -45,6 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager + sig { params(updated_lockfile: []).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -52,6 +59,7 @@ def fetch_version_from_parsed_lockfile(updated_lockfile) &.gsub(/^==/, "") end + sig { params(command: String, fingerprint: T.nilable(String)).void } def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 5f3e6443dd..3a109cb021 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -9,6 +9,7 @@ module Dependabot module Python class UpdateChecker class IndexFinder + extend T::Sig PYPI_BASE_URL = "https://pypi.org/simple/" ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/ @@ -55,6 +56,7 @@ def main_index_url clean_check_and_remove_environment_variables(url) end + sig { returns({ main: NilClass, extra: [] }) } def requirement_file_index_urls urls = { main: nil, extra: [] } @@ -74,6 +76,7 @@ def requirement_file_index_urls urls end + sig { returns({ main: NilClass, extra: [] }) } def pip_conf_index_urls urls = { main: nil, extra: [] } @@ -90,6 +93,7 @@ def pip_conf_index_urls urls end + sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pipfile_index_urls urls = { main: nil, extra: [] } @@ -109,6 +113,7 @@ def pipfile_index_urls urls end + sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pyproject_index_urls urls = { main: nil, extra: [] } @@ -139,6 +144,7 @@ def pyproject_index_urls urls end + sig { returns({ main: NilClass, extra: [] }) } def config_variable_index_urls urls = { main: nil, extra: [] } From 86a31e6d1cbada7c1f147eec948621640efc66a2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 21:01:59 +0000 Subject: [PATCH 02/43] Fixed issue in index_finder --- python/lib/dependabot/python/update_checker/index_finder.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 3a109cb021..2efc34dc06 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -93,7 +93,6 @@ def pip_conf_index_urls urls end - sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pipfile_index_urls urls = { main: nil, extra: [] } @@ -113,7 +112,6 @@ def pipfile_index_urls urls end - sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pyproject_index_urls urls = { main: nil, extra: [] } From 8df8696b26a19727a4fdfe7b899d788c85d2c9ac Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 21:07:13 +0000 Subject: [PATCH 03/43] Fixed issue in setupfilesanitizer --- .../lib/dependabot/python/file_updater/setup_file_sanitizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 75d1d1918c..d89dc16020 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -88,7 +88,7 @@ def parsed_setup_file ).dependency_set end - sig{ returns(String) } + sig { returns(String) } def package_name content = setup_file.content match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) From 2057befbb9f84699445e785212304550269bcffa Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 23:57:22 +0000 Subject: [PATCH 04/43] Included TypeCheck for one more file --- .../lib/dependabot/python/file_parser/setup_file_parser.rb | 7 +++---- .../dependabot/python/file_updater/pyproject_preparer.rb | 5 +++++ .../lib/dependabot/python/update_checker/index_finder.rb | 3 --- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 4413311026..1b841af615 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -100,7 +100,6 @@ def parsed_sanitized_setup_file [] end - sig { params(requirements: T.untyped).void } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -150,14 +149,14 @@ def get_regexed_req_array(regex) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - sig { params(regex: Regexp).returns(T.untyped) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) "{#{mch.post_match[0..closing_bracket_index(mch.post_match, '{')]}" end - sig { params(string: String, bracket: String).returns(T.untyped) } + sig { params(string: String, bracket: String).returns(Integer) } def closing_bracket_index(string, bracket) closes_required = 1 @@ -170,7 +169,7 @@ def closing_bracket_index(string, bracket) 0 end - sig { params(name: String, extras: T::Array[String]).returns(T.untyped) } + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 17254e734b..2e12c190a6 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -14,6 +14,7 @@ module Dependabot module Python class FileUpdater class PyprojectPreparer + extend T::Sig def initialize(pyproject_content:, lockfile: nil) @pyproject_content = pyproject_content @lockfile = lockfile @@ -21,6 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. + sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } @@ -37,6 +39,7 @@ def add_auth_env_vars(credentials) end end + sig { params(requirement: String).void } def update_python_requirement(requirement) pyproject_object = TomlRB.parse(@pyproject_content) if (python_specification = pyproject_object.dig("tool", "poetry", "dependencies", "python")) @@ -48,6 +51,7 @@ def update_python_requirement(requirement) TomlRB.dump(pyproject_object) end + sig { returns(String) } def sanitize # {{ name }} syntax not allowed pyproject_content @@ -111,6 +115,7 @@ def locked_details(dep_name) .find { |d| d["name"] == normalise(dep_name) } end + sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 2efc34dc06..3f84e1f1f0 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -9,7 +9,6 @@ module Dependabot module Python class UpdateChecker class IndexFinder - extend T::Sig PYPI_BASE_URL = "https://pypi.org/simple/" ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/ @@ -56,7 +55,6 @@ def main_index_url clean_check_and_remove_environment_variables(url) end - sig { returns({ main: NilClass, extra: [] }) } def requirement_file_index_urls urls = { main: nil, extra: [] } @@ -76,7 +74,6 @@ def requirement_file_index_urls urls end - sig { returns({ main: NilClass, extra: [] }) } def pip_conf_index_urls urls = { main: nil, extra: [] } From a43fc93b09f91f4bc17321a511f8c1e885bf7f6a Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 23:59:23 +0000 Subject: [PATCH 05/43] Remvoed from index finder file --- python/lib/dependabot/python/update_checker/index_finder.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 3f84e1f1f0..5f3e6443dd 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -139,7 +139,6 @@ def pyproject_index_urls urls end - sig { returns({ main: NilClass, extra: [] }) } def config_variable_index_urls urls = { main: nil, extra: [] } From 271aeb92cb6807cd82ad3426a67d8a629f764773 Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 00:22:24 +0000 Subject: [PATCH 06/43] Removed from pyproject_preparer --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 1 - python/lib/dependabot/python/pipenv_runner.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 2e12c190a6..5b52ad6236 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -39,7 +39,6 @@ def add_auth_env_vars(credentials) end end - sig { params(requirement: String).void } def update_python_requirement(requirement) pyproject_object = TomlRB.parse(@pyproject_content) if (python_specification = pyproject_object.dig("tool", "poetry", "dependencies", "python")) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index f242491a2a..2689afaf51 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -51,7 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - sig { params(updated_lockfile: []).void } + sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} From ecaa4d2ec32257bc3622e7b9a11b53267f68ac26 Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 00:52:25 +0000 Subject: [PATCH 07/43] Commented since return type is not correct --- .../dependabot/python/file_parser/setup_file_parser.rb | 5 +++-- .../dependabot/python/file_updater/pyproject_preparer.rb | 2 +- python/lib/dependabot/python/language_version_manager.rb | 2 +- python/lib/dependabot/python/pipenv_runner.rb | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 1b841af615..403f0ae87a 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -142,14 +142,15 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - sig { params(regex: Regexp).returns(T.untyped) } + + # sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - sig { params(regex: Regexp).returns(T.nilable(String)) } + # sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 5b52ad6236..c43a1c71e2 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - sig { params(credentials: T.nilable(Hash)).void } + # sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 970c41731e..e6a38df299 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -93,7 +93,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end - sig { params(requirements: T.untyped).returns(T.nilable(String)) } + # sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index 2689afaf51..a7031faa0c 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,7 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).void } + # sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -35,7 +35,7 @@ def run_upgrade_and_fetch_version(constraint) fetch_version_from_parsed_lockfile(updated_lockfile) end - sig { params(command: String, fingerprint: T.nilable(String)).void } + sig { params(command: String, fingerprint: T.nilable(String)).returns(String) } def run(command, fingerprint: nil) run_command( "pyenv local #{language_version_manager.python_major_minor}", @@ -51,7 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - sig { params(updated_lockfile: Array).void } + # sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -59,7 +59,7 @@ def fetch_version_from_parsed_lockfile(updated_lockfile) &.gsub(/^==/, "") end - sig { params(command: String, fingerprint: T.nilable(String)).void } + sig { params(command: String, fingerprint: T.nilable(String)).returns(String) } def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end From 4c981affa7120beefd325750bc38c8118007bd8c Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 01:09:19 +0000 Subject: [PATCH 08/43] Added TypeCheck back to setup_file_parser --- python/lib/dependabot/python/file_parser/setup_file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 403f0ae87a..2e634bdb6c 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -143,14 +143,14 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - # sig { params(regex: Regexp).returns(T.nilable(String)) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - # sig { params(regex: Regexp).returns(T.nilable(String)) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) From a383d86a4bfae6f282cf6409a5db9b1e11c0c07d Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 01:38:08 +0000 Subject: [PATCH 09/43] Added TypeCheck pyproject_preparer.rb --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 2 +- python/lib/dependabot/python/language_version_manager.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index c43a1c71e2..5b52ad6236 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - # sig { params(credentials: T.nilable(Hash)).void } + sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index e6a38df299..3ec07661ea 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -93,7 +93,6 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end - # sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) From cfc50448a11a46873a31fdc3436d6a5a2dab200f Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 02:28:33 +0000 Subject: [PATCH 10/43] Removed commented code --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 1 - python/lib/dependabot/python/pipenv_runner.rb | 2 -- 2 files changed, 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 5b52ad6236..a3f0a09afc 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,6 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index a7031faa0c..83502aa812 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,7 +26,6 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - # sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -51,7 +50,6 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - # sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} From a8d8ad597d8015332eaf241a0cbc30316b9f1241 Mon Sep 17 00:00:00 2001 From: Randhir Date: Mon, 13 Jan 2025 22:25:30 +0000 Subject: [PATCH 11/43] Added TypeCheck as strict for file_parser --- python/lib/dependabot/python/file_parser.rb | 90 ++++++++++++------- python/lib/dependabot/python/pipenv_runner.rb | 1 + 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 0de97a942d..62c97c88d1 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/dependency" @@ -18,12 +18,13 @@ module Dependabot module Python class FileParser < Dependabot::FileParsers::Base + extend T::Sig require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/pyproject_files_parser" require_relative "file_parser/setup_file_parser" require_relative "file_parser/python_requirement_parser" - DEPENDENCY_GROUP_KEYS = [ + DEPENDENCY_GROUP_KEYS = T.let([ { pipfile: "packages", lockfile: "default" @@ -32,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze + ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -43,6 +44,7 @@ class FileParser < Dependabot::FileParsers::Base # in any way if any metric collection exception start happening UNDETECTED_PACKAGE_MANAGER_VERSION = "0.0" + sig { override.returns(T::Array[Dependabot::Dependency]) } def parse # TODO: setup.py from external dependencies is evaluated. Provide guards before removing this. raise Dependabot::UnexpectedExternalCode if @reject_external_code @@ -57,7 +59,7 @@ def parse dependency_set.dependencies end - sig { returns(Ecosystem) } + sig { override.returns(Ecosystem) } def ecosystem @ecosystem ||= T.let( Ecosystem.new( @@ -71,18 +73,16 @@ def ecosystem private + sig { returns(Dependabot::Python::LanguageVersionManager) } def language_version_manager - @language_version_manager ||= - LanguageVersionManager.new( - python_requirement_parser: python_requirement_parser - ) + @language_version_manager ||= T.let(LanguageVersionManager.new(python_requirement_parser: + python_requirement_parser), T.nilable(LanguageVersionManager)) end + sig { returns(Dependabot::Python::FileParser::PythonRequirementParser) } def python_requirement_parser - @python_requirement_parser ||= - FileParser::PythonRequirementParser.new( - dependency_files: dependency_files - ) + @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: dependency_files), + T.nilable(FileParser::PythonRequirementParser)) end sig { returns(Ecosystem::VersionManager) } @@ -91,7 +91,7 @@ def package_manager Dependabot.logger.info("Detected package manager : #{detected_package_manager.name}") end - @package_manager ||= detected_package_manager + @package_manager ||= T.let(detected_package_manager, T.nilable(Dependabot::Ecosystem::VersionManager)) end sig { returns(Ecosystem::VersionManager) } @@ -188,7 +188,7 @@ def package_manager_version(package_manager) end # setup python local setup on file parser stage - sig { void } + sig { returns(T.nilable(String)) } def setup_python_environment language_version_manager.install_required_python @@ -231,24 +231,24 @@ def language ) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end + sig { returns(T.nilable(PipfileFilesParser)) } def pipenv_dependencies - @pipenv_dependencies ||= - PipfileFilesParser - .new(dependency_files: dependency_files) - .dependency_set + @pipenv_dependencies ||= T.let(PipfileFilesParser.new(dependency_files: + dependency_files).dependency_set, T.nilable(PipfileFilesParser)) end + sig { returns(T.nilable(PyprojectFilesParser))} def pyproject_file_dependencies - @pyproject_file_dependencies ||= - PyprojectFilesParser - .new(dependency_files: dependency_files) - .dependency_set + @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: + dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) end + sig { returns(DependencySet)} def requirement_dependencies dependencies = DependencySet.new parsed_requirement_files.each do |dep| @@ -286,13 +286,15 @@ def requirement_dependencies dependencies end + sig { params(name: String, version: String).returns(T::Boolean) } def old_pyyaml?(name, version) - major_version = version&.split(".")&.first + major_version = version.split(".").first return false unless major_version name == "pyyaml" && major_version < "6" end + sig { params(filename: String).returns(T::Array[String]) } def group_from_filename(filename) if filename.include?("dev") then ["dev-dependencies"] else @@ -300,6 +302,7 @@ def group_from_filename(filename) end end + sig { params(dep: T.untyped).returns(T::Boolean) } def blocking_marker?(dep) return false if dep["markers"] == "None" @@ -316,6 +319,7 @@ def blocking_marker?(dep) end end + sig { params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) } def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -356,13 +360,15 @@ def evaluate_condition(condition, python_version) end end + sig { void } def setup_file_dependencies - @setup_file_dependencies ||= + @setup_file_dependencies ||= T.let( SetupFileParser .new(dependency_files: dependency_files) - .dependency_set + .dependency_set, T.untyped) end + sig { returns(T.untyped) } def parsed_requirement_files SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files @@ -383,6 +389,7 @@ def parsed_requirement_files raise Dependabot::DependencyFileNotEvaluatable, e.message end + sig { params(requirements: T.untyped).returns(T.untyped) } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -393,18 +400,22 @@ def check_requirements(requirements) end end + sig { returns(T::Boolean) } def pipcompile_in_file requirement_files.any? { |f| f.name.end_with?(PipCompilePackageManager::MANIFEST_FILENAME) } end + sig { returns(T::Boolean) } def pipenv_files dependency_files.any? { |f| f.name == PipenvPackageManager::LOCKFILE_FILENAME } end + sig { returns(T.nilable(TrueClass)) } def poetry_files true if get_original_file(PoetryPackageManager::LOCKFILE_NAME) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def write_temporary_dependency_files dependency_files .reject { |f| f.name == ".python-version" } @@ -415,6 +426,7 @@ def write_temporary_dependency_files end end + sig { params(file: T.untyped).returns(T.untyped) } def remove_imports(file) return file.content if file.path.end_with?(".tar.gz", ".whl", ".zip") @@ -424,10 +436,12 @@ def remove_imports(file) .join end + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras = []) NameNormaliser.normalise_including_extras(name, extras) end + sig { override.void } def check_required_files filenames = dependency_files.map(&:name) return if filenames.any? { |name| name.end_with?(".txt", ".in") } @@ -439,37 +453,45 @@ def check_required_files raise "Missing required files!" end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pipfile - @pipfile ||= get_original_file("Pipfile") + @pipfile ||= T.let(get_original_file("Pipfile"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pipfile_lock - @pipfile_lock ||= get_original_file("Pipfile.lock") + @pipfile_lock ||= T.let(get_original_file("Pipfile.lock"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pyproject - @pyproject ||= get_original_file("pyproject.toml") + @pyproject ||= T.let(get_original_file("pyproject.toml"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def poetry_lock - @poetry_lock ||= get_original_file("poetry.lock") + @poetry_lock ||= T.let(get_original_file("poetry.lock"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile))} def setup_file - @setup_file ||= get_original_file("setup.py") + @setup_file ||= T.let(get_original_file("setup.py"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def setup_cfg_file - @setup_cfg_file ||= get_original_file("setup.cfg") + @setup_cfg_file ||= T.let(get_original_file("setup.cfg"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def pip_compile_files - @pip_compile_files ||= - dependency_files.select { |f| f.name.end_with?(".in") } + @pip_compile_files ||= T.let(dependency_files.select { |f| f.name.end_with?(".in") }, T.untyped) end + sig { returns(Dependabot::Python::PipCompileFileMatcher) } def pip_compile_file_matcher - @pip_compile_file_matcher ||= PipCompileFileMatcher.new(pip_compile_files) + @pip_compile_file_matcher ||= T.let(PipCompileFileMatcher.new(pip_compile_files), + T.nilable(Dependabot::Python::PipCompileFileMatcher)) end end end diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index 83502aa812..d65ab0ff9f 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,6 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end + sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) From 604024fa38b8d72d897dd926f6ea6db962c77190 Mon Sep 17 00:00:00 2001 From: Randhir Date: Mon, 13 Jan 2025 22:29:41 +0000 Subject: [PATCH 12/43] Added TypeCheck as strict for file_parser --- python/lib/dependabot/python/file_parser.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 62c97c88d1..46b6c6e0c8 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -242,13 +242,13 @@ def pipenv_dependencies dependency_files).dependency_set, T.nilable(PipfileFilesParser)) end - sig { returns(T.nilable(PyprojectFilesParser))} + sig { returns(T.nilable(PyprojectFilesParser)) } def pyproject_file_dependencies @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) end - sig { returns(DependencySet)} + sig { returns(DependencySet) } def requirement_dependencies dependencies = DependencySet.new parsed_requirement_files.each do |dep| @@ -319,7 +319,9 @@ def blocking_marker?(dep) end end - sig { params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) } + sig do + params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + end def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -341,6 +343,10 @@ def marker_satisfied?(marker, python_version) result end + sig do + params(condition: T.untyped, + python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + end def evaluate_condition(condition, python_version) operator, version = condition.match(/([<>=!]=?)\s*"?([\d.]+)"?/)&.captures @@ -365,7 +371,8 @@ def setup_file_dependencies @setup_file_dependencies ||= T.let( SetupFileParser .new(dependency_files: dependency_files) - .dependency_set, T.untyped) + .dependency_set, T.untyped + ) end sig { returns(T.untyped) } @@ -473,7 +480,7 @@ def poetry_lock @poetry_lock ||= T.let(get_original_file("poetry.lock"), T.nilable(Dependabot::DependencyFile)) end - sig { returns(T.nilable(Dependabot::DependencyFile))} + sig { returns(T.nilable(Dependabot::DependencyFile)) } def setup_file @setup_file ||= T.let(get_original_file("setup.py"), T.nilable(Dependabot::DependencyFile)) end @@ -483,7 +490,7 @@ def setup_cfg_file @setup_cfg_file ||= T.let(get_original_file("setup.cfg"), T.nilable(Dependabot::DependencyFile)) end - sig { returns(T::Array[Dependabot::DependencyFile]) } + sig { returns(T::Array[Dependabot::Python::Requirement]) } def pip_compile_files @pip_compile_files ||= T.let(dependency_files.select { |f| f.name.end_with?(".in") }, T.untyped) end From 711a96f0154edc40b287d68f7bb39b3db19355c2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 14:48:01 +0000 Subject: [PATCH 13/43] Removed nilable to fix the issue --- python/lib/dependabot/python/file_parser.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 46b6c6e0c8..28679db25c 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) + ].freeze, T::Array[T::Hash[String, String]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -320,7 +320,7 @@ def blocking_marker?(dep) end sig do - params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + params(marker: T.untyped, python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean) end def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -345,7 +345,7 @@ def marker_satisfied?(marker, python_version) sig do params(condition: T.untyped, - python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean) end def evaluate_condition(condition, python_version) operator, version = condition.match(/([<>=!]=?)\s*"?([\d.]+)"?/)&.captures From 44d27ccf781db205f201ff1409291c04a2e80a05 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:00:06 +0000 Subject: [PATCH 14/43] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 28679db25c..f24f052e32 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[String, String]]) + ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError From e0e0646f3e75fc1482a346e9253fdf8111ed0f14 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:05:26 +0000 Subject: [PATCH 15/43] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index f24f052e32..80d07207fd 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -17,7 +17,7 @@ module Dependabot module Python - class FileParser < Dependabot::FileParsers::Base + class FileParser < Dependabot::FileParsers::Base # rubocop:disable Metrics/ClassLength extend T::Sig require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/pyproject_files_parser" From d9320d2eaec95f2ca017072b53a503005245400f Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:08:27 +0000 Subject: [PATCH 16/43] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 80d07207fd..37724bac88 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -81,8 +81,8 @@ def language_version_manager sig { returns(Dependabot::Python::FileParser::PythonRequirementParser) } def python_requirement_parser - @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: dependency_files), - T.nilable(FileParser::PythonRequirementParser)) + @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: + dependency_files), T.nilable(FileParser::PythonRequirementParser)) end sig { returns(Ecosystem::VersionManager) } From 85d266a81c8f93fed4f3cb4b492125b711fc8ac8 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 19:21:03 +0000 Subject: [PATCH 17/43] corrected return type --- python/lib/dependabot/python/file_parser.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 37724bac88..4c150f9481 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,14 +198,13 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).void } + sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found - return true if version.match?(/^\d+(?:\.\d+)*$/) - Dependabot.logger.warn( "Detected #{package_manager} with malformed version #{version}" ) + true if version.match?(/^\d+(?:\.\d+)*$/) end sig { returns(String) } @@ -366,13 +365,10 @@ def evaluate_condition(condition, python_version) end end - sig { void } + sig { returns(T.nilable(SetupFileParser)) } def setup_file_dependencies - @setup_file_dependencies ||= T.let( - SetupFileParser - .new(dependency_files: dependency_files) - .dependency_set, T.untyped - ) + @setup_file_dependencies ||= T.let(SetupFileParser.new(dependency_files: dependency_files) + .dependency_set, T.nilable(SetupFileParser)) end sig { returns(T.untyped) } From 2955a6427746b58b7e8d18eb4b7ebfce6ed7621c Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 19:46:40 +0000 Subject: [PATCH 18/43] Corrected returntype to pass end to end error --- python/lib/dependabot/python/file_parser.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 4c150f9481..811e05bc70 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,13 +198,14 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } + sig { params(package_manager: String, version: String).void } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found + return true if version.match?(/^\d+(?:\.\d+)*$/) + Dependabot.logger.warn( "Detected #{package_manager} with malformed version #{version}" ) - true if version.match?(/^\d+(?:\.\d+)*$/) end sig { returns(String) } @@ -285,9 +286,9 @@ def requirement_dependencies dependencies end - sig { params(name: String, version: String).returns(T::Boolean) } + sig { params(name: T.nilable(String), version: T.nilable(String)).returns(T::Boolean) } def old_pyyaml?(name, version) - major_version = version.split(".").first + major_version = version&.split(".")&.first return false unless major_version name == "pyyaml" && major_version < "6" @@ -444,7 +445,7 @@ def normalised_name(name, extras = []) NameNormaliser.normalise_including_extras(name, extras) end - sig { override.void } + sig { override.returns(T.untyped) } def check_required_files filenames = dependency_files.map(&:name) return if filenames.any? { |name| name.end_with?(".txt", ".in") } From 4f3843941b758d03814445ef3104741eba2bff6b Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:05:04 +0000 Subject: [PATCH 19/43] Changed requirementfile method return type --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 811e05bc70..3356c5a76f 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -231,7 +231,7 @@ def language ) end - sig { returns(T::Array[Dependabot::DependencyFile]) } + sig { returns(T.untyped) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end From b847188e81d16754a71e784757b31833ea00df27 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:19:12 +0000 Subject: [PATCH 20/43] Changed return type to dependabotset --- python/lib/dependabot/python/file_parser.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 3356c5a76f..2a8f478ac0 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -231,21 +231,21 @@ def language ) end - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end - sig { returns(T.nilable(PipfileFilesParser)) } + sig { returns(DependencySet) } def pipenv_dependencies @pipenv_dependencies ||= T.let(PipfileFilesParser.new(dependency_files: - dependency_files).dependency_set, T.nilable(PipfileFilesParser)) + dependency_files).dependency_set, T.nilable(DependencySet)) end - sig { returns(T.nilable(PyprojectFilesParser)) } + sig { returns(DependencySet) } def pyproject_file_dependencies @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: - dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) + dependency_files).dependency_set, T.nilable(DependencySet)) end sig { returns(DependencySet) } From e03d781fcc97b1e4636ef99643c8134626f310a2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:34:44 +0000 Subject: [PATCH 21/43] Changed return type to dependencySet --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 2a8f478ac0..8685eac848 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -366,10 +366,10 @@ def evaluate_condition(condition, python_version) end end - sig { returns(T.nilable(SetupFileParser)) } + sig { returns(DependencySet) } def setup_file_dependencies @setup_file_dependencies ||= T.let(SetupFileParser.new(dependency_files: dependency_files) - .dependency_set, T.nilable(SetupFileParser)) + .dependency_set, T.nilable(DependencySet)) end sig { returns(T.untyped) } From 12d3f3dbaf05659b55271fba31e18a58de86ff5f Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 21:07:59 +0000 Subject: [PATCH 22/43] Corrected log_if_version_malformed --- python/lib/dependabot/python/file_parser.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 8685eac848..9d0940298b 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,14 +198,15 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).void } + sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found - return true if version.match?(/^\d+(?:\.\d+)*$/) - - Dependabot.logger.warn( - "Detected #{package_manager} with malformed version #{version}" - ) + if version.match?(/^\d+(?:\.\d+)*$/) + true + else + Dependabot.logger.warn("Detected #{package_manager} with malformed version #{version}") + false + end end sig { returns(String) } From ee979f992dd390545e914ee8107b972c0674a77d Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 22:08:33 +0000 Subject: [PATCH 23/43] Changed return type in pipenv_runner --- python/lib/dependabot/python/pipenv_runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index d65ab0ff9f..bb7fc6857d 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -17,7 +17,7 @@ def initialize(dependency:, lockfile:, language_version_manager:) @language_version_manager = language_version_manager end - sig { params(constraint: String).void } + sig { params(constraint: String).returns(String) } def run_upgrade(constraint) constraint = "" if constraint == "*" command = "pyenv exec pipenv upgrade --verbose #{dependency_name}#{constraint}" @@ -26,7 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).void } + sig { params(constraint: String).returns(T.untyped) } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) From 90772e10882b96f33074d851b3738b48cf86ffee Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 00:01:03 +0000 Subject: [PATCH 24/43] Corrected return type --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 9d0940298b..9ab46bb3fa 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base # rubocop:disable Metrics/Class pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) + ].freeze, T::Array[T::Hash[Symbol, String]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -198,7 +198,7 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } + sig { params(package_manager: String, version: String).returns(T::Boolean) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found if version.match?(/^\d+(?:\.\d+)*$/) From db2df60ead470b6ae8c60fcc3389fe9f408db14e Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 19:22:46 +0000 Subject: [PATCH 25/43] Added TypeCheck as strict for pipenv_runner --- python/lib/dependabot/python/pipenv_runner.rb | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index bb7fc6857d..a526c72a54 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/shared_helpers" @@ -11,6 +11,14 @@ module Python class PipenvRunner extend T::Sig + sig do + params( + dependency: Dependabot::Dependency, + lockfile: DependencyFile, + language_version_manager: LanguageVersionManager + ) + .void + end def initialize(dependency:, lockfile:, language_version_manager:) @dependency = dependency @lockfile = lockfile @@ -26,7 +34,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).returns(T.untyped) } + sig { params(constraint: String).returns(String) } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -47,10 +55,14 @@ def run(command, fingerprint: nil) private + sig { returns(Dependabot::Dependency) } attr_reader :dependency + sig { returns(DependencyFile) } attr_reader :lockfile + sig { returns(LanguageVersionManager) } attr_reader :language_version_manager + sig { params(updated_lockfile: T.untyped).returns(T.untyped) } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -63,21 +75,24 @@ def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end + sig { returns(String) } def lockfile_section if dependency.requirements.any? - dependency.requirements.first[:groups].first + T.must(dependency.requirements.first)[:groups].first else Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys| section = keys.fetch(:lockfile) - return section if JSON.parse(lockfile.content)[section].keys.any?(dependency_name) + return section if JSON.parse(T.must(lockfile.content))[section].keys.any?(dependency_name) end end end + sig { returns(String) } def dependency_name dependency.metadata[:original_name] || dependency.name end + sig { returns(T::Hash[String, String]) } def pipenv_env_variables { "PIPENV_YES" => "true", # Install new Python ver if needed From ed5a286db234e3cea8a89e05ec874c0c8a18c316 Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 19:42:53 +0000 Subject: [PATCH 26/43] Changed param DependcyFile Nilable --- python/lib/dependabot/python/pipenv_runner.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index a526c72a54..de76e17631 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -14,7 +14,7 @@ class PipenvRunner sig do params( dependency: Dependabot::Dependency, - lockfile: DependencyFile, + lockfile: T.nilable(Dependabot::DependencyFile), language_version_manager: LanguageVersionManager ) .void @@ -57,7 +57,7 @@ def run(command, fingerprint: nil) sig { returns(Dependabot::Dependency) } attr_reader :dependency - sig { returns(DependencyFile) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } attr_reader :lockfile sig { returns(LanguageVersionManager) } attr_reader :language_version_manager @@ -82,7 +82,7 @@ def lockfile_section else Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys| section = keys.fetch(:lockfile) - return section if JSON.parse(T.must(lockfile.content))[section].keys.any?(dependency_name) + return section if JSON.parse(T.must(T.must(lockfile).content))[section].keys.any?(dependency_name) end end end From 4aab29fc611e32a17635c9e3d6d9dbc5782bde2a Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 17:08:22 +0000 Subject: [PATCH 27/43] Changed setup file typecheck as strict --- .../python/file_parser/setup_file_parser.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 2e634bdb6c..e14661241f 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/dependency" @@ -20,12 +20,14 @@ class SetupFileParser TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*\{/m - CLOSING_BRACKET = { "[" => "]", "{" => "}" }.freeze + CLOSING_BRACKET = T.let({ "[" => "]", "{" => "}" }.freeze, T.any(T.untyped, T.untyped)) + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def dependency_set dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -56,8 +58,10 @@ def dependency_set private + sig { returns(T.untyped) } attr_reader :dependency_files + sig { returns(T.untyped) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files @@ -79,6 +83,7 @@ def parsed_setup_file parsed_sanitized_setup_file end + sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } def parsed_sanitized_setup_file SharedHelpers.in_a_temporary_directory do write_sanitized_setup_file @@ -100,16 +105,18 @@ def parsed_sanitized_setup_file [] end + sig { params(requirements: T.untyped).returns(T.nilable(Python::Requirement)) } def check_requirements(requirements) - requirements.each do |dep| + requirements&.each do |dep| next unless dep["requirement"] - Python::Requirement.new(dep["requirement"].split(",")) + T.let(Python::Requirement.new(dep["requirement"].split(",")), Python::Requirement) rescue Gem::Requirement::BadRequirementError => e raise Dependabot::DependencyFileNotEvaluatable, e.message end end + sig { void } def write_temporary_dependency_files dependency_files .reject { |f| f.name == ".python-version" } @@ -125,6 +132,7 @@ def write_temporary_dependency_files # This sanitization is far from perfect (it will fail if any of the # entries are dynamic), but it is an alternative approach to the one # used in parser.py which sometimes succeeds when that has failed. + sig { void } def write_sanitized_setup_file install_requires = get_regexed_req_array(INSTALL_REQUIRES_REGEX) setup_requires = get_regexed_req_array(SETUP_REQUIRES_REGEX) @@ -175,6 +183,7 @@ def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end + sig { returns(T.untyped) } def setup_file dependency_files.find { |f| f.name == "setup.py" } end From 7cdf83b93376a5f873fc9c3d0fe0664e6cd4c844 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 19:15:46 +0000 Subject: [PATCH 28/43] Added files with typecheck strict --- .../file_parser/pyproject_files_parser.rb | 51 +++++++++++++++---- .../python/file_parser/setup_file_parser.rb | 4 +- .../file_updater/setup_file_sanitizer.rb | 33 ++++++++---- .../python/language_version_manager.rb | 15 ++++-- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 744d3457dc..65d4b43704 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "toml-rb" @@ -14,15 +14,18 @@ module Dependabot module Python class FileParser class PyprojectFilesParser + extend T::Sig POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def dependency_set dependency_set = Dependabot::FileParsers::Base::DependencySet.new @@ -34,8 +37,10 @@ def dependency_set private + sig { returns(T.untyped) } attr_reader :dependency_files + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def pyproject_dependencies if using_poetry? missing_keys = missing_poetry_keys @@ -54,10 +59,12 @@ def pyproject_dependencies end end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def poetry_dependencies - @poetry_dependencies ||= parse_poetry_dependencies + @poetry_dependencies ||= T.let(parse_poetry_dependencies, T.untyped) end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def parse_poetry_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -73,6 +80,7 @@ def parse_poetry_dependencies dependencies end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def pep621_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -107,6 +115,7 @@ def pep621_dependencies dependencies end + sig { params(type: T.untyped, deps_hash: T.untyped).returns(Dependabot::FileParsers::Base::DependencySet) } def parse_poetry_dependency_group(type, deps_hash) dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -126,11 +135,13 @@ def parse_poetry_dependency_group(type, deps_hash) dependencies end + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end # @param req can be an Array, Hash or String that represents the constraints for a dependency + sig { params(req: T.untyped, type: T.untyped).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } def parse_requirements_from(req, type) [req].flatten.compact.filter_map do |requirement| next if requirement.is_a?(Hash) && UNSUPPORTED_DEPENDENCY_TYPES.intersect?(requirement.keys) @@ -155,26 +166,31 @@ def parse_requirements_from(req, type) end end + sig { returns(T.nilable(T::Boolean)) } def using_poetry? !poetry_root.nil? end + sig { returns(T::Array[String]) } def missing_poetry_keys package_mode = poetry_root.fetch("package-mode", true) required_keys = package_mode ? %w(name version description authors) : [] required_keys.reject { |key| poetry_root.key?(key) } end + sig { returns(T.untyped) } def using_pep621? !parsed_pyproject.dig("project", "dependencies").nil? || !parsed_pyproject.dig("project", "optional-dependencies").nil? || !parsed_pyproject.dig("build-system", "requires").nil? end + sig { returns(T.untyped) } def poetry_root parsed_pyproject.dig("tool", "poetry") end + sig { returns(T.untyped) } def using_pdm? using_pep621? && pdm_lock end @@ -182,6 +198,7 @@ def using_pdm? # Create a DependencySet where each element has no requirement. Any # requirements will be added when combining the DependencySet with # other DependencySets. + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def lockfile_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -206,10 +223,13 @@ def lockfile_dependencies dependencies end + sig { returns(T::Array[T.nilable(String)]) } def production_dependency_names - @production_dependency_names ||= parse_production_dependency_names + @production_dependency_names ||= T.let(parse_production_dependency_names, + T.nilable(T::Array[T.nilable(String)])) end + sig { returns(T::Array[T.nilable(String)]) } def parse_production_dependency_names SharedHelpers.in_a_temporary_directory do File.write(pyproject.name, pyproject.content) @@ -232,6 +252,7 @@ def parse_production_dependency_names end end + sig { params(dep_name: T.untyped).returns(T.untyped) } def version_from_lockfile(dep_name) return unless parsed_lockfile @@ -240,6 +261,7 @@ def version_from_lockfile(dep_name) &.fetch("version", nil) end + sig { params(req: T.untyped).returns(T::Array[Dependabot::Python::Requirement]) } def check_requirements(req) requirement = req.is_a?(String) ? req : req["version"] Python::Requirement.requirements_array(requirement) @@ -247,31 +269,36 @@ def check_requirements(req) raise Dependabot::DependencyFileNotEvaluatable, e.message end + sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end + sig { returns(T.untyped) } def parsed_pyproject - @parsed_pyproject ||= TomlRB.parse(pyproject.content) + @parsed_pyproject ||= T.let(TomlRB.parse(pyproject.content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, pyproject.path end + sig { returns(T.untyped) } def parsed_poetry_lock - @parsed_poetry_lock ||= TomlRB.parse(poetry_lock.content) + @parsed_poetry_lock ||= T.let(TomlRB.parse(poetry_lock.content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, poetry_lock.path end + sig { returns(T.untyped) } def pyproject - @pyproject ||= - dependency_files.find { |f| f.name == "pyproject.toml" } + @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end + sig { returns(T.untyped) } def lockfile poetry_lock end + sig { returns(T.untyped) } def parsed_pep621_dependencies SharedHelpers.in_a_temporary_directory do write_temporary_pyproject @@ -284,24 +311,26 @@ def parsed_pep621_dependencies end end + sig { returns(Integer) } def write_temporary_pyproject path = pyproject.name FileUtils.mkdir_p(Pathname.new(path).dirname) File.write(path, pyproject.content) end + sig { returns(T.untyped) } def parsed_lockfile parsed_poetry_lock if poetry_lock end + sig { returns(T.untyped) } def poetry_lock - @poetry_lock ||= - dependency_files.find { |f| f.name == "poetry.lock" } + @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, T.untyped) end + sig { returns(T.untyped) } def pdm_lock - @pdm_lock ||= - dependency_files.find { |f| f.name == "pdm.lock" } + @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, T.untyped) end end end diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index e14661241f..3c27a8d456 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -105,12 +105,12 @@ def parsed_sanitized_setup_file [] end - sig { params(requirements: T.untyped).returns(T.nilable(Python::Requirement)) } + sig { params(requirements: T.untyped).returns(T.untyped) } def check_requirements(requirements) requirements&.each do |dep| next unless dep["requirement"] - T.let(Python::Requirement.new(dep["requirement"].split(",")), Python::Requirement) + Python::Requirement.new(dep["requirement"].split(",")) rescue Gem::Requirement::BadRequirementError => e raise Dependabot::DependencyFileNotEvaluatable, e.message end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index d89dc16020..2018c14b6f 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/python/file_updater" @@ -12,11 +12,14 @@ class FileUpdater # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer extend T::Sig + + sig { params(setup_file: DependencyFile, setup_cfg: T.untyped).void } def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg end + sig { returns(String) } def sanitized_content # The part of the setup.py that Pipenv cares about appears to be the # install_requires. A name and version are required by don't end up @@ -33,14 +36,19 @@ def sanitized_content private + sig { returns(DependencyFile) } attr_reader :setup_file + sig { returns(String) } attr_reader :setup_cfg + sig { returns(T::Boolean) } def include_pbr? setup_requires_array.any? { |d| d.start_with?("pbr") } end + sig { returns(T.untyped) } def install_requires_array + @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -50,7 +58,9 @@ def install_requires_array end end + sig { returns(T::Array[String]) } def setup_requires_array + @setup_requires_array = T.let(T.untyped, T.untyped) @setup_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -60,7 +70,9 @@ def setup_requires_array end end + sig { returns(T::Hash[T.untyped, T.untyped]) } def extras_require_hash + @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} @@ -78,20 +90,21 @@ def extras_require_hash end end + sig { returns(T.untyped) } def parsed_setup_file - @parsed_setup_file ||= - Python::FileParser::SetupFileParser.new( - dependency_files: [ - setup_file&.dup&.tap { |f| f.name = "setup.py" }, - setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } - ].compact - ).dependency_set + @parsed_setup_file ||= T.let(Python::FileParser::SetupFileParser.new( + dependency_files: [ + setup_file.dup.tap { |f| f.name = "setup.py" }, + setup_cfg.dup.tap { |f| f.name = "setup.cfg" } + ].compact + ) + .dependency_set, T.untyped) end - sig { returns(String) } + sig { returns(T.nilable(String)) } def package_name content = setup_file.content - match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) + match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) match ? match[:package_name] : "default_package_name" end end diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 3ec07661ea..d921b268ba 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/logger" @@ -19,6 +19,7 @@ class LanguageVersionManager 3.8.20 ).freeze + sig { params(python_requirement_parser: T.untyped).void } def initialize(python_requirement_parser:) @python_requirement_parser = python_requirement_parser end @@ -33,20 +34,23 @@ def install_required_python ) end + sig { returns(String) } def installed_version # Use `pyenv exec` to query the active Python version output, _status = SharedHelpers.run_shell_command("pyenv exec python --version") version = output.strip.split.last # Extract the version number (e.g., "3.13.1") - version + T.must(version) end + sig { returns(T.untyped) } def python_major_minor - @python_major_minor ||= T.must(Python::Version.new(python_version).segments[0..1]).join(".") + @python_major_minor ||= T.let(T.must(Python::Version.new(python_version).segments[0..1]).join("."), T.untyped) end + sig { returns(String) } def python_version - @python_version ||= python_version_from_supported_versions + @python_version ||= T.let(python_version_from_supported_versions, T.nilable(String)) end sig { returns(String) } @@ -81,10 +85,12 @@ def python_version_from_supported_versions raise ToolVersionNotSupported.new("Python", python_requirement_string, supported_versions) end + sig { returns(T.untyped) } def user_specified_python_version @python_requirement_parser.user_specified_requirements.first end + sig { returns(T.nilable(String)) } def python_version_matching_imputed_requirements compiled_file_python_requirement_markers = @python_requirement_parser.imputed_requirements.map do |r| @@ -93,6 +99,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end + sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) From 2a539e621d8d93a235770c7216d1ad6242c9e8fa Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:17:34 +0000 Subject: [PATCH 29/43] Removed typecheck form pyproject_preparer file --- .../lib/dependabot/python/file_updater/pyproject_preparer.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index a3f0a09afc..17254e734b 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -14,7 +14,6 @@ module Dependabot module Python class FileUpdater class PyprojectPreparer - extend T::Sig def initialize(pyproject_content:, lockfile: nil) @pyproject_content = pyproject_content @lockfile = lockfile @@ -49,7 +48,6 @@ def update_python_requirement(requirement) TomlRB.dump(pyproject_object) end - sig { returns(String) } def sanitize # {{ name }} syntax not allowed pyproject_content @@ -113,7 +111,6 @@ def locked_details(dep_name) .find { |d| d["name"] == normalise(dep_name) } end - sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end From d8e59c392cf6f5f7bf757acae78ebffc9304d684 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:38:04 +0000 Subject: [PATCH 30/43] To fix removed hash --- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 2018c14b6f..8993f379fc 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -68,11 +68,11 @@ def setup_requires_array dep.name + dep.requirements.first[:requirement].to_s end - end + ends - sig { returns(T::Hash[T.untyped, T.untyped]) } + sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(T.untyped, T.untyped) + @extras_require_hash = T.let(Hash, T.untyped) @extras_require_hash ||= begin hash = {} From 136d3c11878ff3f74ed14949a49e9f901e35d536 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:41:53 +0000 Subject: [PATCH 31/43] Added removed typo error --- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 8993f379fc..d8aa7f3b32 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -68,11 +68,11 @@ def setup_requires_array dep.name + dep.requirements.first[:requirement].to_s end - ends + end sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(Hash, T.untyped) + @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} From 2dbfed7d8255edffad25ea4696753e5ef40fe7b4 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:07:20 +0000 Subject: [PATCH 32/43] Removed typecheck form setup_file_sanitixer --- .../file_updater/setup_file_sanitizer.rb | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index d8aa7f3b32..8f0abad7d8 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -1,4 +1,4 @@ -# typed: strict +# typed: true # frozen_string_literal: true require "dependabot/python/file_updater" @@ -13,13 +13,11 @@ class FileUpdater class SetupFileSanitizer extend T::Sig - sig { params(setup_file: DependencyFile, setup_cfg: T.untyped).void } def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg end - sig { returns(String) } def sanitized_content # The part of the setup.py that Pipenv cares about appears to be the # install_requires. A name and version are required by don't end up @@ -36,17 +34,13 @@ def sanitized_content private - sig { returns(DependencyFile) } attr_reader :setup_file - sig { returns(String) } attr_reader :setup_cfg - sig { returns(T::Boolean) } def include_pbr? setup_requires_array.any? { |d| d.start_with?("pbr") } end - sig { returns(T.untyped) } def install_requires_array @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= @@ -58,9 +52,7 @@ def install_requires_array end end - sig { returns(T::Array[String]) } def setup_requires_array - @setup_requires_array = T.let(T.untyped, T.untyped) @setup_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -70,9 +62,7 @@ def setup_requires_array end end - sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} @@ -90,18 +80,15 @@ def extras_require_hash end end - sig { returns(T.untyped) } def parsed_setup_file - @parsed_setup_file ||= T.let(Python::FileParser::SetupFileParser.new( + @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( dependency_files: [ setup_file.dup.tap { |f| f.name = "setup.py" }, setup_cfg.dup.tap { |f| f.name = "setup.cfg" } ].compact - ) - .dependency_set, T.untyped) + ).dependency_set end - sig { returns(T.nilable(String)) } def package_name content = setup_file.content match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) From 57952cf9b18f389c0fa21ef4830268e3d6a7553b Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:26:09 +0000 Subject: [PATCH 33/43] Reverted back setup_file_sanitizer --- .../dependabot/python/file_parser/pyproject_files_parser.rb | 6 +++--- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 65d4b43704..acd0fd7b3e 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::Dependency]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } @@ -288,7 +288,7 @@ def parsed_poetry_lock raise Dependabot::DependencyFileNotParseable, poetry_lock.path end - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::Dependency]) } def pyproject @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 8f0abad7d8..24e43749a6 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -83,8 +83,8 @@ def extras_require_hash def parsed_setup_file @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( dependency_files: [ - setup_file.dup.tap { |f| f.name = "setup.py" }, - setup_cfg.dup.tap { |f| f.name = "setup.cfg" } + setup_file&.dup&.tap { |f| f.name = "setup.py" }, + setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } ].compact ).dependency_set end From cf4a42c1d603a93a4f90868a697a99baefab7c7c Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:29:01 +0000 Subject: [PATCH 34/43] Reverted back setup_file_sanitizer --- .../lib/dependabot/python/file_parser/pyproject_files_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index acd0fd7b3e..c80635ae67 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -288,7 +288,7 @@ def parsed_poetry_lock raise Dependabot::DependencyFileNotParseable, poetry_lock.path end - sig { returns(T::Array[Dependabot::Dependency]) } + sig { returns(T.untyped) } def pyproject @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end From 27b91c4c5a1e694e63363015f362ba580c5f8502 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:32:21 +0000 Subject: [PATCH 35/43] Reverted back setup_file_sanitizer --- .../dependabot/python/file_parser/pyproject_files_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index c80635ae67..65d4b43704 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T::Array[Dependabot::Dependency]).void } + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T::Array[Dependabot::Dependency]) } + sig { returns(T.untyped) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } From 5842b20f10e1fb9378cb515e4cf92f574d00bd99 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:48:07 +0000 Subject: [PATCH 36/43] Pulled back --- .../file_updater/setup_file_sanitizer.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 24e43749a6..de74f69fc9 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -3,7 +3,6 @@ require "dependabot/python/file_updater" require "dependabot/python/file_parser/setup_file_parser" -require "sorbet-runtime" module Dependabot module Python @@ -11,8 +10,6 @@ class FileUpdater # Take a setup.py, parses it (carefully!) and then create a new, clean # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer - extend T::Sig - def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg @@ -42,7 +39,6 @@ def include_pbr? end def install_requires_array - @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -81,17 +77,18 @@ def extras_require_hash end def parsed_setup_file - @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( - dependency_files: [ - setup_file&.dup&.tap { |f| f.name = "setup.py" }, - setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } - ].compact - ).dependency_set + @parsed_setup_file ||= + Python::FileParser::SetupFileParser.new( + dependency_files: [ + setup_file&.dup&.tap { |f| f.name = "setup.py" }, + setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } + ].compact + ).dependency_set end def package_name content = setup_file.content - match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) + match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) match ? match[:package_name] : "default_package_name" end end From 2aa399217ca6c19f865a2452a932b9ea013d15c7 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 22:42:15 +0000 Subject: [PATCH 37/43] Reduced untyped --- .../lib/dependabot/python/file_parser/setup_file_parser.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 3c27a8d456..f9bcca8639 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -22,7 +22,7 @@ class SetupFileParser CLOSING_BRACKET = T.let({ "[" => "]", "{" => "}" }.freeze, T.any(T.untyped, T.untyped)) - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -58,10 +58,10 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files - sig { returns(T.untyped) } + sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files From e2b06be16c6c3b7ad6db484427f19d4a39ec0b92 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 22:44:29 +0000 Subject: [PATCH 38/43] Reduced untyped --- python/lib/dependabot/python/file_parser/setup_file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index f9bcca8639..27cef6e9cc 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -61,7 +61,7 @@ def dependency_set sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files - sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } + sig { returns(T.untyped) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files From d6806d1783bc24d67e133fdadb18d089935a6b7e Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 23:23:20 +0000 Subject: [PATCH 39/43] Reduced untyped in pyproject_files_parser --- .../file_parser/pyproject_files_parser.rb | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 65d4b43704..bfb0524332 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } @@ -47,8 +47,8 @@ def pyproject_dependencies if missing_keys.any? raise DependencyFileNotParseable.new( - pyproject.path, - "#{pyproject.path} is missing the following sections:\n" \ + T.must(pyproject).path, + "#{T.must(pyproject).path} is missing the following sections:\n" \ " * #{missing_keys.map { |key| "tool.poetry.#{key}" }.join("\n * ")}\n" ) end @@ -115,7 +115,11 @@ def pep621_dependencies dependencies end - sig { params(type: T.untyped, deps_hash: T.untyped).returns(Dependabot::FileParsers::Base::DependencySet) } + sig do + params(type: String, + deps_hash: T::Hash[String, + T.untyped]).returns(Dependabot::FileParsers::Base::DependencySet) + end def parse_poetry_dependency_group(type, deps_hash) dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -141,7 +145,7 @@ def normalised_name(name, extras) end # @param req can be an Array, Hash or String that represents the constraints for a dependency - sig { params(req: T.untyped, type: T.untyped).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } + sig { params(req: T.untyped, type: String).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } def parse_requirements_from(req, type) [req].flatten.compact.filter_map do |requirement| next if requirement.is_a?(Hash) && UNSUPPORTED_DEPENDENCY_TYPES.intersect?(requirement.keys) @@ -151,14 +155,14 @@ def parse_requirements_from(req, type) if requirement.is_a?(String) { requirement: requirement, - file: pyproject.name, + file: T.must(pyproject).name, source: nil, groups: [type] } else { requirement: requirement["version"], - file: pyproject.name, + file: T.must(pyproject).name, source: requirement.fetch("source", nil), groups: [type] } @@ -178,14 +182,14 @@ def missing_poetry_keys required_keys.reject { |key| poetry_root.key?(key) } end - sig { returns(T.untyped) } + sig { returns(T::Boolean) } def using_pep621? !parsed_pyproject.dig("project", "dependencies").nil? || !parsed_pyproject.dig("project", "optional-dependencies").nil? || !parsed_pyproject.dig("build-system", "requires").nil? end - sig { returns(T.untyped) } + sig { returns(T::Hash[String, T.untyped]) } def poetry_root parsed_pyproject.dig("tool", "poetry") end @@ -232,7 +236,7 @@ def production_dependency_names sig { returns(T::Array[T.nilable(String)]) } def parse_production_dependency_names SharedHelpers.in_a_temporary_directory do - File.write(pyproject.name, pyproject.content) + File.write(T.must(pyproject).name, T.must(pyproject).content) File.write(lockfile.name, lockfile.content) begin @@ -252,7 +256,7 @@ def parse_production_dependency_names end end - sig { params(dep_name: T.untyped).returns(T.untyped) } + sig { params(dep_name: String).returns(T.untyped) } def version_from_lockfile(dep_name) return unless parsed_lockfile @@ -276,21 +280,22 @@ def normalise(name) sig { returns(T.untyped) } def parsed_pyproject - @parsed_pyproject ||= T.let(TomlRB.parse(pyproject.content), T.untyped) + @parsed_pyproject ||= T.let(TomlRB.parse(T.must(pyproject).content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError - raise Dependabot::DependencyFileNotParseable, pyproject.path + raise Dependabot::DependencyFileNotParseable, T.must(pyproject).path end sig { returns(T.untyped) } def parsed_poetry_lock - @parsed_poetry_lock ||= T.let(TomlRB.parse(poetry_lock.content), T.untyped) + @parsed_poetry_lock ||= T.let(TomlRB.parse(T.must(poetry_lock).content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError - raise Dependabot::DependencyFileNotParseable, poetry_lock.path + raise Dependabot::DependencyFileNotParseable, T.must(poetry_lock).path end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pyproject - @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) + @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, + T.nilable(Dependabot::DependencyFile)) end sig { returns(T.untyped) } @@ -306,16 +311,16 @@ def parsed_pep621_dependencies SharedHelpers.run_helper_subprocess( command: "pyenv exec python3 #{NativeHelpers.python_helper_path}", function: "parse_pep621_dependencies", - args: [pyproject.name] + args: [T.must(pyproject).name] ) end end sig { returns(Integer) } def write_temporary_pyproject - path = pyproject.name + path = T.must(pyproject).name FileUtils.mkdir_p(Pathname.new(path).dirname) - File.write(path, pyproject.content) + File.write(path, T.must(pyproject).content) end sig { returns(T.untyped) } @@ -323,14 +328,16 @@ def parsed_lockfile parsed_poetry_lock if poetry_lock end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def poetry_lock - @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, T.untyped) + @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, + T.nilable(Dependabot::DependencyFile)) end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pdm_lock - @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, T.untyped) + @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, + T.nilable(Dependabot::DependencyFile)) end end end From 64be557030a3b31956ce6f4e510d5f3d6b1cf336 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 23:38:37 +0000 Subject: [PATCH 40/43] Reduced untyped in pyproject_files_parser --- .../python/file_parser/pyproject_files_parser.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index bfb0524332..2dcf33ba0f 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -69,11 +69,11 @@ def parse_poetry_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new POETRY_DEPENDENCY_TYPES.each do |type| - deps_hash = poetry_root[type] || {} + deps_hash = T.must(poetry_root)[type] || {} dependencies += parse_poetry_dependency_group(type, deps_hash) end - groups = poetry_root["group"] || {} + groups = T.must(poetry_root)["group"] || {} groups.each do |group, group_spec| dependencies += parse_poetry_dependency_group(group, group_spec["dependencies"]) end @@ -177,9 +177,9 @@ def using_poetry? sig { returns(T::Array[String]) } def missing_poetry_keys - package_mode = poetry_root.fetch("package-mode", true) + package_mode = T.must(poetry_root).fetch("package-mode", true) required_keys = package_mode ? %w(name version description authors) : [] - required_keys.reject { |key| poetry_root.key?(key) } + required_keys.reject { |key| T.must(poetry_root).key?(key) } end sig { returns(T::Boolean) } @@ -189,7 +189,7 @@ def using_pep621? !parsed_pyproject.dig("build-system", "requires").nil? end - sig { returns(T::Hash[String, T.untyped]) } + sig { returns(T.nilable(T::Hash[String, T.untyped])) } def poetry_root parsed_pyproject.dig("tool", "poetry") end From 410dbf1cc777965a3b262eda40b3d0a6ca9ae541 Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:29:27 +0000 Subject: [PATCH 41/43] Add Sorbet Type Checking for file: bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb (#11326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added sorbet typing * Added some typecheck. * Lint error fixes. * Updated as per the review comments --------- Co-authored-by: “Thavachelvam <“thavaahariharangit@git.com”> --- .../bundler/file_updater/gemspec_updater.rb | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb b/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb index 4bd5ab093d..deba3d3ddb 100644 --- a/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb +++ b/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/bundler/file_updater" @@ -9,13 +9,17 @@ class FileUpdater class GemspecUpdater require_relative "requirement_replacer" + extend T::Sig + + sig { params(dependencies: T::Array[Dependabot::Dependency], gemspec: Dependabot::DependencyFile).void } def initialize(dependencies:, gemspec:) - @dependencies = dependencies - @gemspec = gemspec + @dependencies = T.let(dependencies, T::Array[Dependabot::Dependency]) + @gemspec = T.let(gemspec, Dependabot::DependencyFile) end + sig { returns(String) } def updated_gemspec_content - content = gemspec.content + content = T.let(T.must(gemspec.content), String) dependencies.each do |dependency| content = replace_gemspec_version_requirement( @@ -28,21 +32,28 @@ def updated_gemspec_content private + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependencies + + sig { returns(Dependabot::DependencyFile) } attr_reader :gemspec + sig do + params(gemspec: Dependabot::DependencyFile, dependency: Dependabot::Dependency, + content: String).returns(String) + end def replace_gemspec_version_requirement(gemspec, dependency, content) return content unless requirement_changed?(gemspec, dependency) updated_requirement = - dependency.requirements - .find { |r| r[:file] == gemspec.name } - .fetch(:requirement) + T.must(dependency.requirements + .find { |r| r[:file] == gemspec.name }) + .fetch(:requirement) previous_requirement = - dependency.previous_requirements - .find { |r| r[:file] == gemspec.name } - .fetch(:requirement) + T.must(T.must(dependency.previous_requirements) + .find { |r| r[:file] == gemspec.name }) + .fetch(:requirement) RequirementReplacer.new( dependency: dependency, @@ -52,9 +63,10 @@ def replace_gemspec_version_requirement(gemspec, dependency, content) ).rewrite(content) end + sig { params(file: Dependabot::DependencyFile, dependency: Dependabot::Dependency).returns(T::Boolean) } def requirement_changed?(file, dependency) changed_requirements = - dependency.requirements - dependency.previous_requirements + dependency.requirements - T.must(dependency.previous_requirements) changed_requirements.any? { |f| f[:file] == file.name } end From 0fce91e8c7e6c190765211b756f025d4f7caef12 Mon Sep 17 00:00:00 2001 From: Alfred Mazimbe Date: Fri, 17 Jan 2025 12:03:34 +0000 Subject: [PATCH 42/43] Add sorbet type checking for swift/file_parser/manifest_parser.rb (#11329) * Add sorbet type checking for swift/file_parser/manifest_parser.rb * Update manifest_parser.rb --- .../swift/file_parser/manifest_parser.rb | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/swift/lib/dependabot/swift/file_parser/manifest_parser.rb b/swift/lib/dependabot/swift/file_parser/manifest_parser.rb index 6f6b02054b..57289a2541 100644 --- a/swift/lib/dependabot/swift/file_parser/manifest_parser.rb +++ b/swift/lib/dependabot/swift/file_parser/manifest_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strong # frozen_string_literal: true require "dependabot/file_parsers/base" @@ -8,25 +8,35 @@ module Dependabot module Swift class FileParser < Dependabot::FileParsers::Base class ManifestParser + extend T::Sig + extend T::Helpers + DEPENDENCY = /(?\.package\(\s* (?:name:\s+"[^"]+",\s*)?url:\s+"(?[^"]+)",\s*(?#{NativeRequirement::REGEXP})\s* \))/x + sig do + params( + manifest: Dependabot::DependencyFile, + source: T::Hash[Symbol, String] + ).void + end def initialize(manifest, source:) @manifest = manifest @source = source end + sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) } def requirements - found = manifest.content.scan(DEPENDENCY).find do |_declaration, url, _requirement| - SharedHelpers.scp_to_standard(url) == source[:url] + found = manifest.content&.scan(DEPENDENCY)&.find do |_declaration, url, _requirement| + SharedHelpers.scp_to_standard(url.to_s) == source[:url] end return [] unless found - declaration = found.first - requirement = NativeRequirement.new(found.last) + declaration = T.cast(found, T::Array[String]).first + requirement = NativeRequirement.new(T.cast(found, T::Array[String]).last) [ { @@ -41,7 +51,10 @@ def requirements private + sig { returns(Dependabot::DependencyFile) } attr_reader :manifest + + sig { returns(T::Hash[Symbol, String]) } attr_reader :source end end From 1328acbaaf01817bcd552b35fa39bbd6f9c3e0b0 Mon Sep 17 00:00:00 2001 From: "S.Sandhu" <167903774+sachin-sandhu@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:56:32 -0500 Subject: [PATCH 43/43] Fixes [3.2k weekly errors] [pip] exception handlers for Python ecosystem (#11325) * Adds exception handler and test cases * Update python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb Co-authored-by: Alfred Mazimbe --------- Co-authored-by: Alfred Mazimbe --- .../pip_compile_version_resolver.rb | 21 ++++++++ .../pip_compile_version_resolver_spec.rb | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb index 1b8accb8af..dcc98808a8 100644 --- a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +++ b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb @@ -37,6 +37,7 @@ class PipCompileVersionResolver attr_reader :dependency_files attr_reader :credentials attr_reader :repo_contents_path + attr_reader :error_handler def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:) @dependency = dependency @@ -44,6 +45,7 @@ def initialize(dependency:, dependency_files:, credentials:, repo_contents_path: @credentials = credentials @repo_contents_path = repo_contents_path @build_isolation = true + @error_handler = PipCompileErrorHandler.new end def latest_resolvable_version(requirement: nil) @@ -186,6 +188,8 @@ def handle_pip_compile_errors(message) raise Dependabot::OutOfMemory if message.end_with?("MemoryError") + error_handler.handle_pipcompile_error(message) + raise end # rubocop:enable Metrics/AbcSize @@ -494,5 +498,22 @@ def setup_cfg_files end end end + + class PipCompileErrorHandler + SUBPROCESS_ERROR = /subprocess-exited-with-error/ + + INSTALLATION_ERROR = /InstallationError/ + + INSTALLATION_SUBPROCESS_ERROR = /InstallationSubprocessError/ + + HASH_MISMATCH = /HashMismatch/ + + def handle_pipcompile_error(error) + return unless error.match?(SUBPROCESS_ERROR) || error.match?(INSTALLATION_ERROR) || + error.match?(INSTALLATION_SUBPROCESS_ERROR) || error.match?(HASH_MISMATCH) + + raise DependencyFileNotResolvable, "Error resolving dependency" + end + end end end diff --git a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb index 300feb0385..1562c3372c 100644 --- a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb @@ -443,6 +443,54 @@ .to raise_error(Dependabot::OutOfMemory) end end + + context "when HelperSubprocessFailed exception is raised" do + let(:error_handler) { Dependabot::Python::PipCompileErrorHandler.new } + + let(:exception_message) { "HelperSubprocessFailed" } + + context "when dealing with subprocess-exited-with-error error" do + let(:exception_message) do + "Preparing metadata (setup.py): finished with status 'error' + error: subprocess-exited-with-error + + × python setup.py egg_info did not run successfully. + │ exit code: 1 + ╰─> [18 lines of output] + Traceback (most recent call last): + File \"\", line 2, in + exec(compile(''' + ~~~~^^^^^^^^^^^^ + # This is -- a caller that pip uses to run setup.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ...<31 lines>... + exec(compile(setup_py_code, filename, \"exec\")) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ''' % ('/tmp/pip-resolve-84tqp2g1/pillow_f28c34dffdb342a49c27519580a49fd3/setup.py',)" + end + + it "raises a helpful error" do + expect { error_handler.handle_pipcompile_error(exception_message) } + .to raise_error(Dependabot::DependencyFileNotResolvable) + end + end + + context "when dealing with an installation error" do + let(:exception_message) do + "pip._internal.exceptions.InstallationError: Could not install requirement" \ + " rugby-[FILTERED_REPO]@ https://****@github.com/compute-cloud/a300cfb3a4070c923246dd4.zip " \ + "from https://****@github.com/compute-cloud/[FILTERED_REPO]/archive/s.zip" \ + " (from -r requirements.in (line 23)) because of HTTP error 404 Client Error: Not" \ + " Found for url: https://github.com/compute-cloud/[FILTERED_REPO]/archive/a3246dd4.zip for" \ + " URL https://****@github.com/compute-cloud/[FILTERED_REPO]/archive/a3dd4.zip" + end + + it "raises a helpful error" do + expect { error_handler.handle_pipcompile_error(exception_message) } + .to raise_error(Dependabot::DependencyFileNotResolvable) + end + end + end end end end