diff --git a/singer_sdk/_singerlib/catalog.py b/singer_sdk/_singerlib/catalog.py index 77fe884d8..87b528466 100644 --- a/singer_sdk/_singerlib/catalog.py +++ b/singer_sdk/_singerlib/catalog.py @@ -31,11 +31,7 @@ def __missing__(self, breadcrumb: Breadcrumb) -> bool: Returns: True if the breadcrumb is selected, False otherwise. """ - if len(breadcrumb) >= 2: # noqa: PLR2004 - parent = breadcrumb[:-2] - return self[parent] - - return True + return self[breadcrumb[:-2]] if len(breadcrumb) >= 2 else True # noqa: PLR2004 @dataclass @@ -71,7 +67,7 @@ def from_dict(cls: type[Metadata], value: dict[str, t.Any]) -> Metadata: ) def to_dict(self) -> dict[str, t.Any]: - """Convert metadata to a JSON-encodeable dictionary. + """Convert metadata to a JSON-encodable dictionary. Returns: Metadata object. diff --git a/singer_sdk/authenticators.py b/singer_sdk/authenticators.py index 61382daba..970c777ad 100644 --- a/singer_sdk/authenticators.py +++ b/singer_sdk/authenticators.py @@ -462,9 +462,7 @@ def client_id(self) -> str | None: Returns: Optional client secret from stream config if it has been set. """ - if self.config: - return self.config.get("client_id") - return None + return self.config.get("client_id") if self.config else None @property def client_secret(self) -> str | None: @@ -473,9 +471,7 @@ def client_secret(self) -> str | None: Returns: Optional client secret from stream config if it has been set. """ - if self.config: - return self.config.get("client_secret") - return None + return self.config.get("client_secret") if self.config else None def is_token_valid(self) -> bool: """Check if token is valid. @@ -487,9 +483,7 @@ def is_token_valid(self) -> bool: return False if not self.expires_in: return True - if self.expires_in > (utc_now() - self.last_refreshed).total_seconds(): - return True - return False + return self.expires_in > (utc_now() - self.last_refreshed).total_seconds() # Authentication and refresh def update_access_token(self) -> None: @@ -520,7 +514,7 @@ def update_access_token(self) -> None: self.expires_in = int(expiration) if expiration else None if self.expires_in is None: self.logger.debug( - "No expires_in receied in OAuth response and no " + "No expires_in received in OAuth response and no " "default_expiration set. Token will be treated as if it never " "expires.", ) @@ -566,7 +560,7 @@ def oauth_request_body(self) -> dict: @property def oauth_request_payload(self) -> dict: - """Return request paytload for OAuth request. + """Return request payload for OAuth request. Returns: Payload object for OAuth. diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index c6f957589..6b0552a22 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -902,9 +902,6 @@ def merge_sql_types( if issubclass( generic_type, (sqlalchemy.types.String, sqlalchemy.types.Unicode), - ) or issubclass( - generic_type, - (sqlalchemy.types.String, sqlalchemy.types.Unicode), ): # If length None or 0 then is varchar max ? if ( @@ -913,8 +910,6 @@ def merge_sql_types( or (cur_len and (opt_len >= cur_len)) ): return opt - # If best conversion class is equal to current type - # return the best conversion class elif str(opt) == str(current_type): return opt diff --git a/singer_sdk/helpers/_conformers.py b/singer_sdk/helpers/_conformers.py index 46963284e..0ca70e85c 100644 --- a/singer_sdk/helpers/_conformers.py +++ b/singer_sdk/helpers/_conformers.py @@ -16,11 +16,13 @@ def snakecase(string: str) -> str: """ string = re.sub(r"[\-\.\s]", "_", string) string = ( - string[0].lower() - + re.sub( - r"[A-Z]", - lambda matched: "_" + str(matched.group(0).lower()), - string[1:], + ( + string[0].lower() + + re.sub( + r"[A-Z]", + lambda matched: f"_{matched.group(0).lower()!s}", + string[1:], + ) ) if string else string diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index c0abcb6b2..eeb244277 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -465,12 +465,9 @@ def _should_jsondump_value(key: str, value: t.Any, flattened_schema=None) -> boo if isinstance(value, (dict, list)): return True - if ( + return bool( flattened_schema and key in flattened_schema and "type" in flattened_schema[key] and set(flattened_schema[key]["type"]) == {"null", "object", "array"} - ): - return True - - return False + ) diff --git a/singer_sdk/helpers/_state.py b/singer_sdk/helpers/_state.py index 9d0102186..a42a38433 100644 --- a/singer_sdk/helpers/_state.py +++ b/singer_sdk/helpers/_state.py @@ -18,7 +18,7 @@ STARTING_MARKER = "starting_replication_value" -def get_state_if_exists( # noqa: PLR0911 +def get_state_if_exists( tap_state: dict, tap_stream_id: str, state_partition_context: dict | None = None, @@ -47,9 +47,7 @@ def get_state_if_exists( # noqa: PLR0911 stream_state = tap_state["bookmarks"][tap_stream_id] if not state_partition_context: - if key: - return stream_state.get(key, None) - return stream_state + return stream_state.get(key, None) if key else stream_state if "partitions" not in stream_state: return None # No partitions defined @@ -59,9 +57,7 @@ def get_state_if_exists( # noqa: PLR0911 ) if matched_partition is None: return None # Partition definition not present - if key: - return matched_partition.get(key, None) - return matched_partition + return matched_partition.get(key, None) if key else matched_partition def get_state_partitions_list(tap_state: dict, tap_stream_id: str) -> list[dict] | None: @@ -84,10 +80,7 @@ def _find_in_partitions_list( f"{{state_partition_context}}.\nMatching state values were: {found!s}" ) raise ValueError(msg) - if found: - return t.cast(dict, found[0]) - - return None + return t.cast(dict, found[0]) if found else None def _create_in_partitions_list( diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index 6eb7b3879..3a87ab4b9 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -79,7 +79,7 @@ def is_secret_type(type_dict: dict) -> bool: """Return True if JSON Schema type definition appears to be a secret. Will return true if either `writeOnly` or `secret` are true on this type - or any of the type's subproperties. + or any of the type's sub-properties. Args: type_dict: The JSON Schema type to check. @@ -96,7 +96,7 @@ def is_secret_type(type_dict: dict) -> bool: return True if "properties" in type_dict: - # Recursively check subproperties and return True if any child is secret. + # Recursively check sub-properties and return True if any child is secret. return any( is_secret_type(child_type_dict) for child_type_dict in type_dict["properties"].values() @@ -388,6 +388,7 @@ def conform_record_data_types( return rec +# TODO: This is in dire need of refactoring. It's a mess. def _conform_record_data_types( # noqa: PLR0912 input_object: dict[str, t.Any], schema: dict, @@ -405,7 +406,7 @@ def _conform_record_data_types( # noqa: PLR0912 input_object: A single record schema: JSON schema the given input_object is expected to meet level: Specifies how recursive the conformance process should be - parent: '.' seperated path to this element from the object root (for logging) + parent: '.' separated path to this element from the object root (for logging) """ output_object: dict[str, t.Any] = {} unmapped_properties: list[str] = [] diff --git a/singer_sdk/io_base.py b/singer_sdk/io_base.py index 07da6e63e..d389bfd08 100644 --- a/singer_sdk/io_base.py +++ b/singer_sdk/io_base.py @@ -48,7 +48,7 @@ def _assert_line_requires(line_dict: dict, requires: set[str]) -> None: if not requires.issubset(line_dict): missing = requires - set(line_dict) msg = f"Line is missing required {', '.join(missing)} key(s): {line_dict}" - raise Exception(msg) + raise Exception(msg) # TODO: Raise a more specific exception def deserialize_json(self, line: str) -> dict: """Deserialize a line of json. diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py index 5b19f0b07..b48c49727 100644 --- a/singer_sdk/mapper.py +++ b/singer_sdk/mapper.py @@ -273,10 +273,7 @@ def transform(self, record: dict) -> dict | None: The transformed record. """ transformed_record = self._transform_fn(record) - if not transformed_record: - return None - - return super().transform(transformed_record) + return super().transform(transformed_record) if transformed_record else None def get_filter_result(self, record: dict) -> bool: """Return True to include or False to exclude. @@ -291,7 +288,7 @@ def get_filter_result(self, record: dict) -> bool: @property def functions(self) -> dict[str, t.Callable]: - """Get availabale transformation functions. + """Get available transformation functions. Returns: Functions which should be available for expression evaluation. diff --git a/singer_sdk/metrics.py b/singer_sdk/metrics.py index 89d51a338..e4191eadf 100644 --- a/singer_sdk/metrics.py +++ b/singer_sdk/metrics.py @@ -263,10 +263,9 @@ def __exit__( exc_tb: The exception traceback. """ if Tag.STATUS not in self.tags: - if exc_type is None: - self.tags[Tag.STATUS] = Status.SUCCEEDED - else: - self.tags[Tag.STATUS] = Status.FAILED + self.tags[Tag.STATUS] = ( + Status.SUCCEEDED if exc_type is None else Status.FAILED + ) log(self.logger, Point("timer", self.metric, self.elapsed(), self.tags)) def elapsed(self) -> float: diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index ec4664f67..e56d5aab5 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -247,7 +247,7 @@ def _add_sdc_metadata_to_record( tz=datetime.timezone.utc, ).isoformat() record["_sdc_batched_at"] = ( - context.get("batch_start_time", None) + context.get("batch_start_time") or datetime.datetime.now(tz=datetime.timezone.utc) ).isoformat() record["_sdc_deleted_at"] = record.get("_sdc_deleted_at") diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 238e83dec..5c143818c 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -98,13 +98,7 @@ def schema_name(self) -> str | None: if default_target_schema: return default_target_schema - if len(parts) in {2, 3}: - # Stream name is a two-part or three-part identifier. - # Use the second-to-last part as the schema name. - return self.conform_name(parts[-2], "schema") - - # Schema name not detected. - return None + return self.conform_name(parts[-2], "schema") if len(parts) in {2, 3} else None @property def database_name(self) -> str | None: diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index d4b5a8dc3..f7a31e73c 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -121,10 +121,10 @@ def streams(self) -> dict[str, Stream]: Returns: A mapping of names to streams, using discovery or a provided catalog. """ - input_catalog = self.input_catalog - if self._streams is None: self._streams = {} + input_catalog = self.input_catalog + for stream in self.load_streams(): if input_catalog is not None: stream.apply_catalog(input_catalog) diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py index aabd8a640..c9d5d62ef 100644 --- a/singer_sdk/target_base.py +++ b/singer_sdk/target_base.py @@ -370,7 +370,7 @@ def _process_schema_message(self, message_dict: dict) -> None: stream_name = message_dict["stream"] schema = message_dict["schema"] - key_properties = message_dict.get("key_properties", None) + key_properties = message_dict.get("key_properties") do_registration = False if stream_name not in self.mapper.stream_maps: do_registration = True diff --git a/singer_sdk/testing/factory.py b/singer_sdk/testing/factory.py index ce6d2ec4c..f31930017 100644 --- a/singer_sdk/testing/factory.py +++ b/singer_sdk/testing/factory.py @@ -144,6 +144,7 @@ def runner(self) -> TapTestRunner | TargetTestRunner: return TapTestClass + # TODO: Refactor this. It's too long and nested. def _annotate_test_class( # noqa: C901 self, empty_test_class: type[BaseTestClass], diff --git a/singer_sdk/testing/legacy.py b/singer_sdk/testing/legacy.py index 5baa94034..5329b3224 100644 --- a/singer_sdk/testing/legacy.py +++ b/singer_sdk/testing/legacy.py @@ -150,10 +150,7 @@ def _get_tap_catalog( # Test discovery tap.run_discovery() catalog_dict = tap.catalog_dict - if select_all: - return _select_all(catalog_dict) - - return catalog_dict + return _select_all(catalog_dict) if select_all else catalog_dict def _select_all(catalog_dict: dict) -> dict: diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index a9c48303e..37c1f40c1 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -965,12 +965,14 @@ def to_jsonschema_type( msg = "Expected `str` or a SQLAlchemy `TypeEngine` object or type." raise ValueError(msg) - # Look for the type name within the known SQL type names: - for sqltype, jsonschema_type in sqltype_lookup.items(): - if sqltype.lower() in type_name.lower(): - return jsonschema_type - - return sqltype_lookup["string"] # safe failover to str + return next( + ( + jsonschema_type + for sqltype, jsonschema_type in sqltype_lookup.items() + if sqltype.lower() in type_name.lower() + ), + sqltype_lookup["string"], # safe failover to str + ) def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> bool: @@ -981,7 +983,7 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo type_check: A tuple of type strings to look for. Returns: - True if the schema suports the type. + True if the schema supports the type. """ if "type" in jsonschema_type: if isinstance(jsonschema_type["type"], (list, tuple)): @@ -991,12 +993,9 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo elif jsonschema_type.get("type") in type_check: return True - if any( + return any( _jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ()) - ): - return True - - return False + ) def to_sql_type( # noqa: PLR0911, C901