diff --git a/README.md b/README.md index ad5242bb8..fdac6bc03 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Start [here](https://lyft.github.io/cartography/install.html). - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers - [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints +- [Slack](https://lyft.github.io/cartography/modules/slack/index.html) - Users, Groups, Channels ## Usage diff --git a/cartography/cli.py b/cartography/cli.py index 0a3287eb1..3a0ce3645 100644 --- a/cartography/cli.py +++ b/cartography/cli.py @@ -509,6 +509,29 @@ def _build_parser(self): 'Required if you are using the Semgrep intel module. Ignored otherwise.' ), ) + parser.add_argument( + '--slack-token-env-var', + type=str, + default=None, + help=( + 'The name of environment variable containing the Slack Token' + ), + ) + parser.add_argument( + '--slack-teams', + type=str, + default=None, + help=( + 'The Slack Team ID to sync, comma separated.' + ), + ) + parser.add_argument( + '--slack-channels-memberships', + action='store_true', + help=( + 'Pull memberships for Slack Channels (can be time consuming).' + ), + ) return parser def main(self, argv: str) -> int: @@ -685,6 +708,13 @@ def main(self, argv: str) -> int: else: config.semgrep_app_token = None + # Slack config + if config.slack_token_env_var: + logger.debug( + f"Reading Slack token from environment variables {config.slack_token_env_var}", + ) + config.slack_token = os.environ.get(config.slack_token_env_var) + # Run cartography try: return cartography.sync.run_with_config(self.sync, config) diff --git a/cartography/config.py b/cartography/config.py index 556d7082a..f249dcaf4 100644 --- a/cartography/config.py +++ b/cartography/config.py @@ -105,6 +105,12 @@ class Config: :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional. :param semgrep_app_token: The Semgrep api token. Optional. :type semgrep_app_token: str + :type slack_token: str + :param slack_token: Slack API Token. Optional. + :type slack_teams: str + :param slack_teams: Comma-separated list of Slack teams to sync. Optional. + :type slack_channels_memberships: bool + :param slack_channels_memberships: If True, sync Slack channel memberships. Optional. """ def __init__( @@ -160,6 +166,9 @@ def __init__( duo_api_secret=None, duo_api_hostname=None, semgrep_app_token=None, + slack_token=None, + slack_teams=None, + slack_channels_memberships=False, ): self.neo4j_uri = neo4j_uri self.neo4j_user = neo4j_user @@ -212,3 +221,6 @@ def __init__( self.duo_api_secret = duo_api_secret self.duo_api_hostname = duo_api_hostname self.semgrep_app_token = semgrep_app_token + self.slack_token = slack_token + self.slack_teams = slack_teams + self.slack_channels_memberships = slack_channels_memberships diff --git a/cartography/intel/slack/__init__.py b/cartography/intel/slack/__init__.py new file mode 100644 index 000000000..dd081e5b7 --- /dev/null +++ b/cartography/intel/slack/__init__.py @@ -0,0 +1,72 @@ +import logging + +import neo4j +from slack_sdk import WebClient +from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler + +import cartography.intel.slack.channels +import cartography.intel.slack.groups +import cartography.intel.slack.team +import cartography.intel.slack.users +from cartography.config import Config +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def start_slack_ingestion(neo4j_session: neo4j.Session, config: Config) -> None: + """ + If this module is configured, perform ingestion of Slack data. Otherwise warn and exit + :param neo4j_session: Neo4J session for database interface + :param config: A cartography.config object + :return: None + """ + + if not config.slack_token or not config.slack_teams: + logger.info( + 'Slack import is not configured - skipping this module. ' + 'See docs to configure.', + ) + return + + common_job_parameters = { + "UPDATE_TAG": config.update_tag, + "CHANNELS_MEMBERSHIPS": config.slack_channels_memberships, + } + + rate_limit_handler = RateLimitErrorRetryHandler(max_retry_count=1) + slack_client = WebClient(token=config.slack_token) + slack_client.retry_handlers.append(rate_limit_handler) + + for team_id in config.slack_teams.split(','): + logger.info("Syncing team %s", team_id) + common_job_parameters['TEAM_ID'] = team_id + cartography.intel.slack.team.sync( + neo4j_session, + slack_client, + team_id, + config.update_tag, + common_job_parameters, + ) + cartography.intel.slack.users.sync( + neo4j_session, + slack_client, + team_id, + config.update_tag, + common_job_parameters, + ) + cartography.intel.slack.channels.sync( + neo4j_session, + slack_client, + team_id, + config.update_tag, + common_job_parameters, + ) + cartography.intel.slack.groups.sync( + neo4j_session, + slack_client, + team_id, + config.update_tag, + common_job_parameters, + ) diff --git a/cartography/intel/slack/channels.py b/cartography/intel/slack/channels.py new file mode 100644 index 000000000..52805e6dd --- /dev/null +++ b/cartography/intel/slack/channels.py @@ -0,0 +1,85 @@ +import logging +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + +import neo4j +from slack_sdk import WebClient + +from cartography.client.core.tx import load +from cartography.graph.job import GraphJob +from cartography.intel.slack.utils import slack_paginate +from cartography.models.slack.channels import SlackChannelSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + slack_client: WebClient, + team_id: str, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + channels = get(slack_client, team_id, common_job_parameters['CHANNELS_MEMBERSHIPS']) + load_channels(neo4j_session, channels, team_id, update_tag) + cleanup(neo4j_session, common_job_parameters) + + +@timeit +def get(slack_client: WebClient, team_id: str, get_memberships: bool) -> List[Dict[str, Any]]: + channels: List[Dict[str, Any]] = [] + for channel in slack_paginate( + slack_client, + 'conversations_list', + 'channels', + team_id=team_id, + ): + if channel['is_archived']: + channels.append(channel) + elif get_memberships: + for member in slack_paginate( + slack_client, + 'conversations_members', + 'members', + channel=channel['id'], + ): + channel_m = channel.copy() + channel_m['member_id'] = member + channels.append(channel_m) + else: + channels.append(channel) + return channels + + +def _get_membership(slack_client: WebClient, slack_channel: str, cursor: Optional[str] = None) -> List[str]: + result = [] + memberships = slack_client.conversations_members(channel=slack_channel, cursor=cursor) + for m in memberships['members']: + result.append(m) + next_cursor = memberships.get('response_metadata', {}).get('next_cursor', '') + if next_cursor != '': + result.extend(_get_membership(slack_client, slack_channel, cursor=next_cursor)) + return result + + +def load_channels( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + team_id: str, + update_tag: int, +) -> None: + load( + neo4j_session, + SlackChannelSchema(), + data, + lastupdated=update_tag, + TEAM_ID=team_id, + ) + + +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: + GraphJob.from_node_schema(SlackChannelSchema(), common_job_parameters).run(neo4j_session) diff --git a/cartography/intel/slack/groups.py b/cartography/intel/slack/groups.py new file mode 100644 index 000000000..785a5285c --- /dev/null +++ b/cartography/intel/slack/groups.py @@ -0,0 +1,78 @@ +import logging +from itertools import zip_longest +from typing import Any +from typing import Dict +from typing import List + +import neo4j +from slack_sdk import WebClient + +from cartography.client.core.tx import load +from cartography.graph.job import GraphJob +from cartography.intel.slack.utils import slack_paginate +from cartography.models.slack.group import SlackGroupSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + slack_client: WebClient, + team_id: str, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + groups = get(slack_client, team_id) + formated_groups = transform(groups) + load_groups(neo4j_session, formated_groups, team_id, update_tag) + cleanup(neo4j_session, common_job_parameters) + + +@timeit +def get(slack_client: WebClient, team_id: str) -> List[Dict[str, Any]]: + return slack_paginate( + slack_client, + 'usergroups_list', + 'usergroups', + team_id=team_id, + include_count=True, + include_users=True, + include_disabled=True, + ) + + +@timeit +def transform(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + splitted_groups: List[Dict[str, Any]] = [] + for group in groups: + if len(group['description']) == 0: + group['description'] = None + for ms in zip_longest(group['users'], group['prefs']['channels']): + formated_group = group.copy() + formated_group.pop('users') + formated_group.pop('prefs') + formated_group['member_id'] = ms[0] + formated_group['channel_id'] = ms[1] + splitted_groups.append(formated_group) + return splitted_groups + + +def load_groups( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + team_id: str, + update_tag: int, +) -> None: + load( + neo4j_session, + SlackGroupSchema(), + data, + lastupdated=update_tag, + TEAM_ID=team_id, + ) + + +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: + GraphJob.from_node_schema(SlackGroupSchema(), common_job_parameters).run(neo4j_session) diff --git a/cartography/intel/slack/team.py b/cartography/intel/slack/team.py new file mode 100644 index 000000000..05505dfbc --- /dev/null +++ b/cartography/intel/slack/team.py @@ -0,0 +1,43 @@ +import logging +from typing import Any +from typing import Dict + +import neo4j +from slack_sdk import WebClient + +from cartography.client.core.tx import load +from cartography.models.slack.team import SlackTeamSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + slack_client: WebClient, + team_id: str, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + team = get(slack_client, team_id) + load_team(neo4j_session, team, update_tag) + + +@timeit +def get(slack_client: WebClient, team_id: str) -> Dict[str, Any]: + team_info = slack_client.team_info(team_id=team_id) + return team_info['team'] + + +def load_team( + neo4j_session: neo4j.Session, + data: Dict[str, Any], + update_tag: int, +) -> None: + load( + neo4j_session, + SlackTeamSchema(), + [data], + lastupdated=update_tag, + ) diff --git a/cartography/intel/slack/users.py b/cartography/intel/slack/users.py new file mode 100644 index 000000000..9d51e9d1b --- /dev/null +++ b/cartography/intel/slack/users.py @@ -0,0 +1,52 @@ +import logging +from typing import Any +from typing import Dict +from typing import List + +import neo4j +from slack_sdk import WebClient + +from cartography.client.core.tx import load +from cartography.graph.job import GraphJob +from cartography.intel.slack.utils import slack_paginate +from cartography.models.slack.user import SlackUserSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + slack_client: WebClient, + team_id: str, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + users = get(slack_client, team_id) + load_users(neo4j_session, users, team_id, update_tag) + cleanup(neo4j_session, common_job_parameters) + + +@timeit +def get(slack_client: WebClient, team_id: str) -> List[Dict[str, Any]]: + return slack_paginate(slack_client, 'users_list', 'members', team_id=team_id) + + +def load_users( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + team_id: str, + update_tag: int, +) -> None: + load( + neo4j_session, + SlackUserSchema(), + data, + lastupdated=update_tag, + TEAM_ID=team_id, + ) + + +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: + GraphJob.from_node_schema(SlackUserSchema(), common_job_parameters).run(neo4j_session) diff --git a/cartography/intel/slack/utils.py b/cartography/intel/slack/utils.py new file mode 100644 index 000000000..feadf8214 --- /dev/null +++ b/cartography/intel/slack/utils.py @@ -0,0 +1,31 @@ +from typing import Any +from typing import Dict +from typing import List + +from slack_sdk import WebClient + + +def slack_paginate( + slack_client: WebClient, + endpoint: str, + data_key: str, + **kwargs: Any, +) -> List[Dict[str, Any]]: + items: List[Dict[str, Any]] = [] + endpoint_method = getattr(slack_client, endpoint) + + # First query + response = endpoint_method(**kwargs) + for item in response[data_key]: + items.append(item) + + # Iterate over the cursor + cursor = response.get('response_metadata', {}).get('next_cursor', '') + while cursor != '': + kwargs['cursor'] = cursor + response = endpoint_method(**kwargs) + for item in response[data_key]: + items.append(item) + cursor = response.get('response_metadata', {}).get('next_cursor', '') + + return items diff --git a/cartography/models/slack/__init__.py b/cartography/models/slack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cartography/models/slack/channels.py b/cartography/models/slack/channels.py new file mode 100644 index 000000000..bef03ba49 --- /dev/null +++ b/cartography/models/slack/channels.py @@ -0,0 +1,86 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class SlackChannelNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('name', extra_index=True) + is_private: PropertyRef = PropertyRef('is_private') + created: PropertyRef = PropertyRef('created') + is_archived: PropertyRef = PropertyRef('is_archived') + is_general: PropertyRef = PropertyRef('is_general') + is_shared: PropertyRef = PropertyRef('is_shared') + is_org_shared: PropertyRef = PropertyRef('is_org_shared') + topic: PropertyRef = PropertyRef('topic.value') + purpose: PropertyRef = PropertyRef('purpose.value') + num_members: PropertyRef = PropertyRef('num_members') + + +@dataclass(frozen=True) +class SlackChannelToSlackUserRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackUser)-[:CREATED]->(:SlackChannel) +class SlackChannelToCreatorRel(CartographyRelSchema): + target_node_label: str = 'SlackUser' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('creator')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "CREATED" + properties: SlackChannelToSlackUserRelProperties = SlackChannelToSlackUserRelProperties() + + +@dataclass(frozen=True) +# (:SlackUser)-[:MEMBER_OF]->(:SlackChannel) +class SlackChannelToUserRel(CartographyRelSchema): + target_node_label: str = 'SlackUser' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('member_id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "MEMBER_OF" + properties: SlackChannelToSlackUserRelProperties = SlackChannelToSlackUserRelProperties() + + +@dataclass(frozen=True) +class SlackTeamToSlackChannelRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackTeam)-[:RESOURCE]->(:SlackChannel) +class SlackTeamToChannelRel(CartographyRelSchema): + target_node_label: str = 'SlackTeam' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('TEAM_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: SlackTeamToSlackChannelRelProperties = SlackTeamToSlackChannelRelProperties() + + +@dataclass(frozen=True) +class SlackChannelSchema(CartographyNodeSchema): + label: str = 'SlackChannel' + properties: SlackChannelNodeProperties = SlackChannelNodeProperties() + other_relationships: OtherRelationships = OtherRelationships( + rels=[ + SlackChannelToUserRel(), + SlackChannelToCreatorRel(), + ], + ) + sub_resource_relationship: SlackTeamToChannelRel = SlackTeamToChannelRel() diff --git a/cartography/models/slack/group.py b/cartography/models/slack/group.py new file mode 100644 index 000000000..b329673cb --- /dev/null +++ b/cartography/models/slack/group.py @@ -0,0 +1,111 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class SlackGroupNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('name', extra_index=True) + description: PropertyRef = PropertyRef('description') + is_subteam: PropertyRef = PropertyRef('is_subteam') + handle: PropertyRef = PropertyRef('handle') + is_external: PropertyRef = PropertyRef('is_external') + date_create: PropertyRef = PropertyRef('date_create') + date_update: PropertyRef = PropertyRef('date_update') + date_delete: PropertyRef = PropertyRef('date_delete') + created_by: PropertyRef = PropertyRef('created_by') + updated_by: PropertyRef = PropertyRef('updated_by') + user_count: PropertyRef = PropertyRef('user_count') + channel_count: PropertyRef = PropertyRef('channel_count') + + +@dataclass(frozen=True) +class SlackGroupToSlackUserRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackUser)-[:MEMBER_OF]->(:SlackGroup) +class SlackGroupToUserRel(CartographyRelSchema): + target_node_label: str = 'SlackUser' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('member_id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "MEMBER_OF" + properties: SlackGroupToSlackUserRelProperties = SlackGroupToSlackUserRelProperties() + + +@dataclass(frozen=True) +class SlackGroupToCreatorRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackUser)-[:CREATED]->(:SlackGroup) +class SlackGroupToCreatorRel(CartographyRelSchema): + target_node_label: str = 'SlackUser' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('created_by')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "CREATED" + properties: SlackGroupToCreatorRelProperties = SlackGroupToCreatorRelProperties() + + +@dataclass(frozen=True) +class SlackTeamToSlackGroupRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackTeam)-[:RESOURCE]->(:SlackGroup) +class SlackTeamToGroupRel(CartographyRelSchema): + target_node_label: str = 'SlackTeam' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('TEAM_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: SlackTeamToSlackGroupRelProperties = SlackTeamToSlackGroupRelProperties() + + +@dataclass(frozen=True) +class SlackGroupToSlackChannelRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackChannel)<-[:MEMBER_OF]-(:SlackGroup) +class SlackGroupToChannelRel(CartographyRelSchema): + target_node_label: str = 'SlackChannel' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('channel_id')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "MEMBER_OF" + properties: SlackGroupToSlackChannelRelProperties = SlackGroupToSlackChannelRelProperties() + + +@dataclass(frozen=True) +class SlackGroupSchema(CartographyNodeSchema): + label: str = 'SlackGroup' + properties: SlackGroupNodeProperties = SlackGroupNodeProperties() + other_relationships: OtherRelationships = OtherRelationships( + rels=[ + SlackGroupToUserRel(), + SlackGroupToChannelRel(), + SlackGroupToCreatorRel(), + ], + ) + sub_resource_relationship: SlackTeamToGroupRel = SlackTeamToGroupRel() diff --git a/cartography/models/slack/team.py b/cartography/models/slack/team.py new file mode 100644 index 000000000..2509f82db --- /dev/null +++ b/cartography/models/slack/team.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema + + +@dataclass(frozen=True) +class SlackTeamNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('name', extra_index=True) + domain: PropertyRef = PropertyRef('domain') + url: PropertyRef = PropertyRef('url') + is_verified: PropertyRef = PropertyRef('is_verified') + email_domain: PropertyRef = PropertyRef('email_domain') + + +@dataclass(frozen=True) +class SlackTeamSchema(CartographyNodeSchema): + label: str = 'SlackTeam' + properties: SlackTeamNodeProperties = SlackTeamNodeProperties() diff --git a/cartography/models/slack/user.py b/cartography/models/slack/user.py new file mode 100644 index 000000000..b99e327bf --- /dev/null +++ b/cartography/models/slack/user.py @@ -0,0 +1,76 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class SlackUserNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('name', extra_index=True) + real_name: PropertyRef = PropertyRef('real_name') + display_name: PropertyRef = PropertyRef('profile.display_name') + first_name: PropertyRef = PropertyRef('profile.first_name') + last_name: PropertyRef = PropertyRef('profile.last_name') + profile_title: PropertyRef = PropertyRef('profile.title') + profile_phone: PropertyRef = PropertyRef('profile.phone') + email: PropertyRef = PropertyRef('profile.email', extra_index=True) + deleted: PropertyRef = PropertyRef('deleted') + is_admin: PropertyRef = PropertyRef('is_admin') + is_owner: PropertyRef = PropertyRef('is_owner') + is_restricted: PropertyRef = PropertyRef('is_restricted') + is_ultra_restricted: PropertyRef = PropertyRef('is_ultra_restricted') + is_bot: PropertyRef = PropertyRef('is_bot') + is_app_user: PropertyRef = PropertyRef('is_app_user') + is_email_confirmed: PropertyRef = PropertyRef('is_email_confirmed') + team: PropertyRef = PropertyRef('profile.team') + + +@dataclass(frozen=True) +class SlackUserToHumanRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackUser)<-[:IDENTITY_SLACK]-(:Human) +class SlackHumanToUserRel(CartographyRelSchema): + target_node_label: str = 'Human' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'email': PropertyRef('profile.email')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "IDENTITY_SLACK" + properties: SlackUserToHumanRelProperties = SlackUserToHumanRelProperties() + + +@dataclass(frozen=True) +class SlackTeamToSlackUserRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:SlackTeam)-[:RESOURCE]->(:SlackUser) +class SlackTeamToUserRel(CartographyRelSchema): + target_node_label: str = 'SlackTeam' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('TEAM_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: SlackTeamToSlackUserRelProperties = SlackTeamToSlackUserRelProperties() + + +@dataclass(frozen=True) +class SlackUserSchema(CartographyNodeSchema): + label: str = 'SlackUser' + properties: SlackUserNodeProperties = SlackUserNodeProperties() + other_relationships: OtherRelationships = OtherRelationships(rels=[SlackHumanToUserRel()]) + sub_resource_relationship: SlackTeamToUserRel = SlackTeamToUserRel() diff --git a/cartography/sync.py b/cartography/sync.py index 08ee40e01..6a6b49757 100644 --- a/cartography/sync.py +++ b/cartography/sync.py @@ -28,6 +28,7 @@ import cartography.intel.lastpass import cartography.intel.oci import cartography.intel.okta +import cartography.intel.slack from cartography.config import Config from cartography.stats import set_stats_client from cartography.util import STATUS_FAILURE @@ -54,6 +55,7 @@ 'bigfix': cartography.intel.bigfix.start_bigfix_ingestion, 'duo': cartography.intel.duo.start_duo_ingestion, 'analysis': cartography.intel.analysis.run, + 'slack': cartography.intel.slack.start_slack_ingestion, }) diff --git a/docs/root/modules/slack/config.md b/docs/root/modules/slack/config.md new file mode 100644 index 000000000..2aee010e9 --- /dev/null +++ b/docs/root/modules/slack/config.md @@ -0,0 +1,25 @@ +## Slack Configuration + +.. _slack_config: + +Follow these steps to analyze Slack objects with Cartography. + +1. Create a Slack integration + 1. Go to `https://api.slack.com/apps/` and create a new integration + 1. Add bot permissions in `OAuth & Permissions` + - channels:read + - groups:read + - team.preferences:read + - team:read + - usergroups:read + - users.profile:read + - users:read + - users:read.email + 1. Install the App on your Slack Workspace + 1. Get "Bot User OAuth Token" and store it into an env var + 1. Provide env var name with `--slack-token-env-var ENV_VAR_NAME` parameter + +2. Get your Slack Team ID + 1. In a web-browser go to `https://.slack.com` + 1. You will be redirected to `https://app.slack.com/client/` + 1. User `--slack-teams ` parameter (you can provide multiple teams id comma separated) diff --git a/docs/root/modules/slack/index.rst b/docs/root/modules/slack/index.rst new file mode 100644 index 000000000..7fcd83f56 --- /dev/null +++ b/docs/root/modules/slack/index.rst @@ -0,0 +1,14 @@ +Slack +######## + +The slack module has the following coverage: + +* Users +* Channels +* Groups + +.. toctree:: + :hidden: + :glob: + + * diff --git a/docs/root/modules/slack/schema.md b/docs/root/modules/slack/schema.md new file mode 100644 index 000000000..d95ef8940 --- /dev/null +++ b/docs/root/modules/slack/schema.md @@ -0,0 +1,212 @@ +## Slack Schema + +.. _slack_schema: + + +### Human + +Slack use Human node as pivot with other Identity Providers (GSuite, GitHub ...) + +Human nodes are not generated by the Slack module; rather, the Slack module establishes a link between SlackUser and pre-existing Human nodes. + + +#### Relationships + +- Human as an access to Slack + ``` + (Human)-[IDENTITY_SLACK]->(SlackUser) + ``` + + +### SlackTeam + +Reprensation of a Slack Workspace. + +| Field | Description | +|-------|--------------| +| firstseen| Timestamp of when a sync job first created this node | +| lastupdated | Timestamp of the last time the node was updated | +| **id** | Slack ID | +| name | Slack workspace name (eg. Lyft OSS) | +| domain | Slack workspace slug (eg. lyftoss) | +| url | Slack workspace full url (eg. https://lyftoss.slack.com) | +| is_verified | Flag for verified Slack workspace (boolean) | +| email_domain | Slack workspace email domain (eg. lyft.com) | + +#### Relationships + +- A SlackTeam contains SlackUser + + ``` + (SlackTeam)-[RESOURCE]->(SlackUser) + ``` + +- A SlackTeam contains SlackChannels + + ``` + (SlackTeam)-[RESOURCE]->(SlackChannel) + ``` + +- A SlackTeam contains SlackGroup + + ``` + (SlackTeam)-[RESOURCE]->(SlackGroup) + ``` + +### SlackUser + +Representation of a single [User in Slack](https://api.slack.com/types/user). + +| Field | Description | +|-------|--------------| +| firstseen| Timestamp of when a sync job first created this node | +| lastupdated | Timestamp of the last time the node was updated | +| **id** | Slack ID | +| name | Slack username (eg. john.doe) | +| real_name | User full name (eg. John Doe) | +| display_name | User displayed name (eg. John D.) | +| first_name | User first name (eg. John) | +| last_name | User last name (eg. Doe) | +| profile_title | User job function (eg. Cybersecurity Manager) | +| profile_phone | User phone number (eg. +33 6 11 22 33 44) | +| profile_email | User email (eg. john.doe@evilcorp.com) | +| deleted | Flag for deleted users (boolean) | +| is_admin | Flag for admin users (boolean) | +| is_owner | Flag for Slack Workspace owners (boolean) | +| is_restricted | Flag for restricted users, aka guests (boolean) | +| is_ultra_restricted | Flag for ultra restricted users, aka guests (boolean) | +| is_bot | Flag for bot user accounts (boolean) | +| is_app_user | Flag for application user accounts (boolean) | +| is_email_confirmed | Flag for user with confirmed email (boolean) | + +#### Relationships + +- A SlackTeam contains SlackUser + + ``` + (SlackTeam)-[RESOURCE]->(SlackUser) + ``` + +- Human as an access to Slack + + ``` + (Human)-[IDENTITY_SLACK]->(SlackUser) + ``` + +- A SlackChannel is created by a SlackUser + + ``` + (SlackUser)-[:CREATED]->(SlackChannel) + ``` + +- A SlackUser is a member of a SlackChannel + + ``` + (SlackUser)-[:MEMBER_OF]->(:SlackChannel) + ``` + +- A SlackUser is member of a SlackGroup + + ``` + (SlackUser)-[:MEMBER_OF]->(SlackGroup) + ``` + +- A SlackGroup is created by a SlackUser + + ``` + (SlackUser)-[CREATED]->(SlackGroup) + ``` + + +### SlackChannel + +Representation of a single [Channel in Slack](https://api.slack.com/types/channel). + +| Field | Description | +|-------|--------------| +| firstseen| Timestamp of when a sync job first created this node | +| lastupdated | Timestamp of the last time the node was updated | +| **id** | Slack ID | +| name | Slack channel name (eg. concern-it) | +| is_private | Flag for private channels (boolean) | +| created | Slack channel creation timestamp | +| is_archived | Flag for archived channels (boolean) | +| is_general | Flag for users default channels (boolean) | +| is_shared | Flag for channels shared with other workspaces | +| is_org_shared | Flag for channel shared with other workspaces in same organization | +| topic | Slack channel topic | +| purpose | Slack channel purpose | + +#### Relationships + +- A SlackTeam contains SlackChannel + + ``` + (SlackTeam)-[RESOURCE]->(SlackChannel) + ``` + +- A SlackChannel is created by a SlackUser + + ``` + (SlackUser)-[:CREATED]->(SlackChannel) + ``` + +- A SlackUser is a member of a SlackChannel + + ``` + (SlackUser)-[:MEMBER_OF]->(:SlackChannel) + ``` + +- A SlackGroup is member of a SlackChannel + + ``` + (SlackChannel)<-[MEMBER_OF]-(SlackGroup) + ``` + + +### SlackGroup + +Representation of a single [Group in Slack](https://api.slack.com/types/usergroup). + +| Field | Description | +|-------|--------------| +| firstseen| Timestamp of when a sync job first created this node | +| lastupdated | Timestamp of the last time the node was updated | +| **id** | Slack ID | +| name | Slack group name (eg. Security Team) | +| description | Slack group description | +| is_subteam | Flag for sub-groups | +| handle | Slack handle (eg. security-team) | +| is_external | Flag for external groups | +| date_create | Slack group creation timestamp | +| date_update | Slack group last update timestamp | +| date_delete | Slack group deletion timestamp | +| updated_by | User ID who has performed last group update | +| user_count | Number of members | +| channel_count | Number of channels where group is member | + +#### Relationships + +- A SlackTeam contains SlackGroup + + ``` + (SlackTeam)-[RESOURCE]->(SlackGroup) + ``` + +- A SlackUser is member of a SlackGroup + + ``` + (SlackUser)-[MEMBER_OF]->(SlackGroup) + ``` + +- A SlackGroup is member of a SlackChannel + + ``` + (SlackChannel)<-[MEMBER_OF]-(SlackGroup) + ``` + +- A SlackGroup is created by a SlackUser + + ``` + (SlackUser)-[CREATED]->(SlackGroup) + ``` diff --git a/setup.py b/setup.py index 28e412511..794817c1b 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ "oauth2client>=4.1.3", "marshmallow>=3.0.0rc7", "oci>=2.71.0", + "slack-sdk>=3.19.2", "okta<1.0.0", "pyyaml>=5.3.1", "requests>=2.22.0", diff --git a/tests/data/slack/__init__.py b/tests/data/slack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/slack/channels.py b/tests/data/slack/channels.py new file mode 100644 index 000000000..470755cec --- /dev/null +++ b/tests/data/slack/channels.py @@ -0,0 +1,97 @@ +SLACK_CHANNELS = { + "channels": [ + { + "id": "SLACKCHANNEL1", + "name": "concern-marketing-comm", + "is_channel": True, + "is_group": False, + "is_im": False, + "is_mpim": False, + "is_private": False, + "created": 1607502101, + "is_archived": False, + "is_general": False, + "unlinked": 0, + "name_normalized": "concern-marketing-comm", + "is_shared": False, + "is_org_shared": False, + "is_pending_ext_shared": False, + "pending_shared": [], + "context_team_id": "TTPQ4FBPT", + "updated": 1647612550943, + "parent_conversation": None, + "creator": "SLACKUSER1", + "is_ext_shared": False, + "shared_team_ids": [ + "TTPQ4FBPT", + ], + "pending_connected_team_ids": [], + "is_member": False, + "topic": { + "value": "", + "creator": "", + "last_set": 0, + }, + "purpose": { + "value": "", + "creator": "", + "last_set": 0, + }, + "previous_names": [ + "sub-communication", + "communication", + ], + "num_members": 2, + }, + { + "id": "SLACKCHANNEL2", + "name": "random", + "is_channel": True, + "is_group": False, + "is_im": False, + "is_mpim": False, + "is_private": False, + "created": 1607340965, + "is_archived": False, + "is_general": False, + "unlinked": 0, + "name_normalized": "random", + "is_shared": False, + "is_org_shared": False, + "is_pending_ext_shared": False, + "pending_shared": [], + "context_team_id": "TTPQ4FBPT", + "updated": 1681184976157, + "parent_conversation": None, + "creator": "SLACKUSER1", + "is_ext_shared": False, + "shared_team_ids": [ + "TTPQ4FBPT", + ], + "pending_connected_team_ids": [], + "is_member": False, + "topic": { + "value": "", + "creator": "", + "last_set": 0, + }, + "purpose": { + "value": "This channel is for... well, everything else.", + "creator": "U01FVSTG8H5", + "last_set": 1607340965, + }, + "previous_names": [ + "misc-random", + "random", + ], + "num_members": 2, + }, + ], +} + +SLACK_CHANNELS_MEMBERSHIPS = { + "members": [ + "SLACKUSER1", + "SLACKUSER2", + ], +} diff --git a/tests/data/slack/team.py b/tests/data/slack/team.py new file mode 100644 index 000000000..ff345ca29 --- /dev/null +++ b/tests/data/slack/team.py @@ -0,0 +1,23 @@ +SLACK_TEAM = { + "team": + + { + "id": "TTPQ4FBPT", + "name": "Lyft OSS", + "url": "https://lyftoss.slack.com/", + "domain": "lyftoss", + "email_domain": "lyft.com", + "icon": { + "image_default": False, + "image_34": "https://avatars.slack-edge.com/2023-01-23/fakeid_34.png", + "image_44": "https://avatars.slack-edge.com/2023-01-23/fakeid_44.png", + "image_68": "https://avatars.slack-edge.com/2023-01-23/fakeid_68.png", + "image_88": "https://avatars.slack-edge.com/2023-01-23/fakeid_88.png", + "image_102": "https://avatars.slack-edge.com/2023-01-23/fakeid_102.png", + "image_230": "https://avatars.slack-edge.com/2023-01-23/fakeid_230.png", + "image_132": "https://avatars.slack-edge.com/2023-01-23/fakeid_132.png", + }, + "avatar_base_url": "https://ca.slack-edge.com/", + "is_verified": False, + }, +} diff --git a/tests/data/slack/usergroups.py b/tests/data/slack/usergroups.py new file mode 100644 index 000000000..e09e86958 --- /dev/null +++ b/tests/data/slack/usergroups.py @@ -0,0 +1,65 @@ +SLACK_USERGROUPS = { + "usergroups": [ + { + "id": "SLACKGROUP1", + "team_id": "TTPQ4FBPT", + "is_usergroup": True, + "is_subteam": True, + "name": "Mobile Dev team", + "description": "", + "handle": "team-dev-mobile", + "is_external": False, + "date_create": 1620383588, + "date_update": 1679498904, + "date_delete": 0, + "auto_type": None, + "auto_provision": False, + "enterprise_subteam_id": "", + "created_by": "SLACKUSER1", + "updated_by": "SLACKUSER1", + "deleted_by": None, + "prefs": { + "channels": [ + "SLACKCHANNEL1", + ], + "groups": [], + }, + "users": [ + "SLACKUSER1", + "SLACKUSER2", + ], + "user_count": 2, + "channel_count": 1, + }, + { + "id": "SLACKGROUP2", + "team_id": "TTPQ4FBPT", + "is_usergroup": True, + "is_subteam": True, + "name": "Security Team", + "description": "", + "handle": "security-team", + "is_external": False, + "date_create": 1624359274, + "date_update": 1648628348, + "date_delete": 0, + "auto_type": None, + "auto_provision": False, + "enterprise_subteam_id": "", + "created_by": "SLACKUSER1", + "updated_by": "SLACKUSER1", + "deleted_by": None, + "prefs": { + "channels": [ + "SLACKCHANNEL2", + ], + "groups": [], + }, + "users": [ + "SLACKUSER1", + ], + "user_count": 1, + "channel_count": 1, + }, + ], +} diff --git a/tests/data/slack/users.py b/tests/data/slack/users.py new file mode 100644 index 000000000..535cade5d --- /dev/null +++ b/tests/data/slack/users.py @@ -0,0 +1,108 @@ +SLACK_USERS = { + "members": [ + { + "id": "SLACKUSER1", + "team_id": "TTPQ4FBPT", + "name": "john.doe", + "deleted": False, + "color": "9f69e7", + "real_name": "John Doe", + "tz": "Europe/Brussels", + "tz_label": "Central European Summer Time", + "tz_offset": 7200, + "profile": { + "title": "CTO", + "phone": "+33122334455", + "skype": "", + "real_name": "John Doe", + "real_name_normalized": "John Doe", + "display_name": "Johnny", + "display_name_normalized": "Johnny", + "fields": None, + "status_text": "", + "status_emoji": "", + "status_emoji_display_info": [], + "status_expiration": 0, + "avatar_hash": "ba0b64b37829", + "image_original": "https://avatars.slack-edge.com/2020-12-07/xxxx_original.png", + "is_custom_image": True, + "email": "john.doe@lyft.com", + "huddle_state": "default_unset", + "huddle_state_expiration_ts": 0, + "first_name": "John", + "last_name": "Doe", + "image_24": "https://avatars.slack-edge.com/2020-12-07/xxxx_24.png", + "image_32": "https://avatars.slack-edge.com/2020-12-07/xxxx_32.png", + "image_48": "https://avatars.slack-edge.com/2020-12-07/xxxx_48.png", + "image_72": "https://avatars.slack-edge.com/2020-12-07/xxxx_72.png", + "image_192": "https://avatars.slack-edge.com/2020-12-07/xxxx_192.png", + "image_512": "https://avatars.slack-edge.com/2020-12-07/xxxx_512.png", + "image_1024": "https://avatars.slack-edge.com/2020-12-07/xxxx_1024.png", + "status_text_canonical": "", + "team": "TTPQ4FBPT", + }, + "is_admin": True, + "is_owner": True, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "is_app_user": False, + "updated": 1682685932, + "is_email_confirmed": True, + "who_can_share_contact_card": "EVERYONE", + }, + { + "id": "SLACKUSER2", + "team_id": "TTPQ4FBPT", + "name": "jane.smith", + "deleted": False, + "color": "4bbe2e", + "real_name": "Jane Smith", + "tz": "Europe/Brussels", + "tz_label": "Central European Summer Time", + "tz_offset": 7200, + "profile": { + "title": "Marketing Manager", + "phone": "1122334455", + "skype": "", + "real_name": "Jane Smith", + "real_name_normalized": "Jane Smith", + "display_name": "Jane", + "display_name_normalized": "Jane", + "fields": None, + "status_text": "", + "status_emoji": "", + "status_emoji_display_info": [], + "status_expiration": 0, + "avatar_hash": "0cc4fc6c25a9", + "image_original": "https://avatars.slack-edge.com/2021-02-08/yyyyy_original.jpg", + "is_custom_image": True, + "email": "jane.smith@lyft.com", + "huddle_state": "default_unset", + "huddle_state_expiration_ts": 0, + "first_name": "Celine", + "last_name": "lazorthes", + "image_24": "https://avatars.slack-edge.com/2021-02-08/yyyyy_24.jpg", + "image_32": "https://avatars.slack-edge.com/2021-02-08/yyyyy_32.jpg", + "image_48": "https://avatars.slack-edge.com/2021-02-08/yyyyy_48.jpg", + "image_72": "https://avatars.slack-edge.com/2021-02-08/yyyyy_72.jpg", + "image_192": "https://avatars.slack-edge.com/2021-02-08/yyyyy_192.jpg", + "image_512": "https://avatars.slack-edge.com/2021-02-08/yyyyy_512.jpg", + "image_1024": "https://avatars.slack-edge.com/2021-02-08/yyyyy_1024.jpg", + "status_text_canonical": "", + "team": "TTPQ4FBPT", + }, + "is_admin": False, + "is_owner": False, + "is_primary_owner": False, + "is_restricted": False, + "is_ultra_restricted": False, + "is_bot": False, + "is_app_user": False, + "updated": 1683289214, + "is_email_confirmed": True, + "who_can_share_contact_card": "EVERYONE", + }, + ], +} diff --git a/tests/integration/cartography/intel/slack/__init__.py b/tests/integration/cartography/intel/slack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/cartography/intel/slack/test_channels.py b/tests/integration/cartography/intel/slack/test_channels.py new file mode 100644 index 000000000..2bbd4da3a --- /dev/null +++ b/tests/integration/cartography/intel/slack/test_channels.py @@ -0,0 +1,99 @@ +from unittest.mock import Mock + +import cartography.intel.slack.channels +import cartography.intel.slack.team +import cartography.intel.slack.users +import tests.data.slack.channels +import tests.data.slack.team +import tests.data.slack.users +from tests.integration.util import check_nodes +from tests.integration.util import check_rels + +SLACK_TEAM_ID = 'TTPQ4FBPT' +SLACK_TOKEN = 'fake-token' +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG, "TEAM_ID": SLACK_TEAM_ID, 'CHANNELS_MEMBERSHIPS': True} + + +def test_load_slack_channels(neo4j_session): + """ + Ensure that channels actually get loaded + """ + + slack_client = Mock( + team_info=Mock(return_value=tests.data.slack.team.SLACK_TEAM), + users_list=Mock(return_value=tests.data.slack.users.SLACK_USERS), + conversations_list=Mock(return_value=tests.data.slack.channels.SLACK_CHANNELS), + conversations_members=Mock(return_value=tests.data.slack.channels.SLACK_CHANNELS_MEMBERSHIPS), + ) + + # Act + cartography.intel.slack.team.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.users.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.channels.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert Channels exists + expected_nodes = { + ('SLACKCHANNEL2', 'random'), + ('SLACKCHANNEL1', 'concern-marketing-comm'), + } + assert check_nodes(neo4j_session, 'SlackChannel', ['id', 'name']) == expected_nodes + + # Assert Channels are connected to team + expected_rels = { + ('SLACKCHANNEL2', SLACK_TEAM_ID), + ('SLACKCHANNEL1', SLACK_TEAM_ID), + } + assert check_rels( + neo4j_session, + 'SlackChannel', 'id', + 'SlackTeam', 'id', + 'RESOURCE', + rel_direction_right=False, + ) == expected_rels + + # Assert Channels are connected to Creator + expected_rels = { + ('SLACKCHANNEL2', 'SLACKUSER1'), + ('SLACKCHANNEL1', 'SLACKUSER1'), + } + assert check_rels( + neo4j_session, + 'SlackChannel', 'id', + 'SlackUser', 'id', + 'CREATED', + rel_direction_right=False, + ) == expected_rels + + # Assert Channels are connected to Members + expected_rels = { + ('SLACKCHANNEL2', 'SLACKUSER1'), + ('SLACKCHANNEL1', 'SLACKUSER1'), + ('SLACKCHANNEL2', 'SLACKUSER2'), + ('SLACKCHANNEL1', 'SLACKUSER2'), + } + assert check_rels( + neo4j_session, + 'SlackChannel', 'id', + 'SlackUser', 'id', + 'MEMBER_OF', + rel_direction_right=False, + ) == expected_rels diff --git a/tests/integration/cartography/intel/slack/test_groups.py b/tests/integration/cartography/intel/slack/test_groups.py new file mode 100644 index 000000000..b86ee609f --- /dev/null +++ b/tests/integration/cartography/intel/slack/test_groups.py @@ -0,0 +1,121 @@ +from unittest.mock import Mock + +import cartography.intel.slack.channels +import cartography.intel.slack.groups +import cartography.intel.slack.team +import cartography.intel.slack.users +import tests.data.slack.channels +import tests.data.slack.team +import tests.data.slack.usergroups +import tests.data.slack.users +from tests.integration.util import check_nodes +from tests.integration.util import check_rels + +SLACK_TEAM_ID = 'TTPQ4FBPT' +SLACK_TOKEN = 'fake-token' +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG, "TEAM_ID": SLACK_TEAM_ID, 'CHANNELS_MEMBERSHIPS': True} + + +def test_load_slack_groups(neo4j_session): + """ + Ensure that users actually get loaded + """ + + slack_client = Mock( + team_info=Mock(return_value=tests.data.slack.team.SLACK_TEAM), + users_list=Mock(return_value=tests.data.slack.users.SLACK_USERS), + conversations_list=Mock(return_value=tests.data.slack.channels.SLACK_CHANNELS), + conversations_members=Mock(return_value=tests.data.slack.channels.SLACK_CHANNELS_MEMBERSHIPS), + usergroups_list=Mock(return_value=tests.data.slack.usergroups.SLACK_USERGROUPS), + ) + + # Act + cartography.intel.slack.team.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.users.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.channels.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.groups.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert groups exists + expected_nodes = { + ('SLACKGROUP1', 'Mobile Dev team'), + ('SLACKGROUP2', 'Security Team'), + } + assert check_nodes(neo4j_session, 'SlackGroup', ['id', 'name']) == expected_nodes + + # Assert groups are connected to team + expected_rels = { + ('SLACKGROUP1', SLACK_TEAM_ID), + ('SLACKGROUP2', SLACK_TEAM_ID), + } + assert check_rels( + neo4j_session, + 'SlackGroup', 'id', + 'SlackTeam', 'id', + 'RESOURCE', + rel_direction_right=False, + ) == expected_rels + + # Assert groups are connected to Creator + expected_rels = { + ('SLACKGROUP1', 'SLACKUSER1'), + ('SLACKGROUP2', 'SLACKUSER1'), + } + assert check_rels( + neo4j_session, + 'SlackGroup', 'id', + 'SlackUser', 'id', + 'CREATED', + rel_direction_right=False, + ) == expected_rels + + # Assert groups are connected to Members + expected_rels = { + ('SLACKGROUP1', 'SLACKUSER1'), + ('SLACKGROUP2', 'SLACKUSER1'), + ('SLACKGROUP1', 'SLACKUSER2'), + } + assert check_rels( + neo4j_session, + 'SlackGroup', 'id', + 'SlackUser', 'id', + 'MEMBER_OF', + rel_direction_right=False, + ) == expected_rels + + # Assert groups are connected to channels + expected_rels = { + ('SLACKGROUP1', 'SLACKCHANNEL1'), + ('SLACKGROUP2', 'SLACKCHANNEL2'), + } + assert check_rels( + neo4j_session, + 'SlackGroup', 'id', + 'SlackChannel', 'id', + 'MEMBER_OF', + rel_direction_right=True, + ) == expected_rels diff --git a/tests/integration/cartography/intel/slack/test_team.py b/tests/integration/cartography/intel/slack/test_team.py new file mode 100644 index 000000000..e401515a0 --- /dev/null +++ b/tests/integration/cartography/intel/slack/test_team.py @@ -0,0 +1,35 @@ +from unittest.mock import Mock + +import cartography.intel.slack.team +import tests.data.slack.team +from tests.integration.util import check_nodes + +SLACK_TEAM_ID = 'TTPQ4FBPT' +SLACK_TOKEN = 'fake-token' +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG, "TEAM_ID": SLACK_TEAM_ID, 'CHANNELS_MEMBERSHIPS': True} + + +def test_load_slack_team(neo4j_session): + """ + Ensure that users actually get loaded + """ + + slack_client = Mock( + team_info=Mock(return_value=tests.data.slack.team.SLACK_TEAM), + ) + + # Act + cartography.intel.slack.team.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert Team exists + expected_nodes = { + (SLACK_TEAM_ID, 'Lyft OSS'), + } + assert check_nodes(neo4j_session, 'SlackTeam', ['id', 'name']) == expected_nodes diff --git a/tests/integration/cartography/intel/slack/test_users.py b/tests/integration/cartography/intel/slack/test_users.py new file mode 100644 index 000000000..df21cf8a0 --- /dev/null +++ b/tests/integration/cartography/intel/slack/test_users.py @@ -0,0 +1,101 @@ +from unittest.mock import Mock + +import cartography.intel.slack.team +import cartography.intel.slack.users +import tests.data.slack.team +import tests.data.slack.users +from tests.integration.util import check_nodes +from tests.integration.util import check_rels + +SLACK_TEAM_ID = 'TTPQ4FBPT' +SLACK_TOKEN = 'fake-token' +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG, "TEAM_ID": SLACK_TEAM_ID, 'CHANNELS_MEMBERSHIPS': True} + + +def test_load_slack_users(neo4j_session): + """ + Ensure that users actually get loaded + """ + + slack_client = Mock( + team_info=Mock(return_value=tests.data.slack.team.SLACK_TEAM), + users_list=Mock(return_value=tests.data.slack.users.SLACK_USERS), + ) + + # Arrange + # Slack intel only link users to existing Humans (created by other module like Gsuite) + # We have to create mock humans to tests rels are well created by Slack module + query = """ + UNWIND $UserData as user + + MERGE (h:Human{id: user}) + ON CREATE SET h.firstseen = timestamp() + SET h.email = user, + h.email = user, + h.lastupdated = $UpdateTag + """ + data = [] + for v in tests.data.slack.users.SLACK_USERS['members']: + data.append(v['profile']['email']) + neo4j_session.run( + query, + UserData=data, + UpdateTag=TEST_UPDATE_TAG, + ) + + # Act + cartography.intel.slack.team.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.slack.users.sync( + neo4j_session, + slack_client, + SLACK_TEAM_ID, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert Human exists + expected_nodes = { + ('john.doe@lyft.com', 'john.doe@lyft.com'), + ('jane.smith@lyft.com', 'jane.smith@lyft.com'), + } + assert check_nodes(neo4j_session, 'Human', ['id', 'email']) == expected_nodes + + # Assert Users exists + expected_nodes = { + ('SLACKUSER1', 'john.doe@lyft.com'), + ('SLACKUSER2', 'jane.smith@lyft.com'), + } + assert check_nodes(neo4j_session, 'SlackUser', ['id', 'email']) == expected_nodes + + # Assert Users are connected with Team + expected_rels = { + ('SLACKUSER1', SLACK_TEAM_ID), + ('SLACKUSER2', SLACK_TEAM_ID), + } + assert check_rels( + neo4j_session, + 'SlackUser', 'id', + 'SlackTeam', 'id', + 'RESOURCE', + rel_direction_right=False, + ) == expected_rels + + # Assert Users are connected with Humans + expected_rels = { + ('SLACKUSER1', 'john.doe@lyft.com'), + ('SLACKUSER2', 'jane.smith@lyft.com'), + } + assert check_rels( + neo4j_session, + 'SlackUser', 'id', + 'Human', 'email', + 'IDENTITY_SLACK', + rel_direction_right=False, + ) == expected_rels