New Intel: Slack

README.md

@@ -34,6 +34,7 @@ Start [here](https://lyft.github.io/cartography/install.html).
 - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
 - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
+- [Slack](https://lyft.github.io/cartography/modules/slack/index.html) - Users, Groups, Channels
 
 
 ## Usage

cartography/cli.py

@@ -500,6 +500,22 @@ def _build_parser(self):
                 'The Duo api hostname'
             ),
         )
+        parser.add_argument(
+            '--slack-token-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the Slack Token'
+            ),
+        )
+        parser.add_argument(
+            '--slack-teams',
+            type=str,
+            default=None,
+            help=(
+                'The Slack Team ID to sync, comma separated.'
+            ),
+        )
         return parser
 
     def main(self, argv: str) -> int:
@@ -666,6 +682,13 @@ def main(self, argv: str) -> int:
             config.duo_api_key = os.environ.get(config.duo_api_key_env_var)
             config.duo_api_secret = os.environ.get(config.duo_api_secret_env_var)
 
+        # Slack config
+        if config.slack_token_env_var:
+            logger.debug(
+                f"Reading Slack token from environment variables {config.slack_token_env_var}",
+            )
+            config.slack_token = os.environ.get(config.slack_token_env_var)
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -103,6 +103,10 @@ class Config:
     :param duo_api_key: The Duo api secret. Optional.
     :type duo_api_hostname: str
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
+    :type slack_token: str
+    :param slack_token: Slack API Token. Optional.
+    :type slack_teams: str
+    :param slack_teams: Comma-separated list of Slack teams to sync. Optional.
     """
 
     def __init__(
@@ -157,6 +161,8 @@ def __init__(
         duo_api_key=None,
         duo_api_secret=None,
         duo_api_hostname=None,
+        slack_token=None,
+        slack_teams=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -208,3 +214,5 @@ def __init__(
         self.duo_api_key = duo_api_key
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
+        self.slack_token = slack_token
+        self.slack_teams = slack_teams

cartography/intel/slack/__init__.py

@@ -0,0 +1,71 @@
+import logging
+
+import neo4j
+from slack_sdk import WebClient
+from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler
+
+import cartography.intel.slack.channels
+import cartography.intel.slack.groups
+import cartography.intel.slack.team
+import cartography.intel.slack.users
+from cartography.config import Config
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_slack_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Slack data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.slack_token or not config.slack_teams:
+        logger.info(
+            'Slack import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    rate_limit_handler = RateLimitErrorRetryHandler(max_retry_count=1)
+    slack_client = WebClient(token=config.slack_token)
+    slack_client.retry_handlers.append(rate_limit_handler)
+
+    for team_id in config.slack_teams.split(','):
+        logger.info("Syncing team %s", team_id)
+        common_job_parameters['TEAM_ID'] = team_id
+        cartography.intel.slack.team.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )
+        cartography.intel.slack.users.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )
+        cartography.intel.slack.channels.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )
+        cartography.intel.slack.groups.sync(
+            neo4j_session,
+            slack_client,
+            team_id,
+            config.update_tag,
+            common_job_parameters,
+        )

cartography/intel/slack/channels.py

@@ -0,0 +1,78 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.slack.channels import SlackChannelSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    channels = get(slack_client, team_id)
+    load_channels(neo4j_session, channels, team_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(slack_client: WebClient, team_id: str, cursor: Optional[str] = None) -> List[Dict[str, Any]]:
+    channels: List[Dict[str, Any]] = []
+    channels_info = slack_client.conversations_list(cursor=cursor, team_id=team_id)
+    for m in channels_info['channels']:
+        channels.append(m)
+        if m['is_archived']:
+            channels.append(m)
+        else:
+            for i in _get_membership(slack_client, m['id']):
+                channel_m = m.copy()
+                channel_m['member_id'] = i
+                channels.append(channel_m)
+    next_cursor = channels_info.get('response_metadata', {}).get('next_cursor', '')
+    if next_cursor != '':
+        channels.extend(get(slack_client, team_id, cursor=next_cursor))
+    return channels
+
+
+def _get_membership(slack_client: WebClient, slack_channel: str, cursor: Optional[str] = None) -> List[str]:
+    result = []
+    memberships = slack_client.conversations_members(channel=slack_channel, cursor=cursor)
+    for m in memberships['members']:
+        result.append(m)
+    next_cursor = memberships.get('response_metadata', {}).get('next_cursor', '')
+    if next_cursor != '':
+        result.extend(_get_membership(slack_client, slack_channel, cursor=next_cursor))
+    return result
+
+
+def load_channels(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    team_id: str,
+    update_tag: int,
+) -> None:
+
+    load(
+        neo4j_session,
+        SlackChannelSchema(),
+        data,
+        lastupdated=update_tag,
+        TEAM_ID=team_id,
+    )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    GraphJob.from_node_schema(SlackChannelSchema(), common_job_parameters).run(neo4j_session)

cartography/intel/slack/groups.py

@@ -0,0 +1,82 @@
+import logging
+from itertools import zip_longest
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.slack.group import SlackGroupSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    groups = get(slack_client, team_id)
+    formated_groups = transform(groups)
+    load_groups(neo4j_session, formated_groups, team_id, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(slack_client: WebClient, team_id: str, cursor: Optional[str] = None) -> List[Dict[str, Any]]:
+    groups: List[Dict[str, Any]] = []
+    groups_info = slack_client.usergroups_list(
+        cursor=cursor,
+        include_count=True,
+        include_users=True,
+        include_disabled=True,
+        team_id=team_id,
+    )
+    for g in groups_info['usergroups']:
+        groups.append(g)
+    next_cursor = groups_info.get('response_metadata', {}).get('next_cursor', '')
+    if next_cursor != '':
+        groups.extend(get(slack_client, team_id, cursor=next_cursor))
+    return groups
+
+
+@timeit
+def transform(groups: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    splitted_groups: List[Dict[str, Any]] = []
+    for group in groups:
+        for ms in zip_longest(group['users'], group['prefs']['channels']):
+            formated_group = group.copy()
+            formated_group.pop('users')
+            formated_group.pop('prefs')
+            formated_group['member_id'] = ms[0]
+            formated_group['channel_id'] = ms[1]
+            splitted_groups.append(formated_group)
+    return splitted_groups
+
+
+def load_groups(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    team_id: str,
+    update_tag: int,
+) -> None:
+
+    load(
+        neo4j_session,
+        SlackGroupSchema(),
+        data,
+        lastupdated=update_tag,
+        TEAM_ID=team_id,
+    )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    GraphJob.from_node_schema(SlackGroupSchema(), common_job_parameters).run(neo4j_session)

cartography/intel/slack/team.py

@@ -0,0 +1,44 @@
+import logging
+from typing import Any
+from typing import Dict
+
+import neo4j
+from slack_sdk import WebClient
+
+from cartography.client.core.tx import load
+from cartography.models.slack.team import SlackTeamSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    slack_client: WebClient,
+    team_id: str,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    team = get(slack_client, team_id)
+    load_team(neo4j_session, team, update_tag)
+
+
+@timeit
+def get(slack_client: WebClient, team_id: str) -> Dict[str, Any]:
+    team_info = slack_client.team_info(team_id=team_id)
+    return team_info['team']
+
+
+def load_team(
+    neo4j_session: neo4j.Session,
+    data: Dict[str, Any],
+    update_tag: int,
+) -> None:
+
+    load(
+        neo4j_session,
+        SlackTeamSchema(),
+        [data],
+        lastupdated=update_tag,
+    )