From 133a052e056ce09655a5bfc4df2590de62e8baaa Mon Sep 17 00:00:00 2001 From: Will G Date: Mon, 27 Nov 2023 00:29:55 -0500 Subject: [PATCH] Updated Readme; Added Agent Responses for Tasks --- README.md | 160 +++++++++++++++++--- runbooksolutions/agent/API.py | 9 +- runbooksolutions/agent/PluginManager.py | 6 +- runbooksolutions/agent/PluginManager.py.old | 126 --------------- runbooksolutions/queue/Queue.py | 2 +- 5 files changed, 153 insertions(+), 150 deletions(-) delete mode 100644 runbooksolutions/agent/PluginManager.py.old diff --git a/README.md b/README.md index 5ef8657..aab0731 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,57 @@ -# RunbookSolution Network Agent +# RunbookSolution Agent + +This codebase comprises the core of the Agent for RunbookSolutions. + +- [Introduction](#introduction) +- [Installation](#installation) + - [Prebuilt Docker Image](#prebuilt-docker-image) + - [Extending the Default Image](#extending-the-default-image) + - [From Source](#from-source) +- [Configuration](#configuration) + - [`config.ini` Parameters](#configini-parameters) + - [Example Configuration](#example-configuration) +- [API Requests](#api-requests) +- [Additional Notes](#additional-notes) + - [Creating a Keytab File](#creating-a-keytab-file) + - [Security Considerations](#security-considerations) + + +## Introduction + +The RunbookSolution Agent serves as a pivotal component within the RunbookSolutions documentation system. +Designed as a local daemon, its primary objective is to conduct scans, identifying crucial facts and features related to networks and devices. +What sets it apart is its ability to seamlessly relay these results to the backend without necessitating the installation of a client on each device. + +To achieve the required flexibility, the Agent operates on a plugin-based architecture. +Plugins, obtained from a remote server on demand, empower the Agent with diverse functionalities, ensuring the deployment of new features and scans without manual intervention on the Agent itself. + +Key components that constitute the Agent's core functionality include: + +- **Auth**: Facilitates agent registration and authentication with the backend using the OAuth2 "Device Flow". +- **Stores**: Enables the Agent to persist information about its current state, ensuring data retention through restarts. +- **Queue**: Manages a task queue and efficiently executes tasks in a threadpool. +- **Schedule**: Allows the scheduling of tasks at regular intervals, employing cron notation. + +In addition to these, the Agent incorporates the following components: + +- **API**: Establishes communication with the backend server. +- **PluginManager**: Manages the retrieval and execution of plugins. -This codebase comprises the core of the Network Agent for RunbookSolutions. ## Installation +To deploy the RunbookSolution Agent, you have several options depending on your requirements. Follow the instructions below based on your preferred method: ### Prebuilt Docker Image +If you prefer a quick and straightforward installation using Docker, follow these steps: ```sh +# Create necessary directories mkdir agent cd agent mkdir plugins stores kerberos -wget https://raw.githubusercontent.com/RunbookSolutions/agent/staging/config.ini +wget https://raw.githubusercontent.com/RunbookSolutions/agent/production/config.ini +# Run the Docker container docker run \ --name RunbookSolutions_Agent \ -v $(pwd)/config.ini:/app/config.ini \ @@ -22,12 +62,13 @@ docker run \ -d \ --restart unless-stopped \ runbooksolutions/agent:latest - ``` ### Extending the Default Image -The default image includes the following Python libraries by default. To include additional libraries for your custom plugins, simply extend our default image and use it instead. +If you need additional Python libraries for custom plugins, you can extend the default image. Follow these steps: + +Create your custom Docker file: ```Dockerfile FROM runbooksolutions/agent:latest @@ -38,41 +79,110 @@ RUN pip install -r custom_requirements.txt RUN pip install some_package ``` +Build and run the customized Docker image: + +```sh +docker build . --tag YOUR_NAME_OR_COMPANY/agent:latest + +docker run \ + --name RunbookSolutions_Agent \ + -v $(pwd)/config.ini:/app/config.ini \ + -v $(pwd)/plugins:/app/plugins \ + -v $(pwd)/stores:/app/stores \ + -v $(pwd)/kerberos/krb5.conf:/etc/krb5.conf \ + -v $(pwd)/kerberos:/keytabs \ + -d \ + --restart unless-stopped \ + YOUR_NAME_OR_COMPANY/agent:latest +``` + ### From Source +If you prefer building from the source code, execute the following commands: + ```sh git clone https://github.com/RunbookSolutions/agent.git cd agent ./run ``` +Ensure you have the necessary dependencies installed before running the build. + +These installation options provide flexibility based on your specific needs, whether you opt for a Dockerized setup, extend the default image, or build from the source code. Choose the method that aligns with your deployment preferences and system requirements. + ## Configuration -Configuration is maintained in a simple 'config.ini' file consisting of the 'server_url' of the backend and the 'client_id' for device authentication. +Configuring the RunbookSolution Agent is a crucial step to tailor its behavior to your specific environment. The configuration is maintained in a straightforward 'config.ini' file, which includes essential parameters for seamless operation. Here's a breakdown of the key configuration options: + +### `config.ini` Parameters + +- `server_url`: Specifies the URL of the backend server. Avoid including a trailing slash in the URL. +```ini +[agent] +server_url=http://192.168.1.197 +``` +- `client_id`: Identifies the device during the authentication process. This is the Device Code Grant client_id provided by the server. +```ini +[agent] +client_id=9ab55261-bfb7-4bb3-ad29-a6dbdbf8a5af +``` +- `auth`: Enables or disables authentication when not using RunbookSolutions. Set to True to enable authentication. +```ini +[agent] +auth=True +``` + +### Example Configuration +Here's an example configuration snippet illustrating how to structure the 'config.ini' file: ```ini [agent] -server_url=http://192.168.1.197 # Note: Do NOT include a trailing slash on the server_url -client_id=9ab55261-bfb7-4bb3-ad29-a6dbdbf8a5af # Device Code Grant client_id provided by the server -auth=True # To disable auth when not using with RunbookSolutions. +server_url=http://your-backend-url.com +client_id=your-client-id +auth=True ``` -## Expected Server Responses -Due to the agent's nature, it can easily be used by others outside of RunbookSolutions. -To implement a backend for this agent, you will need to provide the following endpoints. +Adjust the values according to your backend server's URL, client ID, and authentication preference. + +The 'config.ini' file plays a pivotal role in customizing the RunbookSolution Agent's interaction with the backend and authentication mechanisms. Ensure accurate configuration to seamlessly integrate the Agent into your network and device environment. -`GET /api/agent` for the agent to load information about itself. This endpoint also provides the agent with a list of PLUGIN_IDs that it needs to load. +## API Requests +The RunbookSolution Agent communicates with the backend server through a set of well-defined API endpoints. Understanding these API requests is essential for implementing a custom backend or extending the functionality of the Agent. Below are the key endpoints and their purposes: -`GET /api/agent/plugin/{PLUGIN_ID}` for the agent to download plugins. This endpoint also provides details about commands the plugin provides. +- `GET /api/agent` + - **Purpose**: This endpoint allows the Agent to load information about itself from the backend. Additionally, it provides a list of PLUGIN_IDs that the Agent needs to load. -`GET /api/agent/tasks` for the agent to load tasks that it needs to run. Tasks include scheduled and one-off tasks to run and will always present tasks until they are removed from the backend. This allows the agent to restart without skipping task execution. +- `GET /api/agent/plugin/{PLUGIN_ID}` + - **Purpose**: Used by the Agent to download plugins dynamically from the backend. This endpoint also provides details about the commands the plugin offers. -Additional details can be found on the [Expected Server Responses](/docs/Responses.md) page. +- `GET /api/agent/tasks` + - **Purpose**: Enables the Agent to load tasks that need to be executed. Tasks include both scheduled and one-off tasks, ensuring continuous operation even after restarts. -## Creating a Keytab File +- `POST /api/agent/task/{TASK_ID}` + - **Purpose**: Enables the Agent to send the results of a task to the backend. + - **Example**: + ```sh + curl -X POST http://YOUR_BACKEND_URL/api/agent/task/TASK_ID \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \ + -d '{"data": "{\"key\": \"value\", \"another_key\": \"another_value\"}"}' + ``` + In this example: -Some plugins may require authentication against your windows domain. + - The URL should be customized with your backend server's details and the actual {TASK_ID}. + - The `-H "Authorization: Bearer YOUR_ACCESS_TOKEN"` header includes the necessary bearer token for authentication. + - The `-d '{"data": "{\"key\": \"value\", \"another_key\": \"another_value\"}"}'` part represents the payload of the POST request. The data variable contains JSON-encoded data representing the results of the executed task. -The simplest way to acomplish this is by using the [Docker Kerberos Keytab Generator](https://github.com/simplesteph/docker-kerberos-get-keytab): +For those implementing a custom backend for the Agent, it's crucial to provide these endpoints to facilitate smooth communication. A deeper dive into the Expected Server Responses page will offer additional details on the expected responses from these endpoints. + +Understanding these API requests empowers users to integrate the RunbookSolution Agent seamlessly into their infrastructure or develop custom functionalities, enhancing the overall capabilities of the system. + +## Additional Notes + +### Creating a Keytab File + +Authentication against a Windows domain may require the use of keytab files, especially for certain plugins. Follow these steps using the [Docker Kerberos Keytab Generator](https://github.com/simplesteph/docker-kerberos-get-keytab) for a straightforward keytab file creation: + +This Docker container facilitates the generation of keytab files, an essential aspect of some plugins' authentication against Windows domains. ```sh cd agent @@ -80,4 +190,14 @@ docker run -it --rm \ -v $(pwd)/kerberos:/output \ -e PRINCIPAL= \ simplesteph/docker-kerberos-get-keytab -``` \ No newline at end of file +``` + +### Security Considerations +When generating keytab files or implementing authentication-related configurations, prioritize security practices. Ensure that sensitive information, such as authentication credentials, is handled and stored securely. Review and adhere to best practices for securing keytab files and associated authentication processes within your network environment. + +By following these guidelines, you enhance the overall security posture of your RunbookSolution Agent deployment, minimizing potential risks associated with authentication processes and ensuring a robust and secure system. + + + + + diff --git a/runbooksolutions/agent/API.py b/runbooksolutions/agent/API.py index b8ba4c4..bd0222c 100644 --- a/runbooksolutions/agent/API.py +++ b/runbooksolutions/agent/API.py @@ -43,6 +43,13 @@ def getAgentDetails(self) -> AgentDetails: def getAgentTasks(self) -> AgentTasks: return AgentTasks(self.sendRequest('/agent/tasks', 'GET')) + + def sendTaskResult(self, task: Task, result: any): + self.sendRequest( + f'/agent/task/{task.id}', + 'POST', + {'data':result} + ) def sendRequest(self, url, method, data=None): url = self.url + url @@ -62,7 +69,7 @@ def sendRequest(self, url, method, data=None): raise ValueError("Invalid HTTP method. Supported methods are GET, POST, PUT, and DELETE.") # You might want to handle response status codes and raise exceptions if needed - if response.status_code != 200: + if response.status_code != 200 and response.status_code != 201: raise Exception(f"Request failed with status code {response.status_code}. Response content: {response.text}") return response.json() # Assuming the response is in JSON format \ No newline at end of file diff --git a/runbooksolutions/agent/PluginManager.py b/runbooksolutions/agent/PluginManager.py index f6d3d08..9889cc2 100644 --- a/runbooksolutions/agent/PluginManager.py +++ b/runbooksolutions/agent/PluginManager.py @@ -1,5 +1,6 @@ from runbooksolutions.agent.Plugin import Plugin from runbooksolutions.agent.API import API +from runbooksolutions.agent.Task import Task import json import logging import os @@ -149,7 +150,7 @@ def loadPluginCommands(self, pluginID: str) -> None: self.loadedCommands.update({command_name: modified_command}) - def executeCommand(self, commandName, *args, **kwargs) -> None: + def executeCommand(self, task: Task, commandName: str, *args, **kwargs) -> None: if not self.commandIsLoaded(commandName): logging.critical(f"Tried to call {commandName} when it wasn't loaded") return @@ -159,6 +160,7 @@ def executeCommand(self, commandName, *args, **kwargs) -> None: function_to_call = getattr(self.plugins.get(pluginID), function_name, None) if callable(function_to_call): - function_to_call(*args, **kwargs) + result = function_to_call(*args, **kwargs) + self.api.sendTaskResult(task, result) else: print(f"Function {function_name} not found in plugin {pluginID}.") \ No newline at end of file diff --git a/runbooksolutions/agent/PluginManager.py.old b/runbooksolutions/agent/PluginManager.py.old deleted file mode 100644 index db5dfa5..0000000 --- a/runbooksolutions/agent/PluginManager.py.old +++ /dev/null @@ -1,126 +0,0 @@ -import importlib -import json -import requests -import hashlib -import os - -class PluginManager: - def __init__(self, plugin_directory="plugins"): - self.plugin_directory = plugin_directory - self.loaded_plugins = {} - self.available_commands = set() - - def load_plugin(self, plugin_name, plugin_url=None): - if plugin_name in self.loaded_plugins: - print(f"Plugin {plugin_name} is already loaded.") - return - - # If plugin is not available locally, perform HTTP request to get it - if plugin_url: - response = requests.get(plugin_url) - if response.status_code == 200: - plugin_data = response.json() - # Verify the integrity of the received plugin using hash - if self.verify_plugin_hash(plugin_data): - self.save_plugin_locally(plugin_name, plugin_data) - self.update_available_commands(plugin_data) - self.loaded_plugins[plugin_name] = self.import_plugin(plugin_name) - print(f"Plugin {plugin_name} loaded successfully.") - else: - print("Plugin integrity verification failed. Aborting.") - else: - print(f"Failed to fetch plugin {plugin_name}. HTTP Error {response.status_code}.") - else: - print(f"Local copy of plugin {plugin_name} not found and no URL provided.") - - def sync_plugins(self, plugin_list): - # Unload plugins not in the provided list - plugins_to_unload = set(self.loaded_plugins.keys()) - set(plugin_list) - for plugin_name in plugins_to_unload: - self.unload_plugin(plugin_name) - - # Load plugins in the provided list - for plugin_name in plugin_list: - if self.is_command_available(plugin_name): - self.load_plugin(plugin_name) - - def unload_plugin(self, plugin_name): - if plugin_name in self.loaded_plugins: - del self.loaded_plugins[plugin_name] - print(f"Plugin {plugin_name} unloaded successfully.") - else: - print(f"Plugin {plugin_name} is not loaded.") - - def get_available_commands(self): - return self.available_commands - - def is_command_available(self, command_name): - return command_name in self.available_commands - - def execute_command(self, command_name, *args, **kwargs): - if self.is_command_available(command_name): - for plugin_name, plugin_data in self.loaded_plugins.items(): - plugin_commands = plugin_data.get('commands', {}) - if command_name in plugin_commands: - function_name = plugin_commands[command_name] - function_to_call = getattr(plugin_data['instance'], function_name, None) - if callable(function_to_call): - function_to_call(*args, **kwargs) - else: - print(f"Function {function_name} not found in plugin {plugin_name}.") - return - print(f"Command {command_name} not found in any loaded plugins.") - else: - print(f"Command {command_name} is not available.") - - def verify_plugin_hash(self, plugin_data): - script = plugin_data.get('script', '').encode('utf-8') - calculated_hash = hashlib.sha256(script).hexdigest() - return calculated_hash == plugin_data.get('hash', '') - - def save_plugin_locally(self, plugin_name, plugin_data): - if not os.path.exists(self.plugin_directory): - os.makedirs(self.plugin_directory) - - # Save JSON data - json_file_path = os.path.join(self.plugin_directory, f"{plugin_name}.json") - with open(json_file_path, 'w') as json_file: - json.dump(plugin_data, json_file) - - # Save Python script - script_file_path = os.path.join(self.plugin_directory, f"{plugin_name}.py") - with open(script_file_path, 'w') as script_file: - script_file.write(plugin_data.get('script', '')) - - def import_plugin(self, plugin_name): - try: - module_name = os.path.splitext(plugin_name)[0] - spec = importlib.util.spec_from_file_location(module_name, os.path.join(self.plugin_directory, f"{plugin_name}.json")) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Store the instance of the plugin in the loaded_plugins dictionary - module.instance = module.Plugin() - return module - except Exception as e: - print(f"Error importing plugin {plugin_name}: {e}") - return None - - def update_available_commands(self, plugin_data): - commands = plugin_data.get('commands', {}).keys() - self.available_commands.update(commands) - -# Example usage: -if __name__ == "__main__": - plugin_manager = PluginManager() - - # Load plugins based on available commands - plugin_list = ["NMAP Plugin", "Another Plugin"] - plugin_manager.sync_plugins(plugin_list) - - # Get available commands - commands = plugin_manager.get_available_commands() - print("Available commands:", commands) - - # Execute a command - command_to_execute = "COMMAND_NAME" - plugin_manager.execute_command(command_to_execute) diff --git a/runbooksolutions/queue/Queue.py b/runbooksolutions/queue/Queue.py index 1ca8791..e2ee0ec 100644 --- a/runbooksolutions/queue/Queue.py +++ b/runbooksolutions/queue/Queue.py @@ -34,7 +34,7 @@ def execute_task(self, task: Task) -> None: # Implement your task execution logic here logging.info(f"Running Task \'{command}\' with arguments {arguments}") - self.pluginManager.executeCommand(command, **arguments) + self.pluginManager.executeCommand(task, command, **arguments) pass async def start(self) -> None: