From 8f6725203905d82523fc0ff89a3f9ef2557c17af Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:25:57 -0600 Subject: [PATCH 01/11] Rework utils file to directory --- src/osintbuddy/utils.py | 38 ------------ src/osintbuddy/utils/__init__.py | 9 +++ src/osintbuddy/utils/generic.py | 101 +++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 38 deletions(-) delete mode 100644 src/osintbuddy/utils.py create mode 100644 src/osintbuddy/utils/__init__.py create mode 100644 src/osintbuddy/utils/generic.py diff --git a/src/osintbuddy/utils.py b/src/osintbuddy/utils.py deleted file mode 100644 index 619d6e6..0000000 --- a/src/osintbuddy/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import re -import unicodedata - - -# Slugify and related code is from the Django project, thanks guys! -# Project URL: https://github.com/django/django -# https://github.com/django/django/blob/main/django/utils/text.py -def slugify(value, allow_unicode=False): - """ - Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated - dashes to single dashes. Remove characters that aren't alphanumerics, - underscores, or hyphens. Convert to lowercase. Also strip leading and - trailing whitespace, dashes, and underscores. - """ - value = str(value) - if allow_unicode: - value = unicodedata.normalize("NFKC", value) - else: - value = ( - unicodedata.normalize("NFKD", value) - .encode("ascii", "ignore") - .decode("ascii") - ) - value = re.sub(r"[^\w\s-]", "", value.lower()) - return re.sub(r"[-\s]+", "-", value).strip("-_") - - -def to_camel_case(value: str): - value_list = value.replace(' ', '_').lower().split('_') - return value_list[0] + ''.join(e.title() for e in value_list[1:]) - - -def to_snake_case(name): - name = to_camel_case(name.replace('-', '_')) - name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - name = re.sub('__([A-Z])', r'_\1', name) - name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', name) - return name.lower() diff --git a/src/osintbuddy/utils/__init__.py b/src/osintbuddy/utils/__init__.py new file mode 100644 index 0000000..53772b1 --- /dev/null +++ b/src/osintbuddy/utils/__init__.py @@ -0,0 +1,9 @@ +from .generic import ( + MAP_KEY, + chunks, + find_emails, + plugin_source_template, + to_clean_domain, + to_camel_case, + to_snake_case +) \ No newline at end of file diff --git a/src/osintbuddy/utils/generic.py b/src/osintbuddy/utils/generic.py new file mode 100644 index 0000000..1a330a2 --- /dev/null +++ b/src/osintbuddy/utils/generic.py @@ -0,0 +1,101 @@ +import re +import unicodedata +from typing import List +from urllib import parse +from pydantic import EmailStr + + +MAP_KEY = '___obmap___' + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +def find_emails(value: str) -> List[EmailStr]: + emails = [] + match = re.search(r"[\w.+-]+@[\w-]+\.[\w.-]+", value) + if match is not None: + email = match.group(0) + # if trailing dot, remove. @todo improve regex + if email[len(email) - 1] == ".": + emails.append(email[0: len(email) - 2]) + else: + emails.append(email) + return list(set(emails)) + + +def to_clean_domain(value: str) -> str: + if "http://" not in value and "https://" not in value: + value = "https://" + value + url = parse.urlparse(value) + split_domain = url.netloc.split(".") + if len(split_domain) >= 3: + split_domain.pop(0) + domain = ".".join(split_domain) + return domain + + +def plugin_source_template(label: str, description: str, author: str) -> str: + class_name = ''.join(x for x in filter(str.isalnum, label.title()) if not x.isspace()) + + return f"""import osintbuddy as ob +from osintbuddy.elements import TextInput + +class {class_name}(ob.Plugin): + label = '{label}' + icon = 'atom' # https://tabler-icons.io/ + color = '#FFD166' + + author = '{author}' + description = '{description}' + + node = [ + TextInput(label='Example', icon='radioactive') + ] + + @ob.transform(label='To example', icon='atom') + async def transform_example(self, node, use): + WebsitePlugin = await ob.Registry.get_plugin('website') + website_plugin = WebsitePlugin() + return website_plugin.blueprint(domain=node.example) +""" + + + +# Slugify and related code is from the Django project, thanks guys! +# Project URL: https://github.com/django/django +# https://github.com/django/django/blob/main/django/utils/text.py +def slugify(value, allow_unicode=False): + """ + Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated + dashes to single dashes. Remove characters that aren't alphanumerics, + underscores, or hyphens. Convert to lowercase. Also strip leading and + trailing whitespace, dashes, and underscores. + """ + value = str(value) + if allow_unicode: + value = unicodedata.normalize("NFKC", value) + else: + value = ( + unicodedata.normalize("NFKD", value) + .encode("ascii", "ignore") + .decode("ascii") + ) + value = re.sub(r"[^\w\s-]", "", value.lower()) + return re.sub(r"[-\s]+", "-", value).strip("-_") + + +def to_camel_case(value: str): + value_list = value.replace(' ', '_').lower().split('_') + return value_list[0] + ''.join(e.title() for e in value_list[1:]) + + +def to_snake_case(name): + name = to_camel_case(name.replace('-', '_')) + name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + name = re.sub('__([A-Z])', r'_\1', name) + name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', name) + return name.lower() From 4872fe2136744bb92b80f54bae9f36238ebfcc3c Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:34:11 -0600 Subject: [PATCH 02/11] Rework templates file to templates directory --- src/osintbuddy/templates/__init__.py | 1 + src/osintbuddy/templates/default.py | 25 +++++++++++++++++++++++++ src/osintbuddy/utils/generic.py | 27 --------------------------- 3 files changed, 26 insertions(+), 27 deletions(-) create mode 100644 src/osintbuddy/templates/__init__.py create mode 100644 src/osintbuddy/templates/default.py diff --git a/src/osintbuddy/templates/__init__.py b/src/osintbuddy/templates/__init__.py new file mode 100644 index 0000000..048fa70 --- /dev/null +++ b/src/osintbuddy/templates/__init__.py @@ -0,0 +1 @@ +from .default import plugin_source_template \ No newline at end of file diff --git a/src/osintbuddy/templates/default.py b/src/osintbuddy/templates/default.py new file mode 100644 index 0000000..96a342d --- /dev/null +++ b/src/osintbuddy/templates/default.py @@ -0,0 +1,25 @@ +def plugin_source_template(label: str, description: str, author: str) -> str: + class_name = ''.join(x for x in filter(str.isalnum, label.title()) if not x.isspace()) + + return f"""import osintbuddy as ob +from osintbuddy.elements import TextInput + +class {class_name}(ob.Plugin): + label = '{label}' + icon = 'atom' # https://tabler-icons.io/ + color = '#FFD166' + + author = '{author}' + description = '{description}' + + node = [ + TextInput(label='Example', icon='radioactive') + ] + + @ob.transform(label='To example', icon='atom') + async def transform_example(self, node, use): + WebsitePlugin = await ob.Registry.get_plugin('website') + website_plugin = WebsitePlugin() + return website_plugin.blueprint(domain=node.example) +""" + diff --git a/src/osintbuddy/utils/generic.py b/src/osintbuddy/utils/generic.py index 1a330a2..8b73e78 100644 --- a/src/osintbuddy/utils/generic.py +++ b/src/osintbuddy/utils/generic.py @@ -38,33 +38,6 @@ def to_clean_domain(value: str) -> str: return domain -def plugin_source_template(label: str, description: str, author: str) -> str: - class_name = ''.join(x for x in filter(str.isalnum, label.title()) if not x.isspace()) - - return f"""import osintbuddy as ob -from osintbuddy.elements import TextInput - -class {class_name}(ob.Plugin): - label = '{label}' - icon = 'atom' # https://tabler-icons.io/ - color = '#FFD166' - - author = '{author}' - description = '{description}' - - node = [ - TextInput(label='Example', icon='radioactive') - ] - - @ob.transform(label='To example', icon='atom') - async def transform_example(self, node, use): - WebsitePlugin = await ob.Registry.get_plugin('website') - website_plugin = WebsitePlugin() - return website_plugin.blueprint(domain=node.example) -""" - - - # Slugify and related code is from the Django project, thanks guys! # Project URL: https://github.com/django/django # https://github.com/django/django/blob/main/django/utils/text.py From e84a729b50f96fbc24df5069c126dfd4fc56891a Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:36:54 -0600 Subject: [PATCH 03/11] Add deps and ob command to osintbuddy pkg --- pyproject.toml | 29 +++++++++++++++++++++++------ requirements.txt | 21 ++++++++++++++++----- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f9ce886..a56a052 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "flit.buildapi" [project] name = "osintbuddy" authors = [ - {name = "jerlendds", email = "theosintbuddyproject@openinfolabs.com"}, + {name = "jerlendds", email = "support@forum.osintbuddy.com"}, ] description = "OSINTBuddy - mine, merge, and map data for novel insights" readme = "README.md" @@ -15,13 +15,22 @@ classifiers = [ "License :: OSI Approved :: GNU Affero General Public License v3", "Programming Language :: Python :: 3.11" ] -requires-python = ">=3.7" +requires-python = ">=3.11" dynamic = ["version"] dependencies = [ - "selenium>=4.8.0", - "pydantic>=1.10.8", - "httpx>=0.23.0", - "SQLAlchemy>=2.0.12", + "playwright>=1.39.0", + "httpx>=0.25.0", + "beautifulsoup4==4.12.2", + "pyfiglet==0.8.post1", + "termcolor==2.3.0", + "fastapi==0.103.2", + "uvicorn==0.22.0", + "uvloop==0.17.0", + "pydantic==2.4.2", + "pydantic-settings==2.0.3", + "yq==3.2.3", + "jedi-language-server==0.41.1", + "websockets==11.0.3" ] [project.optional-dependencies] test = [ @@ -324,3 +333,11 @@ overgeneral-exceptions= [ "BaseException", "Exception" ] +[tool.hatch.build.targets.sdist.force-include] +"bin/ob.py" = "osintbuddy/ob.py" + +[tool.hatch.build.targets.wheel.force-include] +"bin/ob.py" = "osintbuddy/ob.py" + +[project.scripts] +ob = "osintbuddy.ob:main" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 28ce6bd..dd137da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,17 @@ - selenium>=4.9.0 -pydantic>=1.10.8 -httpx>=0.23.3 -SQLAlchemy>=2.0.12 -gremlinpy @ git+https://github.com/jerlendds/gremlinpy.git@7d3033e6a55ed9cb1f982ec3b58ca233e01c58e3 \ No newline at end of file +sqlalchemy-json==0.7.0 +SQLAlchemy-Utils==0.41.1 +playwright==1.39.0 +httpx>=0.25.0 +beautifulsoup4==4.12.2 +gremlinpy @ git+https://github.com/jerlendds/gremlinpy.git@eaba7dca12ad0156eb0d6d8ba2eb5751551c6a6d +pyfiglet==0.8.post1 +termcolor==2.3.0 +fastapi==0.103.2 +uvicorn==0.22.0 +uvloop==0.17.0 +pydantic==2.4.2 +pydantic-settings==2.0.3 +yq==3.2.3 +jedi-language-server==0.41.1 +websockets==11.0.3 From e37a2af3bcbe80f606922bfe685aa07fe87650a2 Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:37:05 -0600 Subject: [PATCH 04/11] Add deps and ob command to osintbuddy pkg --- pyproject.toml | 101 +++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a56a052..c507568 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,52 +18,55 @@ classifiers = [ requires-python = ">=3.11" dynamic = ["version"] dependencies = [ - "playwright>=1.39.0", - "httpx>=0.25.0", - "beautifulsoup4==4.12.2", - "pyfiglet==0.8.post1", - "termcolor==2.3.0", - "fastapi==0.103.2", - "uvicorn==0.22.0", - "uvloop==0.17.0", - "pydantic==2.4.2", - "pydantic-settings==2.0.3", - "yq==3.2.3", - "jedi-language-server==0.41.1", - "websockets==11.0.3" + "selenium>=4.9.0", + "sqlalchemy-json==0.7.0", + "SQLAlchemy-Utils==0.41.1", + "playwright>=1.39.0", + "httpx>=0.25.0", + "beautifulsoup4==4.12.2", + "pyfiglet==0.8.post1", + "termcolor==2.3.0", + "fastapi==0.103.2", + "uvicorn==0.22.0", + "uvloop==0.17.0", + "pydantic==2.4.2", + "pydantic-settings==2.0.3", + "yq==3.2.3", + "jedi-language-server==0.41.1", + "websockets==11.0.3" ] [project.optional-dependencies] test = [ - "astroid==2.15.4", - "colorama==0.4.6", - "dill==0.3.6", - "eradicate==2.2.0", - "exceptiongroup==1.1.1", - "iniconfig==2.0.0", - "isort==5.12.0", - "lazy-object-proxy==1.9.0", - "mando==0.7.1", - "mccabe==0.7.0", - "mypy==1.3.0", - "mypy-extensions==1.0.0", - "packaging==23.1", - "platformdirs==3.5.1", - "pluggy==1.0.0", - "pycodestyle==2.10.0", - "pydocstyle==6.3.0", - "pyflakes==3.0.1", - "pylama==8.4.1", - "pylint==2.17.4", - "pytest==7.3.1", - "radon==6.0.1", - "six==1.16.0", - "snowballstemmer==2.2.0", - "toml==0.10.2", - "tomli==2.0.1", - "tomlkit==0.11.8", - "typing-extensions==4.5.0", - "vulture==2.7", - "wrapt==1.15.0", + "astroid==2.15.4", + "colorama==0.4.6", + "dill==0.3.6", + "eradicate==2.2.0", + "exceptiongroup==1.1.1", + "iniconfig==2.0.0", + "isort==5.12.0", + "lazy-object-proxy==1.9.0", + "mando==0.7.1", + "mccabe==0.7.0", + "mypy==1.3.0", + "mypy-extensions==1.0.0", + "packaging==23.1", + "platformdirs==3.5.1", + "pluggy==1.0.0", + "pycodestyle==2.10.0", + "pydocstyle==6.3.0", + "pyflakes==3.0.1", + "pylama==8.4.1", + "pylint==2.17.4", + "pytest==7.3.1", + "radon==6.0.1", + "six==1.16.0", + "snowballstemmer==2.2.0", + "toml==0.10.2", + "tomli==2.0.1", + "tomlkit==0.11.8", + "typing-extensions==4.5.0", + "vulture==2.7", + "wrapt==1.15.0", ] [project.urls] @@ -127,13 +130,13 @@ pythonVersion = "3.7" pythonPlatform = "Linux" executionEnvironments = [ - { root = "src" } + { root = "src" } ] [tool.pytest.ini_options] addopts = "" pythonpath = [ - "src" + "src" ] testpaths = "tests" junit_family = "xunit2" @@ -162,16 +165,16 @@ commands = [testenv:spark] extras = spark setenv = - PYSPARK_DRIVER_PYTHON = {envpython} - PYSPARK_PYTHON = {envpython} + PYSPARK_DRIVER_PYTHON = {envpython} + PYSPARK_PYTHON = {envpython} commands = pytest -m "spark" {posargs} [testenv:all] extras = all setenv = - PYSPARK_DRIVER_PYTHON = {envpython} - PYSPARK_PYTHON = {envpython} + PYSPARK_DRIVER_PYTHON = {envpython} + PYSPARK_PYTHON = {envpython} commands = pytest {posargs} """ From 097cd77860245eced2811757382628bf718e8d13 Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:37:29 -0600 Subject: [PATCH 05/11] Add ob command --- src/osintbuddy/ob.py | 113 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/osintbuddy/ob.py diff --git a/src/osintbuddy/ob.py b/src/osintbuddy/ob.py new file mode 100644 index 0000000..c6fdacb --- /dev/null +++ b/src/osintbuddy/ob.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +"""OSINTBuddy plugins server script + +This script contains the commands needed to manage an OSINTBuddy Plugins service, which is used by the OSINTBuddy project. + +Basic Commands: + Plugins service command(s): + `start` : Starts the FastAPI microservice (`ctrl+c` to stop the microservice) + `lsp` : Start the language server for code completion in the OSINTBuddy app + Database Command(s): + `plugin create` : Run the setup wizard for creating new plugin(s) + `load $GIT_URL` : Load plugin(s) from a remote git repository +""" + +from os import getpid, devnull +from argparse import ArgumentParser, BooleanOptionalAction +from pyfiglet import figlet_format +from termcolor import colored +import osintbuddy + +APP_INFO = \ +"""____________________________________________________________________ +| Find, share, and get help with OSINTBuddy plugins: +| https://forum.osintbuddy.com/c/plugin-devs/5 +|___________________________________________________________________ +| If you run into any bugs, please file an issue on Github: +| https://github.com/jerlendds/osintbuddy +|___________________________________________________________________ +| +| OSINTBuddy plugins: v{osintbuddy_version} +| PID: {pid} +| Endpoint: 0.0.0.0:42562 +""".rstrip() + + +def _print_server_details(): + print(colored(figlet_format(f"OSINTBuddy plugins", font='smslant'), color="blue")) + print(colored(APP_INFO.format( + osintbuddy_version=osintbuddy.__version__, + pid=getpid(), + ), color="blue")) + colored("Created by", color="blue"), colored("jerlendds", color="red") + + +def _print_lsp_details(): + import jedi_language_server + print(colored(figlet_format(f"OSINTBuddy LSP", font='smslant'), color="blue")) + colored("Created by", color="blue"), colored("jerlendds", color="red") + print(colored(f"""____________________________________________________________________ +| Jedi Language Server: v{jedi_language_server.__version__} +| Endpoint: {'ws://0.0.0.0:54332'} +""", color="blue")) + +def start_lsp(): + _print_lsp_details() + import subprocess + FNULL = open(devnull, 'w') + jedi_language_server = subprocess.Popen( + ["ps", "aux", "|", "pkill", "jedi-language-", "&&","jedi-language-server", "--ws", "--host", "0.0.0.0", "--port", "54332"], + stdout=FNULL, + stderr=subprocess.STDOUT + ) + return jedi_language_server + + +def start(): + jedi_language_server = start_lsp() + _print_server_details() + import signal + import uvicorn + uvicorn.run( + "osintbuddy.server:app", + host="127.0.0.1", + port=42562, + loop='asyncio', + reload=True, + workers=4, + headers=[('server', f"OSINTBuddy")], + log_level='info' + ) + + def signal_handler(sig, frame): + jedi_language_server.wait(timeout=1) + + signal.signal(signal.SIGINT, signal_handler) + signal.pause() + + +def create_plugin_wizard(): + # TODO: setup prompt process for initializing an osintbuddy plugin(s) project + pass + + +def main(): + commands = { + "lsp": start_lsp, + "start": start, + "plugin create": create_plugin_wizard, + } + parser = ArgumentParser() + parser.add_argument('command', type=str, nargs="*", help="[CATEGORY (Optional)] [ACTION]") + + args = parser.parse_args() + command = commands.get(' '.join(args.command)) + + if command: + command() + else: + parser.error("Command not recognized") + + +if __name__ == '__main__': + main() From f9a9eadab632ae03fcf9ab137317b3896d37625c Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 12:38:04 -0600 Subject: [PATCH 06/11] Add FastAPI server --- src/osintbuddy/__init__.py | 0 src/osintbuddy/errors.py | 0 src/osintbuddy/plugins.py | 0 src/osintbuddy/server.py | 5 +++++ 4 files changed, 5 insertions(+) mode change 100644 => 100755 src/osintbuddy/__init__.py mode change 100644 => 100755 src/osintbuddy/errors.py mode change 100644 => 100755 src/osintbuddy/plugins.py create mode 100644 src/osintbuddy/server.py diff --git a/src/osintbuddy/__init__.py b/src/osintbuddy/__init__.py old mode 100644 new mode 100755 diff --git a/src/osintbuddy/errors.py b/src/osintbuddy/errors.py old mode 100644 new mode 100755 diff --git a/src/osintbuddy/plugins.py b/src/osintbuddy/plugins.py old mode 100644 new mode 100755 diff --git a/src/osintbuddy/server.py b/src/osintbuddy/server.py new file mode 100644 index 0000000..1c2530a --- /dev/null +++ b/src/osintbuddy/server.py @@ -0,0 +1,5 @@ + +from fastapi import FastAPI +import osintbuddy + +app = FastAPI(title=f"OSINTBuddy Plugins v{osintbuddy.__version__}") From df6e0d8c5cfafc3fa0b901a1db7bffd7cf545d5f Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 19:27:31 -0600 Subject: [PATCH 07/11] Add dict keys to snake case function and ascii logo --- src/osintbuddy/ascii.py | 127 ++++++++++++++++++++++++++++++++ src/osintbuddy/utils/generic.py | 11 ++- 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/osintbuddy/ascii.py diff --git a/src/osintbuddy/ascii.py b/src/osintbuddy/ascii.py new file mode 100644 index 0000000..b46c593 --- /dev/null +++ b/src/osintbuddy/ascii.py @@ -0,0 +1,127 @@ + +OB_LOGO_SM = """ + ██████████████████████████████████████ + ██████████████████████████████████████ + ██████████████████████████████████████ + █████ ██████ + █████ ██████ + █████ █ ███ ██████ + █████ ███ ████ ██████ + █████ ██ ███ ████ ██████ + █████ ██ ███ ████ ██ ██████ + █████ ██ ███ ████ ███ ██████ + █████ ██ ███ ████ ███ ██████ + █████ ██ ███ ████ ███ ██████ + █████ ███ ███ ███ ██ ██████ + █████ ███ ███ ████ ██ ██████ + █████ ███ ███ ███ ██ ██████ + █████ █ ███ ███ ██ ██████ + █████ ███ ███ ██ ██████ + ██████ ███ ██ █████ + ███████ ██ ██████ + ███████ ███████ + ████████ ████████ + ██████████ ██████████ + ██████████████████████████ + █████████████████████ + ████████████ +""" + +OB_LOGO_LG = """ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓ ▓▓▓▓ ▓▓▓▓ ▓▓▓▓▓▓ + ▓▓▓▓▓▓▓ ▓▓▓▓ ▓▓ ▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓ ▓▓ ▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ + ▓▓▓▓▓▓ +""" + +OB_LOGO_LIGHT = """ + rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr + rrrrr rrrrr + rrrrr rrr rrrrr + rrrrr rr rr rrr rrrrr + rrrrr rr rrr rrr rrrrr + rrrrr rr rrr rrr rrr rrrrr + rrrrr rr rrr rrr rrr rrrrr + rrrrr rr rrr rrr rrr rrrrr + rrrrr rrr rrr rrr rr rrrrr + rrrrr rrr rrr rrr rr rrrrr + rrrrr rrr rrrr rr rr rrrrr + rrrrr r rrrr rr rr rrrrr + rrrrr rrr rrr rr rrrrr + rrrrrr rrr rr rrrrrr + rrrrrr rr rrrrrr + rrrrrr rrrrrr + rrrrrrrr rrrrrrr + rrrrrrrrrrrrrrrrrrrrrrrrr + rrrrrrrrrrrrrrrrrrrrr + rrrrrrrrrrrrrr +""" + +OB_LOGO_XL = """ + █████████████████████████████████████████████████████████ + █████████████████████████████████████████████████████████ + ██████████████████████████████████████████████████████████ + ██████████████████████████████████████████████████████████ + ██████████████████████████████████████████████████████████ + █████████ ████████ + █████████ ████████ + █████████ ██ ████████ + █████████ ████ ████████ + █████████ ████ █████ ████████ + █████████ █████ █████ ████████ + █████████ ███ █████ █████ ████████ + █████████ ████ ████ █████ █ ████████ + █████████ ████ ████ ██████ ████ ████████ + █████████ ███ ████ ██████ ████ ████████ + █████████ ███ ████ ██████ █████ ████████ + █████████ ███ █████ ██████ █████ ████████ + █████████ ███ █████ ██████ ████ ████████ + █████████ █████ ██████ ████ ██ ████████ + █████████ █████ █████ ████ ████ ████████ + █████████ █████ █████ ████ ████ ████████ + ████████ █████ █████ █████ ████ ████████ + ████████ █████ █████ █████ ████ ████████ + ████████ ████ █████ █████ ████ ████████ + ████████ █████ █████ ████ ████████ + █████████ █████ █████ ██ █████████ + █████████ █████ █████ █████████ + █████████ █████ ████ █████████ + ██████████ ████ █████████ + ██████████ █████████ + ██████████ ██████████ + ███████████ ███████████ + █████████████ ████████████ + ███████████████ ████████████████ + ██████████████████████████████████████████ + ██████████████████████████████████████ + ████████████████████████████████ + ████████████████████████ + ████████████ +""" \ No newline at end of file diff --git a/src/osintbuddy/utils/generic.py b/src/osintbuddy/utils/generic.py index 8b73e78..092a3b6 100644 --- a/src/osintbuddy/utils/generic.py +++ b/src/osintbuddy/utils/generic.py @@ -1,6 +1,6 @@ import re import unicodedata -from typing import List +from typing import List, Union from urllib import parse from pydantic import EmailStr @@ -72,3 +72,12 @@ def to_snake_case(name): name = re.sub('__([A-Z])', r'_\1', name) name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', name) return name.lower() + +# Convert all keys in dict to snake_case +def dkeys_to_snake_case(data: dict) -> Union[dict, List[dict]]: + def to_snake(s): + return re.sub('([A-Z]\w+$)', '_\\1', s).lower() + + if isinstance(data, list): + return [dkeys_to_snake_case(i) if isinstance(i, (dict, list)) else i for i in data] + return {to_snake(a):dkeys_to_snake_case(b) if isinstance(b, (dict, list)) else b for a, b in data.items()} From a2200a20696751c6abcb93e1ebc06ea5402c7d3a Mon Sep 17 00:00:00 2001 From: jerlendds Date: Tue, 31 Oct 2023 19:28:10 -0600 Subject: [PATCH 08/11] Fix utils imports and add dkeys_to_snake_case function --- src/osintbuddy/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osintbuddy/utils/__init__.py b/src/osintbuddy/utils/__init__.py index 53772b1..e325784 100644 --- a/src/osintbuddy/utils/__init__.py +++ b/src/osintbuddy/utils/__init__.py @@ -2,8 +2,8 @@ MAP_KEY, chunks, find_emails, - plugin_source_template, to_clean_domain, to_camel_case, - to_snake_case + to_snake_case, + dkeys_to_snake_case ) \ No newline at end of file From c14372e7dfad2d489adaa18400b19e31cdd1a46e Mon Sep 17 00:00:00 2001 From: jerlendds Date: Sun, 5 Nov 2023 20:16:15 -0700 Subject: [PATCH 09/11] Rework plugins to work with updated pydantic plugins.py --- src/osintbuddy/plugins.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/osintbuddy/plugins.py b/src/osintbuddy/plugins.py index 1fad9ec..951126c 100755 --- a/src/osintbuddy/plugins.py +++ b/src/osintbuddy/plugins.py @@ -1,16 +1,19 @@ -import os, imp, importlib, sys +import os, imp, importlib from typing import List, Any, Callable from collections import defaultdict -from pydantic import create_model, BaseModel -# from osintbuddy.utils import slugify +from pydantic import BaseModel, ConfigDict from osintbuddy.elements.base import BaseElement from osintbuddy.errors import OBPluginError from osintbuddy.utils import to_snake_case -# @todo add permission system and display what parts of system plugin can access +OBNodeConfig = ConfigDict(extra="allow", frozen=False, populate_by_name=True, arbitrary_types_allowed=True) + +class OBNode(BaseModel): + model_config = OBNodeConfig + + class OBAuthorUse(BaseModel): - # @todo get_driver: Callable[[], None] get_graph: Callable[[], None] @@ -271,15 +274,14 @@ def _map_element(transform_map: dict, element: dict): transform_map[label][k] = v @classmethod - def _map_to_transform_data(cls, node: dict): - transform_map = {} - data = node.get('data', {}) - model_label = data.get('label') - elements = data.get('elements', []) + def _map_to_transform_data(cls, node: dict) -> OBNode: + transform_map: dict = {} + data: dict = node.get('data', {}) + # model_label: str = data.get('label') + elements: list[dict] = data.get('elements', []) for element in elements: if isinstance(element, list): [cls._map_element(transform_map, elm) for elm in element] else: cls._map_element(transform_map, element) - model = create_model(model_label, **transform_map) - return model() + return OBNode(**transform_map) From b3d395b98853a3fd4da5dd840e628941b9ee1828 Mon Sep 17 00:00:00 2001 From: jerlendds Date: Sun, 5 Nov 2023 22:26:56 -0700 Subject: [PATCH 10/11] Polish default icon displays.py and plugins.py --- src/osintbuddy/elements/displays.py | 2 +- src/osintbuddy/plugins.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/osintbuddy/elements/displays.py b/src/osintbuddy/elements/displays.py index 11f023c..d80a181 100644 --- a/src/osintbuddy/elements/displays.py +++ b/src/osintbuddy/elements/displays.py @@ -21,7 +21,7 @@ def json(self): class Text(BaseDisplay): node_type: str = 'section' - def __init__(self, value='', icon=None, **kwargs): + def __init__(self, value='', icon="123", **kwargs): super().__init__(**kwargs) self.value = value self.icon = icon diff --git a/src/osintbuddy/plugins.py b/src/osintbuddy/plugins.py index 951126c..e09f68d 100755 --- a/src/osintbuddy/plugins.py +++ b/src/osintbuddy/plugins.py @@ -234,8 +234,6 @@ async def get_transform(self, transform_type: str, node, use: OBAuthorUse) -> An for the transform_type. """ transform_type = to_snake_case(transform_type) - print('transform_type', transform_type) - print('transforms', self.transforms) if self.transforms and self.transforms[transform_type]: try: transform = await self.transforms[transform_type]( From e1511ec9464d1b5ce5ec7b5ad7a206c34e790079 Mon Sep 17 00:00:00 2001 From: jerlendds Date: Sat, 9 Dec 2023 08:38:10 -0700 Subject: [PATCH 11/11] Rework and polish plugins for release base.py, ob.py, plugins.py, and default.py --- src/osintbuddy/elements/base.py | 5 +---- src/osintbuddy/ob.py | 12 ++++++------ src/osintbuddy/plugins.py | 23 +++++++++++++++++------ src/osintbuddy/templates/default.py | 4 ++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/osintbuddy/elements/base.py b/src/osintbuddy/elements/base.py index 227d37c..0053d09 100644 --- a/src/osintbuddy/elements/base.py +++ b/src/osintbuddy/elements/base.py @@ -19,19 +19,16 @@ class BaseElement(object): """ def __init__(self, **kwargs): self.label: str = '' - self.style: dict = {} self.placeholder: str = '' for key, value in kwargs.items(): - if key == 'label' or key == 'style' or key == 'placeholder': + if key == 'label' or key == 'placeholder': setattr(self, key, value) def _base_blueprint(self): return { 'type': self.node_type, 'label': self.label, - 'placeholder': self.placeholder, - 'style': self.style } diff --git a/src/osintbuddy/ob.py b/src/osintbuddy/ob.py index c6fdacb..9586344 100644 --- a/src/osintbuddy/ob.py +++ b/src/osintbuddy/ob.py @@ -64,9 +64,9 @@ def start_lsp(): def start(): - jedi_language_server = start_lsp() + # jedi_language_server = start_lsp() _print_server_details() - import signal + # import signal import uvicorn uvicorn.run( "osintbuddy.server:app", @@ -79,11 +79,11 @@ def start(): log_level='info' ) - def signal_handler(sig, frame): - jedi_language_server.wait(timeout=1) + # def signal_handler(sig, frame): + # jedi_language_server.wait(timeout=1) - signal.signal(signal.SIGINT, signal_handler) - signal.pause() + # signal.signal(signal.SIGINT, signal_handler) + # signal.pause() def create_plugin_wizard(): diff --git a/src/osintbuddy/plugins.py b/src/osintbuddy/plugins.py index e09f68d..286275f 100755 --- a/src/osintbuddy/plugins.py +++ b/src/osintbuddy/plugins.py @@ -1,4 +1,4 @@ -import os, imp, importlib +import os, imp, importlib, inspect from typing import List, Any, Callable from collections import defaultdict from pydantic import BaseModel, ConfigDict @@ -13,6 +13,20 @@ class OBNode(BaseModel): model_config = OBNodeConfig +def plugin_results_middleman(f): + def return_result(r): + return r + def yield_result(r): + for i in r: + yield i + def decorator(*a, **kwa): + if inspect.isgeneratorfunction(f): + return yield_result(f(*a, **kwa)) + else: + return return_result(f(*a, **kwa)) + return decorator + + class OBAuthorUse(BaseModel): get_driver: Callable[[], None] get_graph: Callable[[], None] @@ -53,7 +67,7 @@ async def get_plugin(cls, plugin_label: str): :return: The plugin class or None if not found. """ for idx, label in enumerate(cls.labels): - if to_snake_case(label) == to_snake_case(plugin_label): + if label == plugin_label or to_snake_case(label) == to_snake_case(plugin_label): return cls.plugins[idx] return None @@ -71,7 +85,7 @@ def get_plug(cls, plugin_label: str): return cls.plugins[idx] return None - def __getitem__(self, i): + def __getitem__(self, i: str): return self.get_plug[i] # https://stackoverflow.com/a/7548190 @@ -168,7 +182,6 @@ class OBPlugin(object, metaclass=OBRegistry): label: str = '' icon: str = 'atom-2' show_label = True - style: dict = {} author = 'Unknown' description = 'No description.' @@ -212,7 +225,6 @@ def blueprint(cls, **kwargs): node['label'] = cls.label node['color'] = cls.color if cls.color else '#145070' node['icon'] = cls.icon - node['style'] = cls.style node['elements'] = [] for element in cls.node: if isinstance(element, list): @@ -275,7 +287,6 @@ def _map_element(transform_map: dict, element: dict): def _map_to_transform_data(cls, node: dict) -> OBNode: transform_map: dict = {} data: dict = node.get('data', {}) - # model_label: str = data.get('label') elements: list[dict] = data.get('elements', []) for element in elements: if isinstance(element, list): diff --git a/src/osintbuddy/templates/default.py b/src/osintbuddy/templates/default.py index 96a342d..e1e455a 100644 --- a/src/osintbuddy/templates/default.py +++ b/src/osintbuddy/templates/default.py @@ -6,7 +6,7 @@ def plugin_source_template(label: str, description: str, author: str) -> str: class {class_name}(ob.Plugin): label = '{label}' - icon = 'atom' # https://tabler-icons.io/ + icon = 'atom-2' # https://tabler-icons.io/ color = '#FFD166' author = '{author}' @@ -16,7 +16,7 @@ class {class_name}(ob.Plugin): TextInput(label='Example', icon='radioactive') ] - @ob.transform(label='To example', icon='atom') + @ob.transform(label='To example', icon='atom-2') async def transform_example(self, node, use): WebsitePlugin = await ob.Registry.get_plugin('website') website_plugin = WebsitePlugin()