From b1b4ad0cf2b80548325b043a8d680c6651efdb99 Mon Sep 17 00:00:00 2001 From: Julio Faerman <356476+faermanj@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:02:28 +0000 Subject: [PATCH] Initial "aws subs" code --- up_aws/Containerfile | 43 ++++++++ up_aws/opt/aws_utils/README.md | 1 + up_aws/opt/aws_utils/aws_utils/__init__.py | 0 up_aws/opt/aws_utils/aws_utils/subs.py | 14 +++ up_aws/opt/aws_utils/poetry.lock | 114 +++++++++++++++++++++ up_aws/opt/aws_utils/pyproject.toml | 18 ++++ up_aws/opt/aws_utils/tests/__init__.py | 0 up_aws/pyproject.toml | 2 + up_aws/up_aws/containers_for_prompt.py | 28 ++++- uplib/uplib/containers.py | 13 ++- uplib/uplib/main.py | 18 +--- 11 files changed, 232 insertions(+), 19 deletions(-) create mode 100644 up_aws/Containerfile create mode 100644 up_aws/opt/aws_utils/README.md create mode 100644 up_aws/opt/aws_utils/aws_utils/__init__.py create mode 100644 up_aws/opt/aws_utils/aws_utils/subs.py create mode 100644 up_aws/opt/aws_utils/poetry.lock create mode 100644 up_aws/opt/aws_utils/pyproject.toml create mode 100644 up_aws/opt/aws_utils/tests/__init__.py diff --git a/up_aws/Containerfile b/up_aws/Containerfile new file mode 100644 index 0000000..357972f --- /dev/null +++ b/up_aws/Containerfile @@ -0,0 +1,43 @@ +# docker build --no-cache --progress=plain -t caravanacloud/$(basename $PWD) -f Containerfile . +FROM fedora + +# System Updates +# RUN bash -c "sudo dnf -y update" +RUN bash -c "sudo dnf -y install unzip zip pipx wget python3 python3-pip" + +# AWS CLIs +RUN bash -c "curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o 'awscliv2.zip' && unzip awscliv2.zip \ + && sudo ./aws/install \ + && aws --version \ + " + +# RUN bash -c "npm install -g aws-cdk" + +# ARG SAM_URL="https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip" +# RUN bash -c "curl -Ls '${SAM_URL}' -o '/tmp/aws-sam-cli-linux-x86_64.zip' \ +# && unzip '/tmp/aws-sam-cli-linux-x86_64.zip' -d '/tmp/sam-installation' \ +# && sudo '/tmp/sam-installation/install' \ +# && sam --version" + +# RUN bash -c "pip install cloudformation-cli cloudformation-cli-java-plugin cloudformation-cli-go-plugin cloudformation-cli-python-plugin cloudformation-cli-typescript-plugin" + +# COPY +# Creates user up_user inside container +ARG USER=up_user +ARG UID=1000 +ARG GID=1000 +RUN groupadd -g $GID $USER && \ + useradd -l -u $UID -g $USER -m -d /home/$USER $USER && \ + chown -R $USER:$USER /home/$USER + +USER up_user +# Python Poetry +RUN bash -c "pipx install poetry && python3 -m pipx ensurepath" +RUN bash -c "/home/up_user/.local/bin/poetry --version" + +COPY ./opt /opt +# RUN bash -c "find /opt" + +# Done :) +RUN bash -c "echo 'done.'" + diff --git a/up_aws/opt/aws_utils/README.md b/up_aws/opt/aws_utils/README.md new file mode 100644 index 0000000..4b8342f --- /dev/null +++ b/up_aws/opt/aws_utils/README.md @@ -0,0 +1 @@ +This is some sample code to run inside the container \ No newline at end of file diff --git a/up_aws/opt/aws_utils/aws_utils/__init__.py b/up_aws/opt/aws_utils/aws_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/up_aws/opt/aws_utils/aws_utils/subs.py b/up_aws/opt/aws_utils/aws_utils/subs.py new file mode 100644 index 0000000..fde8871 --- /dev/null +++ b/up_aws/opt/aws_utils/aws_utils/subs.py @@ -0,0 +1,14 @@ +def subs(): + print("*** Generating Subtitles using AWS ***") + print("* Validate credentials (sts get client identity)") + print("* Find files to translate (python glob)") + print("* Find or create data bucket (aws s3)") + print("* Upload videos to bucket (aws s3)") + print("* Start and wait transcribe job") + print("* Start and wait translate job") + print("* Download translated subtitles") + print("* ...") + +#main invoke +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/up_aws/opt/aws_utils/poetry.lock b/up_aws/opt/aws_utils/poetry.lock new file mode 100644 index 0000000..6e6f5bd --- /dev/null +++ b/up_aws/opt/aws_utils/poetry.lock @@ -0,0 +1,114 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "boto3" +version = "1.29.2" +description = "The AWS SDK for Python" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "boto3-1.29.2-py3-none-any.whl", hash = "sha256:6617ac176efb21485ebc3a058a3a97feb1300141421ae3d1809562c4cac1d5f9"}, + {file = "boto3-1.29.2.tar.gz", hash = "sha256:f3024bba9ac980007ba7b5f28a9734d111fb5466e2426ac76c5edbd6dedd8db2"}, +] + +[package.dependencies] +botocore = ">=1.32.2,<1.33.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.7.0,<0.8.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.32.2" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">= 3.7" +files = [ + {file = "botocore-1.32.2-py3-none-any.whl", hash = "sha256:a68a33193d8cd59e3b2142bff632e562afc02f9c4417e3dcc81a6e1b1f47148e"}, + {file = "botocore-1.32.2.tar.gz", hash = "sha256:0e231524e9b72169fe0b8d9310f47072c245fb712778e0669f53f264f0e49536"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.19.12)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "s3transfer" +version = "0.7.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.7.0-py3-none-any.whl", hash = "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a"}, + {file = "s3transfer-0.7.0.tar.gz", hash = "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "urllib3" +version = "2.0.7" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.7-py3-none-any.whl", hash = "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e"}, + {file = "urllib3-2.0.7.tar.gz", hash = "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.12" +content-hash = "a47f62688f1109edae10699a1c2cec17310b53550b313f448dc90a616c5cbe5a" diff --git a/up_aws/opt/aws_utils/pyproject.toml b/up_aws/opt/aws_utils/pyproject.toml new file mode 100644 index 0000000..ea6ea0b --- /dev/null +++ b/up_aws/opt/aws_utils/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "aws-utils" +version = "0.1.0" +description = "" +authors = ["Julio Faerman <356476+faermanj@users.noreply.github.com>"] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.12" +boto3 = "^1.29.2" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +subs = "aws_utils.subs:subs" \ No newline at end of file diff --git a/up_aws/opt/aws_utils/tests/__init__.py b/up_aws/opt/aws_utils/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/up_aws/pyproject.toml b/up_aws/pyproject.toml index a5b23d6..4b1e429 100644 --- a/up_aws/pyproject.toml +++ b/up_aws/pyproject.toml @@ -13,3 +13,5 @@ python = "^3.12" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + + diff --git a/up_aws/up_aws/containers_for_prompt.py b/up_aws/up_aws/containers_for_prompt.py index 1ff44da..f4eaeff 100644 --- a/up_aws/up_aws/containers_for_prompt.py +++ b/up_aws/up_aws/containers_for_prompt.py @@ -4,16 +4,34 @@ def get_homedir(): return os.environ['HOME'] -def run_container(prompt): +def run_container(prompt, command=None, working_dir=None): + _command = command if command else prompt + _working_dir = working_dir if working_dir else "/" return up.ContainerRun( - image="amazon/aws-cli", + image="caravanacloud/up_aws", bash_wrap=False, volumes = { - get_homedir()+"/.aws":"/root/.aws" + get_homedir()+"/.aws":{ + "bind":"/root/.aws", + "mode":"ro" + } }, - command=prompt[1:]) + working_dir = _working_dir, + command=_command + ) + +def lookup_container(prompt): + match prompt: + case ["subs", *rest]: + return run_container(rest, + working_dir="/opt/aws_utils/", + command=["/home/up_user/.local/bin/poetry", "run", "subs"]) + return run_container(prompt) @up.hookimpl def containers_for_prompt(prompt): - return up.if_prompt_matches(run_container, prompt, "aws") + match prompt: + case ["aws", *rest]: + return lookup_container(rest) + return None diff --git a/uplib/uplib/containers.py b/uplib/uplib/containers.py index 07af708..b14c8a6 100644 --- a/uplib/uplib/containers.py +++ b/uplib/uplib/containers.py @@ -23,6 +23,8 @@ class ContainerRun: volumes: dict[str, str] = field(kw_only=True, default_factory=dict) auto_remove: bool = field(kw_only=True, default=True) network_mode: str = field(kw_only=True, default="host") + # TODO: user + working_dir: str = field(kw_only=True, default="/") # Feature Flags bash_wrap: bool = field(kw_only=True, default=False) @@ -45,7 +47,9 @@ def volumes_of(cls, run:ContainerRun): "mode": "rw" } } - result = settings_vols | default_vols + print(type(run.volumes)) + print(run.volumes) + result = run.volumes | settings_vols | default_vols return result def run(self, run: ContainerRun): @@ -60,6 +64,8 @@ def run(self, run: ContainerRun): name = run.name if run.name else generate_container_name(run) volumes = DockerContainers.volumes_of(run) ports = settings_maps.get("ports") + user = "up_user" + working_dir = run.working_dir console = Console() console.log(f"Running container: {name}") console.log({ @@ -69,6 +75,8 @@ def run(self, run: ContainerRun): "auto_remove": run.auto_remove, "volumes": volumes, "ports": ports, + "user": user, + "working_dir": working_dir, "detach": True }) try: @@ -79,6 +87,8 @@ def run(self, run: ContainerRun): auto_remove=run.auto_remove, volumes=volumes, ports=ports, + user=user, + working_dir=working_dir, detach=True ) for line in container.logs(stream=True): @@ -91,6 +101,7 @@ def run(self, run: ContainerRun): log.error("Failed to run container") log.debug("%s", run) log.error("%s", e) + raise e _container_name_pattern = '[^a-zA-Z0-9_.-]' def generate_container_name(run): diff --git a/uplib/uplib/main.py b/uplib/uplib/main.py index 257c50a..cb69409 100644 --- a/uplib/uplib/main.py +++ b/uplib/uplib/main.py @@ -21,8 +21,8 @@ def up_main(context: Context, prompt: Prompt): if not container_runs: log.error("No containers found, using defaults") container_runs = [default_container(prompt)] - for container in container_runs: - containers.run(container) + for container_run in container_runs: + containers.run(container_run) def default_container(prompt): @@ -33,18 +33,10 @@ def default_container(prompt): def containers_for_prompt(prompt) -> list[ContainerRun]: from_plugins = containers_from_plugins(prompt) - from_configs = containers_from_dynaconf(prompt) - result = from_plugins + from_configs - return result - - -def containers_from_dynaconf(prompt: list[str]) -> list[ContainerRun]: - return [] - + return from_plugins def containers_from_plugins(prompt: list[str]) -> list[ContainerRun]: - results = pm.hook.containers_for_prompt(prompt=prompt) - result = sum(results, []) + result = pm.hook.containers_for_prompt(prompt=prompt) if not result: - result = [] + return [] return result