diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..c51bb8e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,39 @@ +name: Run tests + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + -r requirements.txt \ + flake8 \ + pytest \ + . + - name: Lint + run: | + flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test + run: | + pytest --capture=sys --ignore=test/test_docker.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..991ab31 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,76 @@ +name: Release + +on: + release: + types: + - published + push: + branches: + - main # publish pre-release packages on every push to main + +jobs: + package: + name: Build package + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install dependencies + run: | + python -m pip install -U pip build + python -m pip install -r requirements.txt . + - name: Build dist + run: | + python -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + needs: + - package + environment: + name: release + url: https://pypi.org/p/aardvark + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + pypi-test-publish: + name: Upload release to TestPyPI + runs-on: ubuntu-latest + needs: + - package + environment: + name: release + url: https://test.pypi.org/p/aardvark + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 32ea6ea..0000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: python -cache: pip -python: -- 3.9 -matrix: - allow_failures: - - python: nightly - - python: pypy - - python: pypy3 -install: -- pip install -r requirements.txt -- python setup.py develop -- pip install flake8 -before_script: -- flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics -- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics -script: -- pytest --capture=sys --ignore=test/test_docker.py -deploy: - provider: pypi - user: __token__ - password: - secure: A+HimWMUsL9T4hN4mgHl3JPJ+/61Zd24OhZ1piDUZfp8GtfCSAd1UmxjyLSMl5+8CaALOYcndf3WWkGSLFz/DBi0Z0M+4YNwX+JfJzHXPWCcJ0lGG89VHi+TRcAEzaT95bhHGkh8OTq3aKqHlY9t6phdwb1S6zwNuvtv1FtJvBIVEsaQ9X7toOcY//7+VegmwPxUE07tP606aKLUXxNDlLiqAIRiEnRIHUB1yWzhY77k2CYEV31P4PvdAjy5wLJzd4CHpmQmZgzY7DgRTIq1AecjsCDR34O+erjvrgx31itpE9GpmPfz76ExhSx5MSYyFoE347FnsJnZXu1n2XgsvYUfx0rrQBX4ti0ozSWe7v+B96YFfr+m84AUJBDB4lFIxeNYxjhc5+9BH70WQkosOLdW/n0GMtcAvYQXo7wyLm7vqOkVgN5dlk1J29Z3F3GXJxNGo7/gytOK/m3UO3wJk8VqNeIO2CV7uRvQ7DwYoar9kyx88wN586nKahGQLdsMKcULujU3ofF0BHoylb0oKRzCf0A9fPlcbpk1M8RJv7VyDBnWlirqou8uflfs2P1tGCjgrCDtVW3U7ER4LBNvbO0wPT6/o2arcaF99Np3mIMIxNh1VUhDhGouUAfgTvKh+sZW/t9u+ZI6kloZ64UhDHODkAUwAfwledcF6GDVzOc= - skip_existing: true - on: - tags: true -notifications: - on_success: change - on_failure: change diff --git a/Dockerfile b/Dockerfile index 8ab8634..ba074e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3.8 RUN apt-get update -y \ && apt-get upgrade -y \ diff --git a/README.md b/README.md index 8cc13b8..b589a14 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Aardvark +Aardvark - Multi-Account AWS IAM Access Advisor API ======== [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/osstracker.svg)]() [![Discord chat](https://img.shields.io/discord/754080763070382130?logo=discord)](https://discord.gg/9kwMWa6) diff --git a/aardvark/__about__.py b/aardvark/__about__.py deleted file mode 100644 index f118e9d..0000000 --- a/aardvark/__about__.py +++ /dev/null @@ -1,16 +0,0 @@ -__all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", -] - -__title__ = "aardvark" -__summary__ = ("Multi-Account AWS IAM Access Advisor API") -__uri__ = "https://github.com/Netflix-Skunkworks/aardvark" - -__version__ = "0.3.5" - -__author__ = "Patrick Kelley, Travis McPeak, Patrick Sanders" -__email__ = "aardvark-maintainers@netflix.com" - -__license__ = "Apache License, Version 2.0" -__copyright__ = "Copyright 2020 {0}".format(__author__) diff --git a/aardvark/__init__.py b/aardvark/__init__.py index 6a6009d..e69de29 100644 --- a/aardvark/__init__.py +++ b/aardvark/__init__.py @@ -1,84 +0,0 @@ -#ensure absolute import for python3 -from __future__ import absolute_import - -import os.path -import logging -from logging import DEBUG, Formatter, StreamHandler -from logging.config import dictConfig -import sys - -from flask_sqlalchemy import SQLAlchemy -from flask import Flask -from flasgger import Swagger - -db = SQLAlchemy() - -from aardvark.view import mod as advisor_bp # noqa - -BLUEPRINTS = [ - advisor_bp -] - -API_VERSION = '1' - - -def create_app(): - app = Flask(__name__, static_url_path='/static') - Swagger(app) - - path = _find_config() - if not path: - print('No config') - app.config.from_pyfile('_config.py') - else: - app.config.from_pyfile(path) - - # For ELB and/or Eureka - @app.route('/healthcheck') - def healthcheck(): - """Healthcheck - Simple healthcheck that indicates the services is up - --- - responses: - 200: - description: service is up - """ - return 'ok' - - # Blueprints - for bp in BLUEPRINTS: - app.register_blueprint(bp, url_prefix="/api/{0}".format(API_VERSION)) - - # Extensions: - db.init_app(app) - setup_logging(app) - - return app - - -def _find_config(): - """Search for config.py in order of preference and return path if it exists, else None""" - CONFIG_PATHS = [os.path.join(os.getcwd(), 'config.py'), - '/etc/aardvark/config.py', - '/apps/aardvark/config.py'] - for path in CONFIG_PATHS: - if os.path.exists(path): - return path - return None - - -def setup_logging(app): - if not app.debug: - if app.config.get('LOG_CFG'): - # initialize the Flask logger (removes all handlers) - app.logger - dictConfig(app.config.get('LOG_CFG')) - app.logger = logging.getLogger(__name__) - else: - handler = StreamHandler(stream=sys.stderr) - - handler.setFormatter(Formatter( - '%(asctime)s %(levelname)s: %(message)s ' - '[in %(pathname)s:%(lineno)d]')) - app.logger.setLevel(app.config.get('LOG_LEVEL', DEBUG)) - app.logger.addHandler(handler) diff --git a/aardvark/app.py b/aardvark/app.py new file mode 100644 index 0000000..1ad9a51 --- /dev/null +++ b/aardvark/app.py @@ -0,0 +1,81 @@ +import os.path +import logging +from logging import DEBUG, Formatter, StreamHandler +from logging.config import dictConfig +import sys + +from flask_sqlalchemy import SQLAlchemy +from flask import Flask +from flasgger import Swagger + +db = SQLAlchemy() + +from aardvark.view import mod as advisor_bp # noqa + +BLUEPRINTS = [ + advisor_bp +] + +API_VERSION = '1' + + +def create_app(): + app = Flask(__name__, static_url_path='/static') + Swagger(app) + + path = _find_config() + if not path: + print('No config') + app.config.from_pyfile('_config.py') + else: + app.config.from_pyfile(path) + + # For ELB and/or Eureka + @app.route('/healthcheck') + def healthcheck(): + """Healthcheck + Simple healthcheck that indicates the services is up + --- + responses: + 200: + description: service is up + """ + return 'ok' + + # Blueprints + for bp in BLUEPRINTS: + app.register_blueprint(bp, url_prefix="/api/{0}".format(API_VERSION)) + + # Extensions: + db.init_app(app) + setup_logging(app) + + return app + + +def _find_config(): + """Search for config.py in order of preference and return path if it exists, else None""" + CONFIG_PATHS = [os.path.join(os.getcwd(), 'config.py'), + '/etc/aardvark/config.py', + '/apps/aardvark/config.py'] + for path in CONFIG_PATHS: + if os.path.exists(path): + return path + return None + + +def setup_logging(app): + if not app.debug: + if app.config.get('LOG_CFG'): + # initialize the Flask logger (removes all handlers) + app.logger + dictConfig(app.config.get('LOG_CFG')) + app.logger = logging.getLogger(__name__) + else: + handler = StreamHandler(stream=sys.stderr) + + handler.setFormatter(Formatter( + '%(asctime)s %(levelname)s: %(message)s ' + '[in %(pathname)s:%(lineno)d]')) + app.logger.setLevel(app.config.get('LOG_LEVEL', DEBUG)) + app.logger.addHandler(handler) diff --git a/aardvark/manage.py b/aardvark/manage.py index f496ce2..97ebbe0 100644 --- a/aardvark/manage.py +++ b/aardvark/manage.py @@ -1,15 +1,8 @@ -# ensure absolute import for python3 -from __future__ import absolute_import - import os -try: - import queue as Queue # Queue renamed to queue in py3 -except ModuleNotFoundError: - import Queue +import queue import re import threading -import better_exceptions # noqa from blinker import Signal from bunch import Bunch from flask import current_app @@ -18,22 +11,12 @@ from swag_client.exceptions import InvalidSWAGDataException from swag_client.util import parse_swag_config_options -from aardvark import create_app, db +from aardvark.app import create_app, db from aardvark.updater import AccountToUpdate -try: # Python 2 - raw_input -except NameError: # Python 3 - raw_input = input - -try: # Python 2 - unicode -except NameError: # Python 3 - unicode = str - manager = Manager(create_app) -ACCOUNT_QUEUE = Queue.Queue() +ACCOUNT_QUEUE = queue.Queue() DB_LOCK = threading.Lock() QUEUE_LOCK = threading.Lock() UPDATE_DONE = False @@ -147,9 +130,9 @@ def create_db(): # All of these default to None rather than the corresponding DEFAULT_* values # so we can tell whether they were passed or not. We don't prompt for any of # the options that were passed as parameters. -@manager.option('-a', '--aardvark-role', dest='aardvark_role_param', type=unicode) -@manager.option('-b', '--swag-bucket', dest='bucket_param', type=unicode) -@manager.option('-d', '--db-uri', dest='db_uri_param', type=unicode) +@manager.option('-a', '--aardvark-role', dest='aardvark_role_param', type=str) +@manager.option('-b', '--swag-bucket', dest='bucket_param', type=str) +@manager.option('-d', '--db-uri', dest='db_uri_param', type=str) @manager.option('--num-threads', dest='num_threads_param', type=int) @manager.option('--no-prompt', dest='no_prompt', action='store_true', default=False) def config(aardvark_role_param, bucket_param, db_uri_param, num_threads_param, no_prompt): @@ -202,10 +185,10 @@ def config(aardvark_role_param, bucket_param, db_uri_param, num_threads_param, n write_swag = True else: print('\nAardvark can use SWAG to look up accounts. See {repo_url}'.format(repo_url=SWAG_REPO_URL)) - use_swag = raw_input('Do you use SWAG to track accounts? [yN]: ') + use_swag = input('Do you use SWAG to track accounts? [yN]: ') if len(use_swag) > 0 and 'yes'.startswith(use_swag.lower()): bucket_prompt = 'SWAG_BUCKET [{default}]: '.format(default=DEFAULT_SWAG_BUCKET) - bucket = raw_input(bucket_prompt) or DEFAULT_SWAG_BUCKET + bucket = input(bucket_prompt) or DEFAULT_SWAG_BUCKET write_swag = True else: write_swag = False @@ -214,9 +197,9 @@ def config(aardvark_role_param, bucket_param, db_uri_param, num_threads_param, n db_uri_prompt = 'DATABASE URI [{default}]: '.format(default=default_db_uri) num_threads_prompt = '# THREADS [{default}]: '.format(default=DEFAULT_NUM_THREADS) - aardvark_role = aardvark_role_param or raw_input(aardvark_role_prompt) or DEFAULT_AARDVARK_ROLE - db_uri = db_uri_param or raw_input(db_uri_prompt) or default_db_uri - num_threads = num_threads_param or raw_input(num_threads_prompt) or DEFAULT_NUM_THREADS + aardvark_role = aardvark_role_param or input(aardvark_role_prompt) or DEFAULT_AARDVARK_ROLE + db_uri = db_uri_param or input(db_uri_prompt) or default_db_uri + num_threads = num_threads_param or input(num_threads_prompt) or DEFAULT_NUM_THREADS log = """LOG_CFG = { 'version': 1, @@ -268,8 +251,8 @@ def config(aardvark_role_param, bucket_param, db_uri_param, num_threads_param, n filedata.write(log) -@manager.option('-a', '--accounts', dest='accounts', type=unicode, default='all') -@manager.option('-r', '--arns', dest='arns', type=unicode, default='all') +@manager.option('-a', '--accounts', dest='accounts', type=str, default='all') +@manager.option('-r', '--arns', dest='arns', type=str, default='all') def update(accounts, arns): """ Asks AWS for new Access Advisor information. @@ -402,7 +385,7 @@ def run(self, *args, **kwargs): app = WSGIApplication() - app.app_uri = 'aardvark:create_app()' + app.app_uri = 'aardvark.app:create_app()' return app.run() diff --git a/aardvark/model.py b/aardvark/model.py index a629faa..70558b9 100644 --- a/aardvark/model.py +++ b/aardvark/model.py @@ -1,6 +1,3 @@ -#ensure absolute import for python3 -from __future__ import absolute_import - import datetime from flask import current_app @@ -9,7 +6,7 @@ from sqlalchemy.orm import relationship from sqlalchemy.schema import ForeignKey -from aardvark import db +from aardvark.app import db from aardvark.utils.sqla_regex import String diff --git a/aardvark/updater/__init__.py b/aardvark/updater/__init__.py index c1be9c3..70c77b3 100644 --- a/aardvark/updater/__init__.py +++ b/aardvark/updater/__init__.py @@ -1,6 +1,3 @@ -# ensure absolute import for python3 -from __future__ import absolute_import - import time from blinker import Signal diff --git a/aardvark/utils/sqla_regex.py b/aardvark/utils/sqla_regex.py index d0568e0..8efd384 100644 --- a/aardvark/utils/sqla_regex.py +++ b/aardvark/utils/sqla_regex.py @@ -5,18 +5,14 @@ # courtesy of Xion: http://xion.io/post/code/sqlalchemy-regex-filters.html -#ensure absolute import for python3 -from __future__ import absolute_import - import re +import sqlite3 from sqlalchemy import String as _String, event, exc from sqlalchemy.engine import Engine from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.expression import BinaryExpression, func, literal from sqlalchemy.sql.operators import custom_op -import sqlite3 - __all__ = ['String'] diff --git a/aardvark/view.py b/aardvark/view.py index 3c0dccf..af5f3d0 100644 --- a/aardvark/view.py +++ b/aardvark/view.py @@ -1,9 +1,4 @@ -#ensure absolute import for python3 -from __future__ import absolute_import - -import better_exceptions # noqa import datetime -import json from flask import abort, jsonify from flask import Blueprint diff --git a/requirements.txt b/requirements.txt index dc67ef8..037bf30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,43 @@ -aniso8601==8.0.0 -astroid==2.4.2 -attrs==19.3.0 -better-exceptions==0.1.7 blinker boto3==1.20.43 botocore==1.23.43 bunch==1.0.1 +cloudaux>=1.8.0 # pinned +Flask==1.0.2 +Flask-RESTful==0.3.5 +Flask-Script==2.0.5 +Flask-SQLAlchemy>=2.5 # pinned +SQLAlchemy==1.3.10 +swag-client==0.4.6 + +# indirect +aniso8601==8.0.0 +astroid==2.4.2 +attrs==19.3.0 certifi==2023.7.22 chardet==3.0.4 Click==7.0 click-log==0.3.2 -cloudaux==1.8.0 decorator==4.4.0 deepdiff==3.3.0 defusedxml==0.6.0 docutils==0.15.2 dogpile.cache==0.8.0 flagpole==1.1.1 -flasgger==0.6.3 -Flask==1.0.2 -Flask-RESTful==0.3.5 -Flask-Script==2.0.5 -Flask-SQLAlchemy>=2.5 +flasgger==0.9.5 gunicorn==19.7.1 idna==2.8 importlib-metadata inflection==0.3.1 isort==4.3.21 itsdangerous==1.1.0 -Jinja2 +Jinja2==3.0.3 jmespath==0.9.4 joblib==0.14.0 jsonpickle==1.2 jsonschema==3.1.1 lazy-object-proxy==1.4.2 -MarkupSafe==1.1.1 +MarkupSafe marshmallow==2.20.5 mccabe==0.6.1 mistune==0.8.4 @@ -51,8 +54,6 @@ requests==2.31.0 retrying==1.3.3 simplejson==3.16.0 six==1.12.0 -SQLAlchemy==1.3.10 -swag-client==0.4.6 tabulate==0.8.5 tqdm==4.40.0 Werkzeug==0.16.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 0f4a1cb..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[metadata] -description-file = README.md - -[wheel] -universal = 1 - -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 \ No newline at end of file diff --git a/setup.py b/setup.py index 408fa97..a373418 100644 --- a/setup.py +++ b/setup.py @@ -1,82 +1,20 @@ -""" -Aardvark -===== -Multi-Account AWS IAM Access Advisor API -:copyright: (c) 2017 by Netflix -:license: Apache, see LICENSE for more details. -""" -from __future__ import absolute_import - -import sys -import os.path - -from setuptools import setup, find_packages - - -ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) - -# When executing the setup.py, we need to be able to import ourselves, this -# means that we need to add the src/ directory to the sys.path. -sys.path.insert(0, ROOT) - -about = {} -with open(os.path.join(ROOT, "aardvark", "__about__.py")) as f: - exec(f.read(), about) - - -install_requires = [ - 'requests~=2.22.0', - 'better_exceptions==0.1.7', - 'blinker~=1.4', - 'Bunch==1.0.1', - 'Flask-SQLAlchemy~=2.5', - 'cloudaux>=1.8.0', - 'Flask==1.0.2', - 'Jinja2==3.0.3', - 'Flask-RESTful==0.3.5', - 'Flask-Script==2.0.5', - 'flasgger==0.9.5', - 'gunicorn==19.7.1', - 'itsdangerous==1.1.0', - 'psycopg2-binary~=2.9.3', - 'pytz==2017.2', - 'swag-client==0.4.6', - 'tqdm==4.40.0', - 'deepdiff==3.3.0' # Pinning to last py2 compatible version. Needed for swag-client. -] - -tests_require = [ - 'pexpect>=4.2.1' -] - -docs_require = [ -] - -dev_requires = [ -] +from setuptools import setup setup( - name=about["__title__"], - version=about["__version__"], - author=about["__author__"], - author_email=about["__email__"], - url=about["__uri__"], - description=about["__summary__"], - long_description=open(os.path.join(ROOT, 'README.md')).read(), - long_description_content_type="text/markdown", - packages=find_packages(), - include_package_data=True, - zip_safe=False, - install_requires=install_requires, + name="aardvark", + author="Patrick Kelley, Travis McPeak, Patrick Sanders", + author_email="aardvark-maintainers@netflix.com", + url="https://github.com/Netflix-Skunkworks/aardvark", + setup_requires="setupmeta", + versioning="dev", extras_require={ - 'tests': tests_require, - 'docs': docs_require, - 'dev': dev_requires, + 'tests': ['pexpect>=4.2.1'], }, entry_points={ 'console_scripts': [ 'aardvark = aardvark.manage:main', ], - } + }, + python_requires=">=3.8,<3.10", )