From 4a8f1448c45090338a02f971663caee57349b74d Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Fri, 21 Jul 2017 13:00:34 +0200 Subject: [PATCH] Add getenv filter --- .appveyor.yml | 41 ++-- .codecov.yml | 27 --- .gitlab-ci.yml | 84 ++++---- .travis.yml | 11 +- Changelog.md | 19 ++ Dockerfile | 30 ++- Dockerfile.base | 14 ++ Makefile | 21 +- VERSION | 2 +- appr/__init__.py | 2 +- appr/auth.py | 2 - appr/commands/command_base.py | 4 +- appr/commands/jsonnet.py | 3 +- appr/formats/appr/kub.py | 2 - appr/formats/appr/kub_base.py | 5 +- appr/jsonnet/lib/appr-native-ext.libsonnet | 9 +- appr/models/package_base.py | 12 +- appr/platforms/kubernetes.py | 8 +- appr/plugins/helm.py | 1 - appr/render_jsonnet.py | 4 +- appr/template_filters.py | 10 +- appr/tests/test_apiserver.py | 9 + appr/tests/test_models.py | 28 ++- appr/utils.py | 24 ++- appveyor/install.ps1 | 229 +++++++++++++++++++++ appveyor/run_with_env.cmd | 88 ++++++++ bin/apprc | 3 + requirements_dev.txt | 28 ++- requirements_tests.txt | 10 - setup.cfg | 3 +- setup.py | 11 +- tests/commands/test_inspect.py | 6 - tests/platforms/test_kubernetes.py | 10 +- 33 files changed, 572 insertions(+), 188 deletions(-) delete mode 100644 .codecov.yml create mode 100644 Dockerfile.base create mode 100644 appveyor/install.ps1 create mode 100644 appveyor/run_with_env.cmd create mode 100755 bin/apprc delete mode 100644 requirements_tests.txt diff --git a/.appveyor.yml b/.appveyor.yml index b1c3d25..a3598be 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,5 +1,8 @@ +image: +- Visual Studio 2015 environment: - + global: + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" matrix: # For Python versions available on Appveyor, see @@ -13,46 +16,46 @@ environment: # - PYTHON: "C:\\Python35" - PYTHON: "C:\\Python27-x64" BIN_NAME: appr-%APPVEYOR_REPO_TAG_NAME%-win-x64 + PYTHON_ARCH: "64" + PYTHON_VERSION: "2.7.x" # - PYTHON: "C:\\Python33-x64" # - PYTHON: "C:\\Python34-x64" # - PYTHON: "C:\\Python35-x64" install: + - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\"" - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "pip install setuptools>=6.0" - "python --version" - # Upgrade to the latest version of pip to avoid it displaying warnings # about it being out of date. - "pip install --disable-pip-version-check --user --upgrade pip" - # We need wheel installed to build wheels - - "pip install -r requirements_tests.txt" - - "pip install -U pyinstaller" + # # We need wheel installed to build wheels + - "pip install -r requirements_dev.txt" + # - "pip install jsonnet" - "pip install -e ." - + - "pip install -e git+https://github.com/pyinstaller/pyinstaller.git#egg=pyinstaller" build: off -test_script: + +test_script: [] # Run the project tests - - "py.test --cov=appr --cov-report=html --cov-report=term-missing --verbose tests" + #- "py.test --cov=appr --cov-report=html --cov-report=term-missing --verbose tests" after_test: # If tests are successful, create binary packages for the project. - - "pyinstaller --onefile bin/appr" + - "pyinstaller --add-data appr/jsonnet;appr/jsonnet --onefile bin/appr" - ps: "ls dist" - "copy dist\\appr.exe %BIN_NAME%.exe" - - "mkdir registry" - - "copy dist\\appr.exe registry\\appr.exe" - - "copy appr\\commands\\plugins\\helm\\appr.sh registry\\appr.sh" - - "copy appr\\commands\\plugins\\helm\\plugin.yaml registry\\plugin.yaml" - - "7z a registry-%BIN_NAME%-helm-plugin.zip registry\\*.*" - - ps: "ls dist" - - ps: "ls registry" + # - "mkdir registry" + # - "copy dist\\appr.exe registry\\appr.exe" + # - "7z a registry-%BIN_NAME%-helm-plugin.zip registry\\*.*" + # - ps: "ls dist" + # - ps: "ls registry" + - "C:\\projects\\appr\\dist\\appr.exe version quay.io" artifacts: # Archive the generated packages in the ci.appveyor.com build report. - path: "%BIN_NAME%.exe" - - path: registry-%BIN_NAME%-helm-plugin.zip - type: zip - diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index dbd3302..0000000 --- a/.codecov.yml +++ /dev/null @@ -1,27 +0,0 @@ -codecov: - notify: - require_ci_to_pass: yes - token: 82510329-4647-4e4d-88ee-98ee1b2536c4 - -coverage: - precision: 1 - round: down - range: "70...100" - - status: - project: yes - patch: yes - changes: no - -parsers: - gcov: - branch_detection: - conditional: yes - loop: yes - method: no - macro: no - -comment: off - # layout: "reach, diff, flags, files, footer" - # behavior: default - # require_changes: no diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c82143a..766fe53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,12 @@ stages: - init - - unit-tests - - code-style - - integration-tests + - tests - build - release + variables: FAILFASTCI_NAMESPACE: 'failfast-ci' IMAGE: quay.io/appr/appr - CODECOV: 82510329-4647-4e4d-88ee-98ee1b2536c4 cache: paths: @@ -22,23 +20,21 @@ cache: - make test db=$APPR_TEST_DB tags: - kubernetes - image: quay.io/appr/appr:test + image: quay.io/appr/appr:base test-filesystem: <<: *job - stage: unit-tests + stage: tests script: - pip install -U python-coveralls - - pip install -U codecov - make test - coveralls - - codecov --token=$CODECOV variables: APPR_TEST_DB: filesystem test-etcd: <<: *job - stage: integration-tests + stage: tests services: - quay.io/coreos/etcd:v3.0.6 variables: @@ -47,7 +43,7 @@ test-etcd: test-redis: <<: *job - stage: integration-tests + stage: tests services: - redis:3 variables: @@ -56,30 +52,41 @@ test-redis: flake8: <<: *job - stage: code-style + stage: tests script: - make flake8 pylint: <<: *job - stage: code-style + stage: tests script: - pip install pylint - make pylint + compile: - <<: *job + variables: + GIT_STRATEGY: none + DOCKER_HOST: tcp://localhost:2375 + DOCKER_DRIVER: overlay2 + image: docker:17.06-git stage: release + services: + - docker:17.06-dind script: - - pip install pyinstaller - - pyinstaller --onefile bin/appr --hidden-import gunicorn.glogging --hidden-import gunicorn.workers.gthread --hidden-import gunicorn.workers - artifacts: - paths: - - dist/appr + - mkdir /dist + - docker run --rm --entrypoint="/bin/sh" -v /dist:/dist quay.io/appr/appr:$CI_BUILD_REF_NAME -c 'cp /opt/bin/appr /dist/appr-alpine-x64' + - docker run --rm -v /dist:/dist quay.io/ant31/ghrelease:master --api-key $GITHUB_TOKEN --repo app-registry/appr --file /dist/appr-alpine-x64 --tag $CI_BUILD_REF_NAME + tags: + - kubernetes + only: + - tags + .docker: &docker variables: DOCKER_HOST: tcp://localhost:2375 + DOCKER_DRIVER: overlay2 image: docker:17.06-git before_script: - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io @@ -93,45 +100,26 @@ docker-build: stage: build script: - docker build -t $IMAGE:$CI_BUILD_REF_NAME . - except: - - tags - - master + - docker push $IMAGE:$CI_BUILD_REF_NAME -docker-build: +docker-build-base: <<: *docker stage: init script: - - make build-test + - docker build -f Dockerfile.base -t quay.io/appr/appr:base . + - docker push quay.io/appr/appr:base when: manual -.docker-push: &docker-push +docker-release: <<: *docker stage: release script: - - docker build -t $IMAGE:$CI_BUILD_REF_NAME . - - docker push $IMAGE:$CI_BUILD_REF_NAME + - docker build --build-arg with_kubectl=true -t quay.io/appr/appr:v$(VERSION)-kubectl . + - docker build --build-arg with_kubectl=false -t quay.io/appr/appr . -docker-release: - <<: *docker-push + - docker tag quay.io/appr/appr:v$(VERSION)-kubectl quay.io/appr/appr:kubectl + - docker push quay.io/appr/appr:v$(VERSION)-kubectl + - docker push quay.io/appr/appr:kubectl + - docker push quay.io/appr/appr only: - - tags - master - -docker-push: - <<: *docker-push - except: - - tags - - master - when: manual - -pypi-release: - <<: *job - image: python:2.7 - stage: release - script: - - make release - tags: - - kubernetes - when: manual - only: - - tags diff --git a/.travis.yml b/.travis.yml index 4bf9d76..2964736 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,10 @@ matrix: - BIN_NAME=appr-$TRAVIS_OS_NAME-x64 install: - - if [ "$MAKECMD" == "test" ] ; then pip install -U pyinstaller ; fi + - if [ "$MAKECMD" == "test" ] ; then pip install https://github.com/pyinstaller/pyinstaller/archive/develop.zip ; fi - if [ "$MAKECMD" == "test" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then pip install -U coveralls ; fi - - if [ "$MAKECMD" == "test" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then pip install -U codecov ; fi - - pip install -r requirements_tests.txt - - pip install -r requirements_dev.txt + - pip install -r requirements_dev.txt -U + - pip install jsonnet - pip install -e . script: @@ -31,13 +30,12 @@ script: after_success: - echo $BIN_NAME - - if [ "$MAKECMD" == "test" ] ; then pyinstaller --onefile bin/appr --hidden-import gunicorn.glogging --hidden-import gunicorn.workers.gthread --hidden-import gunicorn.workers ; fi + - if [ "$MAKECMD" == "test" ] ; then pyinstaller --add-data "appr/jsonnet/:appr/jsonnet" --onefile bin/appr ; fi - if [ "$MAKECMD" == "test" ] ; then mkdir -p appr-helm/registry; fi - if [ "$MAKECMD" == "test" ] ; then mv dist/appr $BIN_NAME; fi - if [ "$MAKECMD" == "test" ] ; then chmod +x $BIN_NAME; fi - if [ "$MAKECMD" == "test" ] ; then cp $BIN_NAME appr-helm/registry/appr; fi - if [ "$MAKECMD" == "test" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then coveralls ; fi - - if [ "$MAKECMD" == "test" ] && [ "$TRAVIS_OS_NAME" == "linux" ] ; then codecov ; fi deploy: @@ -46,6 +44,7 @@ deploy: overwrite: true file: - $BIN_NAME + - bin/apprc skip_cleanup: true on: tags: true diff --git a/Changelog.md b/Changelog.md index 34c6268..1e8c3d5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,22 @@ +## 0.7.0 Released on 2017-07-24 + +- Introduce client-side deployments handling multiple formats: + - [ ] [Helm](https://github.com/kubernetes/helm) + - [x] [appr](https://github.com/coreos/kpm) + - [ ] Plain kubernetes configuration + +- Add jsonnet support for future integration with [ksonnet](https://github.com/ksonnet/ksonnet-lib) + +- New commands: + +``` shell +# Resolve a jsonnet file, with a set of nativeExtension available +appr jsonnet FILE + +# Client side deployment command +appr deploy quay.io/ant31/kube-lego --namespace kube-lego +``` + ## 0.6.2 Released on 2017-07-10 - Add options to allow unverified certs or custom CA. diff --git a/Dockerfile b/Dockerfile index f5e895e..e6b59ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,41 @@ -FROM six8/pyinstaller-alpine:latest +FROM quay.io/appr/appr:base ENV workdir /opt/appr-server RUN mkdir -p $workdir RUN apk --no-cache --update add python py-pip openssl ca-certificates RUN apk --no-cache --update add --virtual build-dependencies \ - python-dev build-base wget openssl-dev libffi-dev -ADD . $workdir + python-dev build-base wget openssl-dev libffi-dev libstdc++ +COPY . $workdir WORKDIR $workdir - -RUN pip install pip -U \ - && pip install gunicorn -U && pip install -e . +RUN pip install jsonnet +RUN pip install -e . RUN /pyinstaller/pyinstaller.sh --onefile --noconfirm \ + --add-data "appr/jsonnet/:appr/jsonnet" \ --onefile \ + --hidden-import _jsonnet \ --log-level DEBUG \ --clean \ bin/appr from alpine:latest +ARG with_kubectl=false +ENV HOME=/appr +RUN mkdir -p /opt/bin && mkdir -p /opt/bin/k8s && mkdir $HOME && mkdir -p $HOME/local +ENV PATH=${PATH}:/opt/bin:/opt/bin/k8s RUN apk --no-cache add ca-certificates -COPY --from=0 /opt/appr-server/dist/appr /usr/bin/ +ENV WITH_KUBECTL ${with_kubectl} +RUN if [ "$WITH_KUBECTL" = true ]; then \ + apk add --update curl && rm -rf /var/cache/apk/* \ + && curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ + && chmod +x ./kubectl \ + && cp ./kubectl /opt/bin/k8s; \ + fi + +WORKDIR /appr/local + +COPY --from=0 /opt/appr-server/dist/appr /opt/bin RUN appr plugins install helm + ENTRYPOINT ["appr"] diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 0000000..7ff90b7 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,14 @@ +FROM six8/pyinstaller-alpine:apline-v3.4-pyinstaller-develop + +ENV workdir /opt/appr-server +RUN mkdir -p $workdir +RUN apk --no-cache --update add python py-pip openssl ca-certificates git curl +RUN apk --no-cache --update add --virtual build-dependencies \ + python-dev build-base wget openssl-dev libffi-dev libstdc++ +COPY . $workdir +WORKDIR $workdir + +RUN pip install pip -U \ + && pip install -r requirements_dev.txt \ + && pip install -e . + diff --git a/Makefile b/Makefile index 06c2712..f2f3e85 100644 --- a/Makefile +++ b/Makefile @@ -108,17 +108,22 @@ yapf-diff: yapf-test: yapf-diff if [ `yapf -r appr -d | wc -l` -gt 0 ] ; then false ; else true ;fi - docker-build: - docker build -t quay.io/appr/appr:$(VERSION) . - docker tag quay.io/appr/appr:$(VERSION) quay.io/appr/appr:latest + docker build -t quay.io/appr/appr:v$(VERSION) . + docker tag quay.io/appr/appr:v$(VERSION) quay.io/appr/appr:latest + +docker-kubectl: + docker build --build-arg with_kubectl=true -t quay.io/appr/appr:v$(VERSION)-kubectl . + docker tag quay.io/appr/appr:v$(VERSION)-kubectl quay.io/appr/appr:kubectl + docker push quay.io/appr/appr:v$(VERSION)-kubectl + docker push quay.io/appr/appr:kubectl docker-push-tag: docker-push - docker push quay.io/appr/appr:$(VERSION) + docker push quay.io/appr/appr:v$(VERSION) -docker-push: docker-build +docker-push: docker-build docker-kubectl docker push quay.io/appr/appr:latest -docker-test: - docker build -t quay.io/appr/appr:test -f test.Dockerfile . - docker push quay.io/appr/appr:test +docker-base: + docker build -f Dockerfile.base -t quay.io/appr/appr:base . + docker push quay.io/appr/appr:base diff --git a/VERSION b/VERSION index b616048..faef31a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.2 +0.7.0 diff --git a/appr/__init__.py b/appr/__init__.py index 3ad6e2d..9814b95 100644 --- a/appr/__init__.py +++ b/appr/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- __author__ = 'Antoine Legrand' __email__ = '2t.antoine@gmail.com' -__version__ = '0.6.2' +__version__ = '0.7.0' diff --git a/appr/auth.py b/appr/auth.py index 5a1ef36..f65359e 100644 --- a/appr/auth.py +++ b/appr/auth.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import os import os.path diff --git a/appr/commands/command_base.py b/appr/commands/command_base.py index b14c497..ab3c02e 100644 --- a/appr/commands/command_base.py +++ b/appr/commands/command_base.py @@ -172,9 +172,7 @@ def _add_output_option(cls, parser): 'text', 'none', 'json', 'yaml'], help="output format") @classmethod - def _add_mediatype_option(cls, parser, default=None, required=False): - if default is None: - default = cls.default_media_type + def _add_mediatype_option(cls, parser, default="-", required=False): if default is not None: required = False diff --git a/appr/commands/jsonnet.py b/appr/commands/jsonnet.py index 8235d3c..aa27185 100644 --- a/appr/commands/jsonnet.py +++ b/appr/commands/jsonnet.py @@ -1,11 +1,12 @@ import os import json import argparse -from appr.render_jsonnet import RenderJsonnet from appr.commands.command_base import CommandBase, LoadVariables +from appr.render_jsonnet import RenderJsonnet class JsonnetCmd(CommandBase): + name = 'jsonnet' help_message = "Resolve a jsonnet file" diff --git a/appr/formats/appr/kub.py b/appr/formats/appr/kub.py index d7afd37..528ee8d 100644 --- a/appr/formats/appr/kub.py +++ b/appr/formats/appr/kub.py @@ -1,6 +1,5 @@ from __future__ import absolute_import, division, print_function -import hashlib import json import logging import os @@ -51,7 +50,6 @@ def _resource_build(self, kub, resource): # @TODO do it in jsonnet def _annotate_resource(self, kub, resource): - sha = None if 'annotations' not in resource['value']['metadata']: resource['value']['metadata']['annotations'] = {} diff --git a/appr/formats/appr/kub_base.py b/appr/formats/appr/kub_base.py index 2ecb2f1..9ac05a7 100644 --- a/appr/formats/appr/kub_base.py +++ b/appr/formats/appr/kub_base.py @@ -113,9 +113,8 @@ def _fetch_deps(self): 'variables': self.variables} kub = self.kubClass(dep['name'], endpoint=self.endpoint, version=parse_version_req(dep.get('version', None)), - variables=variables, - resources=dep.get('resources', None), shards=dep.get( - 'shards', None), namespace=self.namespace) + variables=variables, resources=dep.get('resources', None), + shards=dep.get('shards', None), namespace=self.namespace) self._dependencies.append(kub) else: self._dependencies.append(self) diff --git a/appr/jsonnet/lib/appr-native-ext.libsonnet b/appr/jsonnet/lib/appr-native-ext.libsonnet index 26e3bcc..07243ad 100644 --- a/appr/jsonnet/lib/appr-native-ext.libsonnet +++ b/appr/jsonnet/lib/appr-native-ext.libsonnet @@ -19,6 +19,11 @@ std.native("read")(filepath, encode) ), + # Get a ENV value + getenv(name, default=null):: ( + std.native("getenv")(name, default) + ), + # Random alpha-numeric string of length `size` randAlphaNum(size=32, seed=""):: ( std.native("rand_alphanum")(std.toString(size), seed=seed) @@ -90,9 +95,9 @@ std.native("yaml_loads")(data)), tests: { - capitalize: (assert apprnative.capitalize("test") == 'Test'; true), b64encode: (assert apprnative.b64encode("toto") == "dG90bw=="; true), b64decode: (assert apprnative.b64decode(apprnative.b64encode("toto")) == "toto"; true), - + getenv: (assert std.length(apprnative.getenv("HOME")) > 0; + assert apprnative.getenv("BAD_ENV_APPR", "default_value") == "default_value"; true), }, } diff --git a/appr/models/package_base.py b/appr/models/package_base.py index befb0cc..5b304a2 100644 --- a/appr/models/package_base.py +++ b/appr/models/package_base.py @@ -175,8 +175,6 @@ def get(cls, package, release, media_type): returns: (package blob(targz) encoded in base64, release) """ p = cls(package, release) - if media_type == "-": - media_type = cls._find_media_type(package, release) p.pull(release, media_type) return p @@ -194,9 +192,7 @@ def get_release(cls, package, release_query, stable=False): raise InvalidRelease(e.message, {"release": release_query}) def pull(self, release_query=None, media_type=None): - media_type = get_media_type(media_type) - if media_type is None: - media_type = self.media_type + # Find release if release_query is None: release_query = self.release package = self.package @@ -206,6 +202,12 @@ def pull(self, release_query=None, media_type=None): package), {"package": package, "release_query": release_query}) + # Find media_type + if media_type == "-": + media_type = self._find_media_type(package, str(release)) + media_type = get_media_type(media_type) + if media_type is None: + media_type = self.media_type self.data = self._fetch(package, str(release), media_type) return self diff --git a/appr/platforms/kubernetes.py b/appr/platforms/kubernetes.py index ddf2321..b65de3f 100644 --- a/appr/platforms/kubernetes.py +++ b/appr/platforms/kubernetes.py @@ -114,10 +114,12 @@ def _resource_load(self): def _gethash(self, src): # Copy rand value - if (src is not None and ANNOTATIONS['rand'] in src['metadata'].get('annotations', {}) - and ANNOTATIONS['rand'] not in self.obj['metadata']['annotations']): - self.obj['metadata']['annotations'][ANNOTATIONS['rand']] = src['metadata']['annotations'][ANNOTATIONS['rand']] + if (src is not None and ANNOTATIONS['rand'] in src['metadata'].get('annotations', {}) and + ANNOTATIONS['rand'] not in self.obj['metadata']['annotations']): + self.obj['metadata']['annotations'][ANNOTATIONS['rand']] = src['metadata'][ + 'annotations'][ANNOTATIONS['rand']] + # TODO(ant31) it should hash before the custom annotations if ANNOTATIONS['hash'] in self.obj['metadata'].get('annotations', {}): if self.obj['metadata']['annotations'][ANNOTATIONS['hash']] is None: sha = hashlib.sha256(json.dumps(self.obj, sort_keys=True)).hexdigest() diff --git a/appr/plugins/helm.py b/appr/plugins/helm.py index cc08c6f..e73871d 100644 --- a/appr/plugins/helm.py +++ b/appr/plugins/helm.py @@ -10,7 +10,6 @@ from appr.pack import ApprPackage from appr.utils import mkdir_p, parse_package_name, parse_version_req - DEFAULT_CHARTS = "appr_charts" diff --git a/appr/render_jsonnet.py b/appr/render_jsonnet.py index 6468ce9..0f96b34 100644 --- a/appr/render_jsonnet.py +++ b/appr/render_jsonnet.py @@ -5,8 +5,6 @@ import os import os.path import re - -import _jsonnet import jinja2 import yaml @@ -96,6 +94,8 @@ def import_callback(self, path, rel): raise RuntimeError('File not found') def render_jsonnet(self, manifeststr, tla_codes=None): + # @TODO(ant31): workaround until jsonnet compile on windows + import _jsonnet try: json_str = _jsonnet.evaluate_snippet( # pylint: disable=no-member "snippet", manifeststr, import_callback=self.import_callback, diff --git a/appr/template_filters.py b/appr/template_filters.py index a1b7520..d057863 100644 --- a/appr/template_filters.py +++ b/appr/template_filters.py @@ -92,6 +92,10 @@ def jinja_env(): return jinjaenv +def getenv(name, default=None): + return os.getenv(name, default) + + def jinja_template(val, env=None): from appr.utils import convert_utf8 jinjaenv = jinja_env() @@ -120,6 +124,7 @@ def walkdir(path): files.append(os.path.join(root, filename)) return files + def jsonnet(val, env=None): from appr.render_jsonnet import RenderJsonnet from appr.utils import convert_utf8 @@ -199,12 +204,13 @@ def jinja_filters(): def jsonnet_callbacks(): filters = { + 'getenv': (('value', 'default', ), getenv), 'b64encode': (('value', ), b64encode), 'b64decode': (('value', ), b64decode), - 'path_exists': (('path', 'isfile',), path_exists), + 'path_exists': (('path', 'isfile', ), path_exists), 'walkdir': (('path', ), walkdir), 'listdir': (('path', ), listdir), - 'read': (('filepath', 'b64encodee',), readfile), + 'read': (('filepath', 'b64encodee', ), readfile), 'hash': (('data', 'hashtype'), get_hash), 'to_yaml': (('value', ), json_to_yaml), 'rand_alphanum': (('size', 'seed'), rand_alphanum), diff --git a/appr/tests/test_apiserver.py b/appr/tests/test_apiserver.py index a230c14..328b1da 100644 --- a/appr/tests/test_apiserver.py +++ b/appr/tests/test_apiserver.py @@ -112,6 +112,15 @@ def test_pull_package_no_release(self, db_with_data1, client): res = self.Client(client, self.headers()).get(url) assert res.status_code == 404 + def test_pull_package_default_media_type(self, db_with_data1, client): + package = "titi/rocketchat" + release = "1.0.1" + media_type = "-" + url = self._url_for("api/v1/packages/%s/%s/%s/pull" % (package, release, media_type)) + res = self.Client(client, self.headers()).get(url, params={'format': 'json'}) + assert res.status_code == 200 + assert self.json(res)['media_type'] == "kpm" + def test_pull_package_bad_release(self, db_with_data1, client): package = "titi/rocketchat" release = "abc" diff --git a/appr/tests/test_models.py b/appr/tests/test_models.py index 3a663b1..e428603 100644 --- a/appr/tests/test_models.py +++ b/appr/tests/test_models.py @@ -6,7 +6,7 @@ import pytest from appr.exception import (ChannelNotFound, Forbidden, InvalidRelease, PackageAlreadyExists, - PackageNotFound, PackageReleaseNotFound) + PackageNotFound, PackageReleaseNotFound, InvalidUsage) def convert_utf8(data): @@ -62,16 +62,31 @@ def test_db_restore(self, newdb, dbdata1): assert sorted(newdb.Channel.dump_all(newdb.Blob)) == sorted(dbdata1['channels']) @pytest.mark.integration - def test_get_default_package(self, db_with_data1): + def test_get_default_package_media_type(self, db_with_data1): p = db_with_data1.Package.get("titi/rocketchat", 'default', 'kpm') assert p.package == "titi/rocketchat" + @pytest.mark.integration + def test_get_default_package(self, db_with_data1): + p = db_with_data1.Package.get("titi/rocketchat", 'default', '-') + assert p.package == "titi/rocketchat" + assert p.media_type == "kpm" + @pytest.mark.integration def test_get_package_release_query(self, db_with_data1): p = db_with_data1.Package.get("titi/rocketchat", ">1.2", 'kpm') assert p.package == "titi/rocketchat" assert p.release == "2.0.1" assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80" + assert p.media_type == "kpm" + + @pytest.mark.integration + def test_get_package_detect_format(self, db_with_data1): + p = db_with_data1.Package.get("titi/rocketchat", ">1.2", '-') + assert p.package == "titi/rocketchat" + assert p.release == "2.0.1" + assert p.media_type == "kpm" + assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80" @pytest.mark.integration def test_get_blob(self, db_with_data1): @@ -154,6 +169,15 @@ def test_list_package_releases(self, db_with_data1): p = db_with_data1.Package.get("titi/rocketchat", "default", "kpm") assert sorted(p.releases()) == sorted(['0.0.1', '1.0.1', '2.0.1']) + @pytest.mark.integration + def test_list_package_media_types(self, db_with_data1): + assert sorted(db_with_data1.Package.manifests("titi/rocketchat", "0.0.1")) == ['helm', 'kpm'] + + @pytest.mark.integration + def test_get_package_multi_media_type(self, db_with_data1): + with pytest.raises(InvalidUsage): + db_with_data1.Package.get("titi/rocketchat", "0.0.1", "-") + @pytest.mark.integration def test_list_package_channels(self, db_with_data1): p = db_with_data1.Package.get("titi/rocketchat", '2.0.1', "kpm") diff --git a/appr/utils.py b/appr/utils.py index 7b7670b..c2546c8 100644 --- a/appr/utils.py +++ b/appr/utils.py @@ -1,12 +1,11 @@ from __future__ import absolute_import, division, print_function - +import sys import collections import errno import importlib import os import os.path import re -import sys import itertools from termcolor import colored @@ -137,9 +136,10 @@ def convert_utf8(data): return type(data)(map(convert_utf8, data)) else: return data - except UnicodeEncodeError as exc: + except UnicodeEncodeError: return data + # from celery/kombu https://github.com/celery/celery (BSD license) def symbol_by_name(name, aliases={}, imp=None, package=None, sep='.', default=None, **kwargs): """Get symbol by qualified name. @@ -205,6 +205,14 @@ def _reraise(tp, value, tb=None): return default +def flatten(array): + return list(itertools.chain(*array)) + + +def isbundled(): + return getattr(sys, 'frozen', False) + + def get_current_script_path(): executable = sys.executable if os.path.basename(executable) == "appr": @@ -214,5 +222,11 @@ def get_current_script_path(): return os.path.realpath(path) -def flatten(array): - return list(itertools.chain(*array)) +def abspath(relative_path): + """ Get absolute path """ + if isbundled(): + base_path = sys.executable + else: + base_path = os.path.abspath(".") + + return os.path.realpath(os.path.join(base_path, relative_path)) diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 new file mode 100644 index 0000000..160ba55 --- /dev/null +++ b/appveyor/install.ps1 @@ -0,0 +1,229 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + +$PYTHON_PRERELEASE_REGEX = @" +(?x) +(?\d+) +\. +(?\d+) +\. +(?\d+) +(?[a-z]{1,2}\d+) +"@ + + +function Download ($filename, $url) { + $webclient = New-Object System.Net.WebClient + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for ($i = 0; $i -lt $retry_attempts; $i++) { + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function ParsePythonVersion ($python_version) { + if ($python_version -match $PYTHON_PRERELEASE_REGEX) { + return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro, + $matches.prerelease) + } + $version_obj = [version]$python_version + return ($version_obj.major, $version_obj.minor, $version_obj.build, "") +} + + +function DownloadPython ($python_version, $platform_suffix) { + $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version + + if (($major -le 2 -and $micro -eq 0) ` + -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) ` + ) { + $dir = "$major.$minor" + $python_version = "$major.$minor$prerelease" + } else { + $dir = "$major.$minor.$micro" + } + + if ($prerelease) { + if (($major -le 2) ` + -or ($major -eq 3 -and $minor -eq 1) ` + -or ($major -eq 3 -and $minor -eq 2) ` + -or ($major -eq 3 -and $minor -eq 3) ` + ) { + $dir = "$dir/prev" + } + } + + if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) { + $ext = "msi" + if ($platform_suffix) { + $platform_suffix = ".$platform_suffix" + } + } else { + $ext = "exe" + if ($platform_suffix) { + $platform_suffix = "-$platform_suffix" + } + } + + $filename = "python-$python_version$platform_suffix.$ext" + $url = "$BASE_URL$dir/$filename" + $filepath = Download $filename $url + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = "amd64" + } + $installer_path = DownloadPython $python_version $platform_suffix + $installer_ext = [System.IO.Path]::GetExtension($installer_path) + Write-Host "Installing $installer_path to $python_home" + $install_log = $python_home + ".log" + if ($installer_ext -eq '.msi') { + InstallPythonMSI $installer_path $python_home $install_log + } else { + InstallPythonEXE $installer_path $python_home $install_log + } + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallPythonEXE ($exepath, $python_home, $install_log) { + $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home" + RunCommand $exepath $install_args +} + + +function InstallPythonMSI ($msipath, $python_home, $install_log) { + $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" + $uninstall_args = "/qn /x $msipath" + RunCommand "msiexec.exe" $install_args + if (-not(Test-Path $python_home)) { + Write-Host "Python seems to be installed else-where, reinstalling." + RunCommand "msiexec.exe" $uninstall_args + RunCommand "msiexec.exe" $install_args + } +} + +function RunCommand ($command, $command_args) { + Write-Host $command $command_args + Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $python_path = $python_home + "\python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + & $python_path $GET_PIP_PATH + } else { + Write-Host "pip already installed." + } +} + + +function DownloadMiniconda ($python_version, $platform_suffix) { + if ($python_version -eq "3.4") { + $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" + } else { + $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" + } + $url = $MINICONDA_URL + $filename + $filepath = Download $filename $url + return $filepath +} + + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $install_log = $python_home + ".log" + $args = "/S /D=$python_home" + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallMinicondaPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $conda_path = $python_home + "\Scripts\conda.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $args = "install --yes pip" + Write-Host $conda_path $args + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON +} + +main diff --git a/appveyor/run_with_env.cmd b/appveyor/run_with_env.cmd new file mode 100644 index 0000000..5da547c --- /dev/null +++ b/appveyor/run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/bin/apprc b/bin/apprc new file mode 100755 index 0000000..61a5bd7 --- /dev/null +++ b/bin/apprc @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir -p ~/.appr +docker run -ti --rm -v $PWD:/appr/local -v ~/.kube:/appr/.kube -v ~/.minikube:$HOME/.minikube -v ~/.appr:/appr/.appr quay.io/appr/appr:kubectl $@ diff --git a/requirements_dev.txt b/requirements_dev.txt index 8cf1c45..b27ab31 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,16 +1,22 @@ -bumpversion>=0.5.3 -coverage>=4.0 +# Missing no:color in upstream +# -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar + +# Missing: --add-data in pypi +# -e git+https://github.com/pyinstaller/pyinstaller.git#egg=pyinstaller + +bumpversion +coverage flake8 -pytest>=2.9.1 -pytest-cov>=2.2.1 -pytest-flask>=0.10.0 +gunicorn +pep8 + +pylint +pytest +pytest-cov +pytest-flask pytest-ordering -python-etcd>=0.4.3 +python-etcd requests-mock -tox>=2.1.1 sphinxcontrib-napoleon -gunicorn>=0.19 +tox yapf -pep8 -pylint -pyinstaller diff --git a/requirements_tests.txt b/requirements_tests.txt deleted file mode 100644 index 434fee8..0000000 --- a/requirements_tests.txt +++ /dev/null @@ -1,10 +0,0 @@ -pytest -coverage -pytest-cov -pytest-ordering -requests-mock -python-coveralls -flake8 -pylint -pep8 -tox diff --git a/setup.cfg b/setup.cfg index e745f98..6560591 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.2 +current_version = 0.7.0 commit = True tag = True @@ -19,4 +19,3 @@ max-line-length = 120 [pep8] exclude = docs max-line-length = 120 - diff --git a/setup.py b/setup.py index ffed1b2..1471dc9 100755 --- a/setup.py +++ b/setup.py @@ -29,14 +29,14 @@ ] cli_requirements = [ + 'urllib3<1.22', 'tabulate', 'termcolor', 'pyyaml', ] extra_requirements = [ - 'urllib3[secure]', - 'jsonnet', + 'urllib3[secure]<1.22', 'jinja2>=2.8', 'jsonpatch', 'tabulate', @@ -44,6 +44,8 @@ 'cryptography', ] +optional_requirements = ['jsonnet'] + test_requirements = [ "pytest", "coverage", @@ -63,7 +65,7 @@ setup( name='appr', - version='0.6.2', + version='0.7.0-pre', description="cloud-native app registry server", long_description=readme, author="Antoine Legrand", @@ -85,7 +87,7 @@ 'appr.models.kv.redis', 'appr.models.kv.filesystem', ], - scripts=['bin/appr'], + scripts=['bin/appr', 'bin/apprc'], package_dir={'appr': 'appr'}, include_package_data=True, install_requires=requirements, @@ -101,4 +103,5 @@ ], setup_requires=['pytest-runner'], test_suite='tests', + dependency_links=['https://github.com/requests/requests/tarball/master#egg=requests'], tests_require=test_requirements,) diff --git a/tests/commands/test_inspect.py b/tests/commands/test_inspect.py index 471cf21..16fecf4 100644 --- a/tests/commands/test_inspect.py +++ b/tests/commands/test_inspect.py @@ -31,12 +31,6 @@ def test_inspect_tree(cli_parser, package_blob, capsys): assert out == "\n".join(default_out) -def test_inspect_no_media_type(cli_parser, package_blob, capsys): - with pytest.raises(SystemExit) as exc: - inspectcmd = get_inspectcmd(cli_parser, ["kpm.sh/foo/bar@1.0.0", "--tree"]) - assert exc.value.code == 2 - - def test_inspect_default(cli_parser, package_blob, capsys): """ Default is the tree view """ inspectcmd = get_inspectcmd(cli_parser, ["kpm.sh/foo/bar@1.0.0", "-t", "helm", "--tree"]) diff --git a/tests/platforms/test_kubernetes.py b/tests/platforms/test_kubernetes.py index 154de1c..644bf7e 100644 --- a/tests/platforms/test_kubernetes.py +++ b/tests/platforms/test_kubernetes.py @@ -18,7 +18,7 @@ def test_endpoints(): def test_endpoints_missing(): - assert get_endpoint("bad") is None + assert get_endpoint("bad") is 'unknown' def test_protected_is_true_when_annoted(ns_resource): @@ -68,13 +68,13 @@ def test_kind(svc_resource, rc_resource, ns_resource): def test_get_hash(svc_resource): ks = Kubernetes(body=svc_resource['body']) kbody = json.loads(svc_resource['body']) - assert ks.apprhash == ks._get_apprhash(kbody) - assert ks.apprhash == kbody['metadata']['annotations']['resource.appr/hash'] + rhash = ks._gethash(kbody) + assert rhash == kbody['metadata']['annotations']['resource.appr/hash'] def test_get_empty(ns_resource): k = Kubernetes(body=ns_resource['body']) - assert k.apprhash is None + assert k._gethash(k.obj) is None def test_check_cmd(svc_resource, subcall_cmd): @@ -201,7 +201,7 @@ def get(*args): monkeypatch.setattr("appr.platforms.kubernetes.Kubernetes.get", get) k = Kubernetes(body=svc_resource['body']) - assert k.create(force=True) == "replaced" + assert k.create(force=True) == "updated" def test_create_ok_nohash(ns_resource, subcall_cmd, monkeypatch):