diff --git a/Makefile b/Makefile index a93aa9f553..8850efc972 100644 --- a/Makefile +++ b/Makefile @@ -131,3 +131,32 @@ docker-test: .PHONY: all all: dep build + +.PHONY: test-wercker +test-wercker: checkfmt pull-images test-basic-wercker test-middleware-wercker test-extensions-wercker test-system-wercker + +.PHONY: test-middleware-wercker +test-middleware-wercker: test-basic-wercker + cd examples/middleware && go build + +.PHONY: test-extensions-wercker +test-extensions-wercker: test-basic-wercker + cd examples/extensions && go build + +.PHONY: test-basic-wercker +test-basic-wercker: checkfmt pull-images fn-test-utils fn-status-checker + ./test_wercker.sh + +.PHONY: test-system-wercker +test-system-wercker: test-basic-wercker + ./system_test.sh sqlite3 + ./system_test.sh mysql + ./system_test.sh postgres + +.PHONY: release-dind-wercker +release-dind-wercker: + (cd images/dind && ./release_wercker.sh) + +.PHONY: release-fnserver-wercker +release-fnserver-wercker: + ./release_wercker.sh diff --git a/api/agent/agent_test.go b/api/agent/agent_test.go index d6941afd7f..7eafc3234d 100644 --- a/api/agent/agent_test.go +++ b/api/agent/agent_test.go @@ -646,7 +646,6 @@ func TestTmpFsRW(t *testing.T) { // Let's check what mounts are on... mounts := strings.Split(resp.R.MountsRead, "\n") isFound := false - isRootFound := false for _, mnt := range mounts { tokens := strings.Split(mnt, " ") if len(tokens) < 3 { @@ -657,17 +656,20 @@ func TestTmpFsRW(t *testing.T) { opts := tokens[3] // tmp dir with RW and no other options (size, inodes, etc.) - if point == "/tmp" && opts == "rw,nosuid,nodev,noexec,relatime" { - // good + if point == "/tmp" { isFound = true - } else if point == "/" && strings.HasPrefix(opts, "ro,") { - // Read-only root, good... - isRootFound = true + optsPresent := strings.Split(opts, ",") + optsRequired := strings.Split("rw,nosuid,nodev,noexec,relatime", ",") + for _, opt := range optsRequired { + if !contains(optsPresent, opt) { + isFound = false + } + } } } - if !isFound || !isRootFound { - t.Fatal(`didn't get proper mounts for /tmp or /, got /proc/mounts content of:\n`, resp.R.MountsRead) + if !isFound { + t.Fatal(`didn't get proper mounts for /tmp, got /proc/mounts content of:\n`, resp.R.MountsRead) } // write file should not have failed... @@ -745,7 +747,6 @@ func TestTmpFsSize(t *testing.T) { // Let's check what mounts are on... mounts := strings.Split(resp.R.MountsRead, "\n") isFound := false - isRootFound := false for _, mnt := range mounts { tokens := strings.Split(mnt, " ") if len(tokens) < 3 { @@ -756,17 +757,20 @@ func TestTmpFsSize(t *testing.T) { opts := tokens[3] // rw tmp dir with size and inode limits applied. - if point == "/tmp" && opts == "rw,nosuid,nodev,noexec,relatime,size=1024k,nr_inodes=1024" { - // good + if point == "/tmp" { isFound = true - } else if point == "/" && strings.HasPrefix(opts, "ro,") { - // Read-only root, good... - isRootFound = true + optsPresent := strings.Split(opts, ",") + optsRequired := strings.Split("rw,nosuid,nodev,noexec,relatime,size=1024k,nr_inodes=1024", ",") + for _, opt := range optsRequired { + if !contains(optsPresent, opt) { + isFound = false + } + } } } - if !isFound || !isRootFound { - t.Fatal(`didn't get proper mounts for /tmp or /, got /proc/mounts content of:\n`, resp.R.MountsRead) + if !isFound { + t.Fatal(`didn't get proper mounts for /tmp, got /proc/mounts content of:\n`, resp.R.MountsRead) } // write file should have failed... @@ -1167,3 +1171,12 @@ func TestNBIOResourceTracker(t *testing.T) { t.Fatalf("Expected successes, but got %d", ok) } } + +func contains(arr []string, str string) bool { + for _, a := range arr { + if a == str { + return true + } + } + return false +} diff --git a/helpers_wercker.sh b/helpers_wercker.sh new file mode 100644 index 0000000000..c4fe8f5acd --- /dev/null +++ b/helpers_wercker.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -exo pipefail + +function get_host { + echo $1 +} + +function get_port { + local NAME=$1 + + local SERVICE_LIST=( + "fn_basic_tests_minio" + "fn_basic_tests_mysql" + "fn_basic_tests_postgres" + "fn_api_tests_minio" + "fn_api_tests_mysql" + "fn_api_tests_postgres" + "fn_system_tests_minio" + "fn_system_tests_mysql" + "fn_system_tests_postgres" + ) + + local SERVICE_PORT_LIST=( + 9000 + 3306 + 5432 + 9000 + 3306 + 5432 + 9000 + 3306 + 5432 + ) + + local IDX=0 + while [ ${IDX} -lt ${#SERVICE_LIST[@]} ] + do + if [ ${SERVICE_LIST[$IDX]} = "${NAME}" ]; then + echo ${SERVICE_PORT_LIST[$IDX]} + return + fi + IDX=$(($IDX+1)) + done + + echo "Invalid context/component: ${NAME} not in service list" + exit -1 +} + +function spawn_sqlite3 { + local CONTEXT=$1 + touch /tmp/${CONTEXT}_sqllite3.db + echo "sqlite3:///tmp/${CONTEXT}_sqllite3.db" +} + +function spawn_mysql { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_mysql) + local HOST=$(get_host ${CONTEXT}_mysql) + local ID=$(docker run --name ${CONTEXT}_mysql \ + -p ${PORT}:3306 \ + -e MYSQL_DATABASE=funcs \ + -e MYSQL_ROOT_PASSWORD=root \ + --network=$DOCKER_NETWORK_NAME \ + -d mysql:5.7.22) + + echo "mysql://root:root@tcp(${HOST}:${PORT})/funcs" +} + +function spawn_postgres { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_postgres) + local HOST=$(get_host ${CONTEXT}_postgres) + local ID=$(docker run --name ${CONTEXT}_postgres \ + -e "POSTGRES_DB=funcs" \ + -e "POSTGRES_PASSWORD=root" \ + -p ${PORT}:5432 \ + --network=$DOCKER_NETWORK_NAME \ + -d postgres:9.3-alpine) + + echo "postgres://postgres:root@${HOST}:${PORT}/funcs?sslmode=disable" +} + +function spawn_minio { + local CONTEXT=$1 + local PORT=$(get_port ${CONTEXT}_minio) + local HOST=$(get_host ${CONTEXT}_minio) + local ID=$(docker run --name ${CONTEXT}_minio \ + -p ${PORT}:9000 \ + -e "MINIO_ACCESS_KEY=admin" \ + -e "MINIO_SECRET_KEY=password" \ + --network=$DOCKER_NETWORK_NAME \ + -d minio/minio server /data) + + echo "s3://admin:password@${HOST}:${PORT}/us-east-1/fnlogs" +} + +function docker_pull_postgres { + docker pull postgres:9.3-alpine +} + +function docker_pull_mysql { + docker pull mysql:5.7.22 +} + +function docker_pull_minio { + docker pull minio/minio +} + +function remove_containers { + local CONTEXT=$1 + for i in mysql minio postgres + do + docker rm -fv ${CONTEXT}_${i} 2>/dev/null || true + done + + rm -f /tmp/${CONTEXT}_sqllite3.db +} diff --git a/images/dind/release_wercker.sh b/images/dind/release_wercker.sh new file mode 100755 index 0000000000..e47281cdf5 --- /dev/null +++ b/images/dind/release_wercker.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -exo pipefail + +# Ensure working dir is clean +## +#git status +#if [[ -z $(git status -s) ]] +#then +# echo "tree is clean" +#else +# echo "tree is dirty, please commit changes before running this" +# exit 1 +#fi + +# This script should be run after its sibliing, build.sh, and +# after any related tests have passed. + +# Match version with Docker version +docker_info=$(docker run --rm fnproject/dind:latest docker -v 2>/dev/null | grep "^Docker version") +version=$(echo $docker_info | cut -d ' ' -f 3 | tr -d ,) + +echo "Version: $version" + +M=$(echo $version | cut -d '.' -f 1) +Mm=$(echo $version | cut -d '.' -f 1,2) + +# Calculate new release version +DIND_NEW=$(echo "$DIND_PREV" | perl -pe 's/\d+\.\d+\.\K(\d+)/$1+1/e') + +# Add appropriate docker tags +docker tag fnproject/dind:latest $DOCKER_USER/dind:$version +docker tag fnproject/dind:latest $DOCKER_USER/dind:latest +# be nice to have bump image do all of this tagging and pushing too (mount docker sock and do it all) +docker tag fnproject/dind:$version $DOCKER_USER/dind:$Mm +docker tag fnproject/dind:$version $DOCKER_USER/dind:$M +docker tag fnproject/dind:$version $DOCKER_USER/dind:release-$DIND_NEW + +docker push $DOCKER_USER/dind:$version +docker push $DOCKER_USER/dind:$Mm +docker push $DOCKER_USER/dind:$M +docker push $DOCKER_USER/dind:release-$DIND_NEW +docker push $DOCKER_USER/dind:latest + +# Mark this release with a tag +# No code changes so only the tag requires a push +## +#git tag -f -a "$DIND_NEW" -m "DIND release $DIND_NEW of $version" +#git push origin "$DIND_NEW" +# diff --git a/images/fn-status-checker/release_wercker.sh b/images/fn-status-checker/release_wercker.sh new file mode 100755 index 0000000000..8515a41fd8 --- /dev/null +++ b/images/fn-status-checker/release_wercker.sh @@ -0,0 +1,3 @@ +set -e +docker tag fnproject/fn-status-checker:latest $DOCKER_USER/fn-status-checker:latest +docker push $DOCKER_USER/fn-status-checker:latest diff --git a/images/fn-test-utils/release_wercker.sh b/images/fn-test-utils/release_wercker.sh new file mode 100755 index 0000000000..9ca68aefa4 --- /dev/null +++ b/images/fn-test-utils/release_wercker.sh @@ -0,0 +1,3 @@ +set -e +docker tag fnproject/fn-test-utils:latest $DOCKER_USER/fn-test-utils:latest +docker push $DOCKER_USER/fn-test-utils:latest diff --git a/release_wercker.sh b/release_wercker.sh new file mode 100755 index 0000000000..d685547d4e --- /dev/null +++ b/release_wercker.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -exuo pipefail + +user="fnproject" +image="fnserver" +image_deprecated="functions" + +# ensure working dir is clean +## +#git status +#if [[ -z $(git status -s) ]] +#then +# echo "tree is clean" +#else +# echo "tree is dirty, please commit changes before running this" +# exit 1 +#fi + +version_file="api/version/version.go" +if [ -z $(grep -m1 -Eo "[0-9]+\.[0-9]+\.[0-9]+" $version_file) ]; then + echo "did not find semantic version in $version_file" + exit 1 +fi +perl -i -pe 's/\d+\.\d+\.\K(\d+)/$1+1/e' $version_file +version=$(grep -m1 -Eo "[0-9]+\.[0-9]+\.[0-9]+" $version_file) +echo "Version: $version" + +make docker-build + +## +#git add -u +#git commit -m "$image: $version release [skip ci]" +#git tag -f -a "$version" -m "version $version" +#git push +#git push origin $version + +# Push the version bump and tags laid down previously +#gtag=$image-$version +#git push +#git push origin $version +# +# Finally, push docker images +docker tag $user/$image:latest $DOCKER_USER/$image:$version +docker tag $user/$image:latest $DOCKER_USER/$image:latest +docker push $DOCKER_USER/$image:$version +docker push $DOCKER_USER/$image:latest + +# Deprecated images, should remove this sometime in near future +docker tag $user/$image:latest $DOCKER_USER/$image_deprecated:$version +docker tag $user/$image:latest $DOCKER_USER/$image_deprecated:latest +docker push $DOCKER_USER/$image_deprecated:$version +docker push $DOCKER_USER/$image_deprecated:latest + +# release test utils docker image +(cd images/fn-test-utils && ./release_wercker.sh) +(cd images/fn-status-checker && ./release_wercker.sh) + diff --git a/test_wercker.sh b/test_wercker.sh new file mode 100755 index 0000000000..435b55a99d --- /dev/null +++ b/test_wercker.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Top level test script to start all other tests +set -exuo pipefail + +export CONTEXT="fn_basic_tests" +source ./helpers_wercker.sh +remove_containers ${CONTEXT} + +export POSTGRES_URL=$(spawn_postgres ${CONTEXT}) +export MYSQL_URL=$(spawn_mysql ${CONTEXT}) +export MINIO_URL=$(spawn_minio ${CONTEXT}) +export FN_DS_DB_PING_MAX_RETRIES=60 + +go test -v $(go list ./... | \ + grep -v vendor | \ + grep -v examples | \ + grep -v test/fn-api-tests | \ + grep -v test/fn-system-tests | \ + grep -v images/fn-test-utils\ +) + +go vet $(go list ./... | grep -v vendor) + +remove_containers ${CONTEXT} + +docker volume create fn +docker create --mount source=fn,target=/fn --name ubuntu_ephemeral ubuntu +docker cp `pwd`/. ubuntu_ephemeral:/fn/. +docker rm ubuntu_ephemeral +docker run --mount source=fn,target=/go/src/github.com/fnproject/fn --network=$DOCKER_NETWORK_NAME --rm fnproject/swagger:0.0.1 /go/src/github.com/fnproject/fn/docs/swagger.yml +docker run --mount source=fn,target=/go/src/github.com/fnproject/fn --network=$DOCKER_NETWORK_NAME --rm fnproject/swagger:0.0.1 /go/src/github.com/fnproject/fn/docs/swagger_v2.yml \ No newline at end of file diff --git a/wercker.yml b/wercker.yml new file mode 100644 index 0000000000..0a49b4d5a4 --- /dev/null +++ b/wercker.yml @@ -0,0 +1,78 @@ +box: golang:1.10 +no-response-timeout: 15 + +build: + docker: true + base-path: /go/src/github.com/fnproject/fn + steps: + - script: + name: Set env variables + code: | + mkdir -p /home/circleci/go + export GOVERSION=1.10 + export OS=linux + export ARCH=amd64 + export FN_LOG_LEVEL=debug + + - script: + name: go version + code: go version + + - script: + name: install docker + code: | + rm -rf /var/cache/apt/archives && ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial + echo "Acquire::Check-Valid-Until \"0\";" > /etc/apt/apt.conf.d/10no--check-valid-until + curl -fsSL https://get.docker.com/ | sudo sh + docker version + + - script: + name: docker version + code: docker version + + - script: + name: nproc + code: nproc + + - script: + name: make clear-images + code: make clear-images + + - script: + name: changed since last + code: | + ( ./changed-since-last.sh + echo "export DIND_NEEDED DIND_TAG DIND_PREV" + echo "export FN_NEEDED FN_TAG FN_PREV" + ) > /tmp/exports.sh + . /tmp/exports.sh + + - script: + name: DIND + code: | + if [[ -n "$DIND_NEEDED" ]]; then + make build-dind + fi + + - script: + name: FNSERVER + code: | + if [[ -n "$FN_NEEDED" ]]; then + make test-wercker -j $(nproc) + fi + + - script: + name: deploy + code: | + if [[ "${WERCKER_GIT_BRANCH}" == "master" ]]; then + printenv DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin + git config --global user.email "ci@fnproject.com" + git config --global user.name "CI" + git branch --set-upstream-to=origin/${WERCKER_GIT_BRANCH} ${WERCKER_GIT_BRANCH} + if [[ -n "$DIND_NEEDED" ]]; then + make release-dind-wercker + fi + if [[ -n "$FN_NEEDED" ]]; then + make release-fnserver-wercker + fi + fi \ No newline at end of file