diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..233a7bb4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +ARG RUBY_IMAGE +FROM ${RUBY_IMAGE:-ruby:latest} + +RUN gem update --system && gem install bundler \ + && ruby --version && gem --version && bundle --version \ + && echo "--- :package: Installing system deps" \ + # Postgres apt sources + && curl -sS https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + # Node apt sources + && curl -sS https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ + && echo "deb http://deb.nodesource.com/node_10.x stretch main" > /etc/apt/sources.list.d/nodesource.list \ + # Yarn apt sources + && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb http://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ + # Install all the things + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client mysql-client sqlite3 \ + git nodejs yarn lsof \ + ffmpeg mupdf mupdf-tools poppler-utils \ + # await (for waiting on dependent services) + && cd /tmp \ + && wget -qc https://github.com/betalo-sweden/await/releases/download/v0.4.0/await-linux-amd64 \ + && install await-linux-amd64 /usr/local/bin/await \ + # clean up + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* \ + && mkdir /rails + +WORKDIR /rails +ENV RAILS_ENV=test RACK_ENV=test +ENV JRUBY_OPTS="--dev -J-Xmx1024M" + +ADD .buildkite/await-all /usr/local/bin/ + +ADD actioncable/package.json actioncable/ +ADD actiontext/package.json actiontext/ +ADD actionview/package.json actionview/ +ADD activestorage/package.json activestorage/ +ADD package.json yarn.lock .yarnrc ./ + +RUN echo "--- :javascript: Installing JavaScript deps" \ + && yarn install \ + && yarn cache clean + +ADD */*.gemspec tmp/ +ADD railties/exe/ railties/exe/ +ADD Gemfile Gemfile.lock RAILS_VERSION rails.gemspec ./ + +RUN echo "--- :bundler: Installing Ruby deps" \ + && (cd tmp && for f in *.gemspec; do d="$(basename -s.gemspec "$f")"; mkdir -p "../$d" && mv "$f" "../$d/"; done) \ + && rm Gemfile.lock && bundle install -j 8 && cp Gemfile.lock tmp/Gemfile.lock.updated \ + && rm -rf /usr/local/bundle/gems/cache \ + && echo "--- :floppy_disk: Copying repository contents" + +ADD . ./ + +RUN mv -f tmp/Gemfile.lock.updated Gemfile.lock diff --git a/Dockerfile.beanstalkd b/Dockerfile.beanstalkd new file mode 100644 index 00000000..1b204d57 --- /dev/null +++ b/Dockerfile.beanstalkd @@ -0,0 +1,6 @@ +FROM alpine + +RUN apk add --no-cache beanstalkd + +EXPOSE 11300 +ENTRYPOINT ["/usr/bin/beanstalkd"] diff --git a/await-all b/await-all new file mode 100755 index 00000000..bcfe9f1e --- /dev/null +++ b/await-all @@ -0,0 +1,7 @@ +#!/bin/bash + +exec await $( + getent hosts $( + for v in ${!AWAIT_*}; do echo ${v#AWAIT_}; done + ) | while read ip name; do v="AWAIT_${name}"; echo ${!v}; done +) -- "$@" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..df3a57da --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,115 @@ +version: "3.6" + +services: + base: + build: + context: ../ + dockerfile: .buildkite/Dockerfile + args: + - RUBY_IMAGE + + default: &default + image: "${IMAGE_NAME-buildkite_base}" + environment: + CI: + BUILDKITE_PARALLEL_JOB: + BUILDKITE_PARALLEL_JOB_COUNT: + + # Sauce Labs username and access key. Obfuscated, purposefully not encrypted. + ENCODED: "U0FVQ0VfQUNDRVNTX0tFWT1hMDM1MzQzZi1lOTIyLTQwYjMtYWEzYy0wNmIzZWE2MzVjNDggU0FVQ0VfVVNFUk5BTUU9cnVieW9ucmFpbHM=" + + BEANSTALK_URL: "beanstalk://beanstalkd" + MEMCACHE_SERVERS: "memcached:11211" + MYSQL_HOST: "${MYSQL_SERVICE-mysql}" + PGHOST: postgres + PGUSER: postgres + QC_DATABASE_URL: "postgres://postgres@postgres/active_jobs_qc_int_test" + QUE_DATABASE_URL: "postgres://postgres@postgres/active_jobs_que_int_test" + RABBITMQ_URL: "amqp://guest:guest@rabbitmq:5672" + REDIS_URL: "redis://redis:6379/1" + SELENIUM_DRIVER_URL: "http://chrome:4444/wd/hub" + + AWAIT_redis: tcp://redis:6379 + AWAIT_memcached: tcp://memcached:11211 + AWAIT_mysql: tcp://mysql:3306 + AWAIT_mariadb: tcp://mariadb:3306 + AWAIT_postgres: postgres://postgres@postgres:5432/postgres + AWAIT_rabbitmq: tcp://rabbitmq:5672 + AWAIT_beanstalkd: tcp://beanstalkd:11300 + AWAIT_chrome: tcp://chrome:4444 + + entrypoint: await-all + + depends_on: + - redis + - memcached + + postgresdb: + <<: *default + depends_on: + - redis + - memcached + - postgres + + mysqldb: + <<: *default + depends_on: + - redis + - memcached + - "${MYSQL_SERVICE-mysql}" + + railties: + <<: *default + depends_on: + - redis + - memcached + - "${MYSQL_SERVICE-mysql}" + - postgres + + activejob: + <<: *default + depends_on: + - redis + - memcached + - postgres + - rabbitmq + - beanstalkd + + actionview: + <<: *default + depends_on: + - redis + - memcached + - chrome + + memcached: + image: memcached:alpine + + redis: + image: redis:alpine + + mysql: &mysql + image: mysql:latest + command: "--default-authentication-plugin=mysql_native_password" + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + volumes: + - "./mysql-initdb.d:/docker-entrypoint-initdb.d" + + mariadb: + <<: *mysql + image: mariadb:latest + + postgres: + image: postgres:alpine + + rabbitmq: + image: rabbitmq:alpine + + beanstalkd: + build: + context: ./ + dockerfile: Dockerfile.beanstalkd + + chrome: + image: selenium/standalone-chrome:latest diff --git a/initial.yml b/initial.yml new file mode 100644 index 00000000..56c6274e --- /dev/null +++ b/initial.yml @@ -0,0 +1,48 @@ +# This file is never read -- it's just a copy of the pipeline's +# configuration in the Buildkite UI. + +steps: + - name: ":pipeline:" + command: | + PATH=/bin:/usr/bin + treesha="$$(git ls-tree -d HEAD .buildkite | awk '{print $$3}')" + if [ -z "$${treesha}" ]; then + echo "+++ No .buildkite/; using fallback repository" + rm -rf .buildkite + git clone "https://github.com/rails/buildkite-config" .buildkite + sh -c "$$PIPELINE_COMMAND" + elif [ "$${#treesha}" -lt 40 ]; then + echo "Short SHA for .buildkite/" + exit 1 + elif curl -s -S "https://gist.githubusercontent.com/matthewd/3f98bcc9957c8ddf2204a390bf3a6cdd/raw/list" | grep -a -F -x "$${treesha}"; then + echo "+++ Known tree; generating pipeline" + echo ".buildkite/ tree is $${treesha}" + sh -c "$$PIPELINE_COMMAND" + else + echo "+++ Unknown tree; requesting approval" + echo ".buildkite/ tree is $${treesha}" + buildkite-agent pipeline upload <<'NESTED' + steps: + - block: "Review Build Script" + prompt: | + This commit uses new build configuration. Please review the changes in .buildkite/ carefully before unblocking. + - name: ":pipeline:" + command: >- + $$PIPELINE_COMMAND + timeout_in_minutes: 5 + NESTED + fi + env: + PIPELINE_COMMAND: >- + docker run --rm + -v "$$PWD":/app:ro -w /app + -e CI + -e BUILDKITE_BRANCH + -e BUILDKITE_BUILD_ID + -e BUILDKITE_PULL_REQUEST + -e BUILDKITE_PULL_REQUEST_BASE_BRANCH + -e BUILDKITE_REBUILT_FROM_BUILD_ID + ruby:latest + .buildkite/pipeline-generate | + buildkite-agent pipeline upload + timeout_in_minutes: 5 diff --git a/mysql-initdb.d/create.sql b/mysql-initdb.d/create.sql new file mode 100644 index 00000000..4f03e894 --- /dev/null +++ b/mysql-initdb.d/create.sql @@ -0,0 +1,6 @@ +create user rails@'%'; +grant all privileges on activerecord_unittest.* to rails@'%'; +grant all privileges on activerecord_unittest2.* to rails@'%'; +grant all privileges on inexistent_activerecord_unittest.* to rails@'%'; +create database activerecord_unittest default character set utf8mb4; +create database activerecord_unittest2 default character set utf8mb4; diff --git a/pipeline-generate b/pipeline-generate new file mode 100755 index 00000000..b5bd909e --- /dev/null +++ b/pipeline-generate @@ -0,0 +1,225 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "yaml" +require "net/http" + +IMAGE_BASE = "973266071021.dkr.ecr.us-east-1.amazonaws.com/builds" + +BRANCH = ([ENV["BUILDKITE_PULL_REQUEST_BASE_BRANCH"], ENV["BUILDKITE_BRANCH"], "master"] - [""]).first +PULL_REQUEST = ([ENV["BUILDKITE_PULL_REQUEST"]] - ["false"]).first + +BUILD_ID = ENV["BUILDKITE_BUILD_ID"] + +DOCKER_COMPOSE_PLUGIN = "matthewd/docker-compose#master" +ARTIFACTS_PLUGIN = "artifacts#v1.2.0" + +File.read(File.expand_path("../rails.gemspec", __dir__)) =~ /required_ruby_version[^0-9]+([0-9]+\.[0-9]+)/ +MIN_RUBY = Gem::Version.new($1 || "2.0") + +def available_tags_for_image(image) + uri = URI("https://registry.hub.docker.com/v1/repositories/#{image}/tags") + json = Net::HTTP.get(uri) + JSON.parse(json).map { |x| x["name"] } +end + +RUBIES = + available_tags_for_image("ruby"). + grep(/\A[0-9]+\.[0-9]+\z/). + map { |s| Gem::Version.new(s) }. + select { |v| v >= MIN_RUBY }. + sort. + map { |v| "ruby:#{v}" } + +ONE_RUBY = RUBIES.last + +STEPS = [] + +def image_name_for(ruby, suffix = BUILD_ID, short: false) + tag = "#{mangle_name(ruby)}-#{suffix}" + + if short + tag + else + "#{IMAGE_BASE}:#{tag}" + end +end + +def mangle_name(name) + name.tr("^A-Za-z0-9", "-") +end + +def step_for(subdirectory, rake_task, ruby: nil, service: "default") + return unless Dir.exist?(File.expand_path("../#{subdirectory}", __dir__)) + + label = +"#{subdirectory} #{rake_task.sub(/[:_]test|test:/, "")}" + label.sub!(/ test/, "") + if ruby + label << " (#{ruby.sub(/^ruby:|:latest$/, "")})" + end + + if rake_task.start_with?("mysql2:") + rake_task = "db:mysql:rebuild #{rake_task}" + elsif rake_task.start_with?("postgresql:") + rake_task = "db:postgresql:rebuild #{rake_task}" + end + + command = "echo; echo \"+++ #{subdirectory}: rake #{rake_task}\"; cd #{subdirectory} && bundle exec rake #{rake_task}" + + timeout = 30 + + group = + if rake_task.include?("isolated") + "isolated" + else + ruby || ONE_RUBY + end + + hash = { + "label" => label, + "command" => command, + "group" => group, + "plugins" => [ + { + ARTIFACTS_PLUGIN => { + "download" => [".buildkite/*", ".buildkite/**/*"], + }, + }, + { + DOCKER_COMPOSE_PLUGIN => { + "run" => service, + "pull" => service, + "config" => ".buildkite/docker-compose.yml", + }, + }, + ], + "env" => { + "IMAGE_NAME" => image_name_for(ruby || ONE_RUBY), + }, + "timeout_in_minutes" => timeout, + } + + yield hash if block_given? + + STEPS << hash +end + +def steps_for(subdirectory, rake_task, service: "default", &block) + RUBIES.each do |ruby| + step_for(subdirectory, rake_task, ruby: ruby, service: service, &block) + end +end + +# GROUP 1: Runs additional isolated tests for non-PR builds +%w( + actionpack test default + actionmailer test default + activemodel test default + activesupport test default + actionview test default + activejob test default + activerecord mysql2:test mysqldb + activerecord postgresql:test postgresdb + activerecord sqlite3:test default +).each_slice(3) do |dir, task, service| + steps_for(dir, task, service: service) + + next if PULL_REQUEST + next if BRANCH != "master" && BRANCH !~ /\A[0-9-]+(?:-stable)?\z/ + + if task.match?(/:test/) + step_for(dir, task.sub(":test", ":isolated_test"), service: service) + else + step_for(dir, "#{task}:isolated", service: service) + end +end + +# GROUP 2: No isolated tests, runs for each supported ruby +%w( + actioncable test postgresdb + activestorage test default + actionmailbox test default + actiontext test default + guides test default +).each_slice(3) do |dir, task, service| + steps_for(dir, task, service: service) +end + +# GROUP 3: Special cases + +step_for("activerecord", "sqlite3_mem:test", service: "default") +step_for("activerecord", "mysql2:test", service: "mysqldb") do |x| + x["label"] += " [mariadb]" + x["env"]["MYSQL_SERVICE"] = "mariadb" +end +steps_for("actioncable", "test:integration", service: "default") do |x| + x["retry"] = { "automatic" => { "limit" => 3 } } +end +step_for("actionview", "test:ujs", service: "actionview") +steps_for("activejob", "test:integration", service: "activejob") +steps_for("railties", "test", service: "railties") do |x| + x["parallelism"] = 12 +end + +### + +STEPS.sort_by! do |step| + [ + -step["timeout_in_minutes"], + step["group"] == "isolated" ? 2 : 1, + step["command"].include?("test:") ? 2 : 1, + step["label"], + ] +end + +groups = STEPS.group_by { |s| s.delete("group") }.map do |group, steps| + { "group" => group, "steps" => steps } +end + +puts YAML.dump("steps" => [ + "wait", + { + "group" => "build", + "steps" => [ + *RUBIES.map do |ruby| + { + "label" => ":docker: #{ruby}", + "plugins" => [ + { + ARTIFACTS_PLUGIN => { + "upload" => ".buildkite/**/*", + }, + }, + { + DOCKER_COMPOSE_PLUGIN => { + "build" => "base", + "config" => ".buildkite/docker-compose.yml", + "image-name" => image_name_for(ruby, short: true), + "cache-from" => [ + ENV.fetch("BUILDKITE_REBUILT_FROM_BUILD_ID", "") == "" ? nil : + "base:" + image_name_for(ruby, ENV["BUILDKITE_REBUILT_FROM_BUILD_ID"]), + PULL_REQUEST && "base:" + image_name_for(ruby, "pr-#{PULL_REQUEST}"), + "base:" + image_name_for(ruby, "br-#{BRANCH}"), + BRANCH == "master" ? nil : "base:" + image_name_for(ruby, "br-master"), + ].compact, + "push" => [ + PULL_REQUEST ? + "base:" + image_name_for(ruby, "pr-#{PULL_REQUEST}") : + "base:" + image_name_for(ruby, "br-#{BRANCH}"), + ], + "image-repository" => IMAGE_BASE, + }, + }, + ], + "env" => { + "RUBY_IMAGE" => ruby, + }, + "timeout_in_minutes" => 15, + } + end, + ], + }, + "wait", + *groups, +])