master.cfg

import os

from buildbatter.build import BuildManager, GitBranch, SVNBranch, \
                              BuildTarget, PythonModuleBuildRules, BuildRules
from buildbatter.multirepo import SVN
from buildbatter.steps import EasyInstall, UploadDist, RotateFiles, NoseTests, \
                              BuildEgg, BuildSDist
from buildbatter.util import create_slave_list
from buildbot.process import factory
from buildbot.process.properties import WithProperties
from buildbot.scheduler import Nightly
from buildbot.status import words
from buildbot.steps.shell import ShellCommand, Test
from buildbot.steps.transfer import FileUpload, FileDownload
from buildbot.steps.python import PyFlakes


PUBLIC_HTML_DIR = os.path.join(os.path.dirname(__file__), "public_html")

MIRROR_PATH = "/var/www/downloads.reviewboard.org/htdocs/mirror"
BLEEDING_EDGE_PATH = "/var/www/downloads.reviewboard.org/htdocs/bleeding"
NIGHTLY_PATH = "/var/www/downloads.reviewboard.org/htdocs/nightlies"

MIRROR_URL = "http://downloads.reviewboard.org/mirror/"
BLEEDING_EDGE_URL = "http://downloads.reviewboard.org/bleeding/"

DJANGO_SVN_ROOT = "http://code.djangoproject.com/svn/django"

RB_WEBSITE_DIR = '/var/www/reviewboard.org/'

RB_EXCLUDES = [
    'reviewboard_1.6_django_trunk_py2.4',
    'reviewboard_django_trunk_py2.4',
    'reviewboard_django_1.3_py2.4',
    'reviewboard_django_1.3_py2.5',
    'reviewboard_django_1.3_py2.6',
    'djblets_django_1.3_py2.4',
    'djblets_django_trunk_py2.4',
    'django-evolution_django_trunk_py2.4',
    'rb_nightly_django_trunk_py2.4',
]

TRIGGER_EXCLUDES = [
    'triggered_%s' % exclude
    for exclude in RB_EXCLUDES
]


RELEASE_PY_VER = "2.7"


# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

c['administrators'] = (
    "chipx86@chipx86.com",
)

slaves, slave_info = create_slave_list()
c['slaves'] = slaves


# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9989


# Specify the location to find the eggs for easy_install
#EasyInstall.pypi_url = 'http://downloads.reviewboard.org:8887/'
#EasyInstall.allow_hosts_pattern = '*downloads.reviewboard.org*'


class RBProjectBuildRules(PythonModuleBuildRules):
    def __init__(self, dist_basename=None, upload_path=BLEEDING_EDGE_PATH,
                 upload_url=BLEEDING_EDGE_URL,
                 upload=True,
                 force_upload_path=False, *args, **kwargs):
        PythonModuleBuildRules.__init__(self, upload_path=upload_path,
                                        upload_url=upload_url,
                                        *args, **kwargs)
        self.dist_basename = dist_basename
        self.force_upload_path = force_upload_path
        self.upload = upload

    def addCheckoutSteps(self, f):
        PythonModuleBuildRules.addCheckoutSteps(self, f)
        self.addDevelopSteps(f)

    def addBuildSteps(self, f):
        f.addStep(ShellCommand,
                  command=["rm", "-rf", "build", "dist"],
                  description="removing build directory",
                  descriptionDone="removed build directory",
                  usePTY=False,
                  workdir=self.workdir)
        f.addStep(BuildSDist,
                  workdir=self.workdir,
                  use_egg_info=self.build_eggs,
                  env=self.env)

        if self.build_eggs:
            f.addStep(BuildEgg,
                      workdir=self.workdir,
                      env=self.env)

    def addEggSteps(self, f):
        if self.combination == ("django", "trunk"):
            # Use Django SVN, not the egg.
            f.addStep(SVN, reponame="django",
                      svnurl="%s/trunk" % DJANGO_SVN_ROOT,
                      mode="update",
                      alwaysUseLatest=True,
                      workdir="django")
            f.addStep(ShellCommand,
                      command=[self.python, "setup.py", "install"],
                      description="installing Django SVN",
                      descriptionDone="installed Django SVN",
                      workdir="django",
                      env=self.env,
                      haltOnFailure=True)
        else:
            self.egg_deps.append("Django")

        f.addStep(ShellCommand,
                  command=["rm", "-rf",
                           "lib/%s/site-packages/Djblets*" % self.python],
                  description="Cleaning up for build",
                  descriptionDone="Cleaned up for build")

        PythonModuleBuildRules.addEggSteps(self, f)

    def addDevelopSteps(self, f):
        # This will fetch the remaining eggs.
        f.addStep(ShellCommand,
                  command=[self.python, "setup.py", "develop"],
                  description="Installing in develop mode",
                  descriptionDone="Installed in develop mode",
                  env=self.env,
                  haltOnFailure=True,
                  workdir=self.workdir)

    def addUploadSteps(self, f):
        if (self.dist_basename and self.upload and
            self.combination == ("django", DJANGO_RELEASE_UPLOAD_VER) and
            not self.sandbox):

            rotate_patterns = []

            if self.pyver == "2.6":
                f.addStep(UploadDist,
                          default_upload_path=self.upload_path,
                          slavesrc=WithProperties("dist/%(sdist_filename)s"),
                          dest_filename=WithProperties("%(sdist_filename)s"),
                          mode=0644,
                          workdir=self.workdir)
                rotate_patterns.append("%s*-py%s.tar.gz" %
                                       (self.dist_basename, self.pyver))

            if self.build_eggs:
                f.addStep(UploadDist,
                          default_upload_path=self.upload_path,
                          slavesrc=WithProperties("dist/%(egg_filename)s"),
                          dest_filename=WithProperties("%(egg_filename)s"),
                          mode=0644,
                          workdir=self.workdir)
                rotate_patterns.append("%s*-py%s.egg" %
                                       (self.dist_basename, self.pyver))

            f.addStep(RotateFiles,
                      default_directory=self.upload_path,
                      patterns=rotate_patterns)


class DjangoEvolutionBuildRules(RBProjectBuildRules):
    def __init__(self, *args, **kwargs):
        RBProjectBuildRules.__init__(
            self,
            dist_basename='django_evolution',
            egg_deps=[
                'setuptools',
                'Nose',
                'coverage',
                'pysqlite',
            ],
            find_links=[MIRROR_URL],
            *args, **kwargs)

    def addTestSteps(self, f):
        f.addStep(NoseTests,
                  command=[self.python, "tests/runtests.py"],
                  env=self.env,
                  workdir=self.workdir,
                  flunkOnWarnings=True,
                  haltOnFailure=True)
        f.addStep(PyFlakes,
                  command=[self.python, "./tests/run-pyflakes.py"],
                  env=self.env,
                  workdir=self.workdir)


class ReviewBoardBuildRules(RBProjectBuildRules):
    def __init__(self, *args, **kwargs):
        RBProjectBuildRules.__init__(
            self, dist_basename="ReviewBoard",
            egg_deps=[
                "PIL==1.1.6", # Force our version, for the PIL directory
                "coverage",
                "mercurial",
                "Nose",
                "P4PythonInstaller",
                "pysqlite",
                "django_evolution",
                WithProperties("%(djblets_workdir)s/dist/" +
                               "%(djblets_egg_filename)s"),
            ],
            find_links=[
                MIRROR_URL,
            ],
            *args, **kwargs)

    def addCheckoutSteps(self, f):
        f.addStep(ShellCommand,
                  command=["../../scripts/build-svn.sh"],
                  description="installing pysvn",
                  descriptionDone="installed pysvn",
                  env=self.env,
                  haltOnFailure=True)

        RBProjectBuildRules.addCheckoutSteps(self, f)

    def addDevelopSteps(self, f):
        f.addStep(ShellCommand,
                  command=["rm", "-rf", "settings_local.py"],
                  description="removing settings_local.py",
                  descriptionDone="removed settings_local.py",
                  usePTY=False,
                  workdir=self.workdir)

        # This will fetch the remaining eggs.
	RBProjectBuildRules.addDevelopSteps(self, f)

    def addTestSteps(self, f):
        f.addStep(FileDownload,
                  mastersrc="scripts/settings_local.py",
                  slavedest="settings_local.py",
                  workdir=self.workdir)
        f.addStep(ShellCommand,
                  name="syncdb",
                  description="syncing database",
                  descriptionDone="synced database",
                  command=[self.python, "reviewboard/manage.py",
                           "syncdb", "--noinput"],
                  env=self.env,
                  workdir=self.workdir,
                  haltOnFailure=True)
        f.addStep(ShellCommand,
                  name="evolve",
                  description="evolving database",
                  descriptionDone="evolved database",
                  command=[self.python, "reviewboard/manage.py", "evolve",
                           "--execute", "--noinput"],
                  env=self.env,
                  workdir=self.workdir,
                  haltOnFailure=True)
        f.addStep(NoseTests,
                  command=[self.python, "reviewboard/manage.py", "test"],
                  env=self.env,
                  workdir=self.workdir,
                  flunkOnWarnings=True,
                  haltOnFailure=True)
        f.addStep(PyFlakes,
                  command=[self.python, "contrib/internal/run-pyflakes.py"],
                  env=self.env,
                  workdir=self.workdir)


class DjbletsBuildRules(RBProjectBuildRules):
    def __init__(self, egg_deps=[], *args, **kwargs):
        RBProjectBuildRules.__init__(
            self,
            dist_basename="Djblets",
            egg_deps=[
                "setuptools",
                "pysqlite",
                "django_evolution",
                "Nose",
                "coverage",
                "PIL",
                'mock',
            ] + egg_deps,
            find_links=[MIRROR_URL],
            *args, **kwargs)

    def addTestSteps(self, f):
        f.addStep(NoseTests,
                  command=[self.python, "tests/runtests.py"],
                  env=self.env,
                  workdir=self.workdir,
                  flunkOnWarnings=True,
                  haltOnFailure=True)
        f.addStep(PyFlakes,
                  command=[self.python, "./tests/run-pyflakes.py"],
                  env=self.env,
                  workdir=self.workdir)


class RBToolsBuildRules(PythonModuleBuildRules):
    def __init__(self, *args, **kwargs):
        PythonModuleBuildRules.__init__(self,
                                        upload_path=BLEEDING_EDGE_PATH,
                                        upload_url=BLEEDING_EDGE_URL,
                                        egg_deps=["setuptools>=0.6c11",
                                                  "simplejson"],
                                        find_links=[MIRROR_URL],
                                        *args, **kwargs)
        self.dist_basename = "RBTools"

    def addUploadSteps(self, f):
        if not self.sandbox:
            rotate_patterns = []

            if self.pyver == "2.6":
                f.addStep(UploadDist,
                          default_upload_path=self.upload_path,
                          slavesrc=WithProperties("dist/%(sdist_filename)s"),
                          dest_filename="%(sdist_filename)s",
                          mode=0644,
                          workdir=self.workdir)
                rotate_patterns.append("%s*-py%s.tar.gz" %
                                       (self.dist_basename, self.pyver))

            f.addStep(UploadDist,
                      default_upload_path=self.upload_path,
                      slavesrc=WithProperties("dist/%(egg_filename)s"),
                      dest_filename="%(egg_filename)s",
                      mode=0644,
                      workdir=self.workdir)
            rotate_patterns.append("%s*-py%s.egg" %
                                   (self.dist_basename, self.pyver))

            f.addStep(RotateFiles,
                      default_directory=self.upload_path,
                      patterns=rotate_patterns)

    def addTestSteps(self, f):
        f.addStep(PyFlakes,
                  command=["pyflakes", "scripts"],
                  env=self.env,
                  workdir=self.workdir)


class NightlyBuildRules(BuildRules):
    def addCheckoutSteps(self, f):
        pass


c['change_source']  = []
c['schedulers']     = []
c['builders']       = []


#
# Review Board build configuration
#
DJANGO_RELEASE_VERS = ["1.3"]
DJANGO_RELEASE_UPLOAD_VER = 'trunk'

rb_build_manager = BuildManager(
    slave_info=slave_info,
    combinations=[
        ("django", "trunk"),
    ] + [
        ("django", version)
        for version in DJANGO_RELEASE_VERS
    ]
)
rb_build_manager.add([
    BuildTarget(
        name="rb_nightly",
        branches=[
            SVNBranch("trunk", ""),
        ],
        triggers=["django"],
        wait_for_triggers=True,
        nightly=True,
        nightly_hour=8,
        nightly_stagger_interval=30,
        trigger_properties = {
            'upload_path': NIGHTLY_PATH,
        },
        build_rules=NightlyBuildRules()
    ),
    BuildTarget(
        name="django",
        branches=[
            SVNBranch("trunk", "%s/trunk" % DJANGO_SVN_ROOT),
        ] + [
            SVNBranch(version, "%s/tags/releases/%s" %
                      (DJANGO_SVN_ROOT, version),
                      poll_frequency=0)
            for version in DJANGO_RELEASE_VERS
        ],
        triggers=["django-evolution"],
        build_rules=BuildRules()
    ),
    BuildTarget(
        name="django-evolution",
        branches=[
            SVNBranch("trunk",
                     "http://django-evolution.googlecode.com/svn/trunk",
                     poll_frequency=60*100),
        ],
        dependencies=["django"],
        triggers=["djblets", "djblets-0.6"],
        trigger_excludes=TRIGGER_EXCLUDES,
        build_rules=DjangoEvolutionBuildRules()
    ),
    BuildTarget(
        name="djblets",
        branches=[
            GitBranch("master", "master",
                      "git://github.com/djblets/djblets.git"),
        ],
        dependencies=["django"],
        allow_sandbox=True,
        triggers=["reviewboard"],
        trigger_excludes=TRIGGER_EXCLUDES,
        trigger_properties={
            'djblets_egg_filename': WithProperties("%(egg_filename)s"),
        },
        build_rules=DjbletsBuildRules(egg_deps=[
            'django-pipeline',
        ])
    ),
    BuildTarget(
        name="djblets-0.6",
        branches=[
            GitBranch("release-0.6.x", "0.6.x",
                      "git://github.com/djblets/djblets.git"),
        ],
        dependencies=["django"],
        allow_sandbox=True,
        triggers=["reviewboard-1.6"],
        trigger_excludes=TRIGGER_EXCLUDES + [
            'triggered_reviewboard-1.6_django_trunk_py2.4',
            'triggered_reviewboard-1.6_django_trunk_py2.5',
            'triggered_reviewboard-1.6_django_trunk_py2.6',
        ],
        trigger_properties={
            'djblets_egg_filename': WithProperties("%(egg_filename)s"),
            'djblets_workdir': WithProperties("%(djblets-0.6_workdir)s"),
        },
        build_rules=DjbletsBuildRules(egg_deps=[
            'django-pipeline==1.1.27'
        ])
    ),
    BuildTarget(
        name="reviewboard",
        branches=[
            GitBranch("master", "master",
                      "git://github.com/reviewboard/reviewboard.git"),
        ],
        exclude_from=[('django', '1.3')],
        dependencies=["django", "django-evolution", "djblets"],
        allow_sandbox=True,
        build_rules=ReviewBoardBuildRules()
    ),
    BuildTarget(
        name="reviewboard-1.6",
        branches=[
            GitBranch("release-1.6.x", "1.6.x",
                      "git://github.com/reviewboard/reviewboard.git"),
        ],
        dependencies=["django", "django-evolution", "djblets"],
        allow_sandbox=True,
        build_rules=ReviewBoardBuildRules(upload=False)
    ),
])

c['change_source'] += rb_build_manager.get_pollers()
c['schedulers']    += rb_build_manager.get_schedulers(exclude=RB_EXCLUDES)
c['builders']      += rb_build_manager.get_builders(exclude=RB_EXCLUDES)


#
# RBTools build configuration
#
rbtools_build_manager = BuildManager(
    slave_info=slave_info,
    combinations=[
        ("rbtools", "master"),
    ]
)
rbtools_build_manager.add([
    BuildTarget(
        name="rbtools_nightly",
        branches=[
            GitBranch("master", "master", ""),
        ],
        triggers=["rbtools"],
        wait_for_triggers=True,
        nightly=True,
        nightly_hour=8,
        trigger_properties = {
            'upload_path': NIGHTLY_PATH,
        },
        build_rules=NightlyBuildRules()
    ),
    BuildTarget(
        name="rbtools",
        branches=[
            GitBranch("master", "master",
                      "git://github.com/reviewboard/rbtools.git"),
        ],
        allow_sandbox=True,
        build_rules=RBToolsBuildRules()
    ),
])


c['change_source'] += rbtools_build_manager.get_pollers()
c['schedulers']    += rbtools_build_manager.get_schedulers()
c['builders']      += rbtools_build_manager.get_builders()


#
# Review Board docs build configuration
#
c['schedulers'].append(Nightly(name='nightly_docs',
                               builderNames=['rbdocs'],
                               hour=10,
                               branch="master",
                               minute=0))

f = factory.BuildFactory()
f.addStep(ShellCommand(
    command=['%s/scripts/update-docs.py' % RB_WEBSITE_DIR],
    description="Generating website docs",
    descriptionDone="Generated website docs"))
c['builders'].append({
    'name': 'rbdocs',
    'slavename': 'rbwebsite',
    'factory': f,
    'category': 'builds',
    'builddir': 'rbdocs',
})


####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

from buildbot.status import html, mail

c['status'] = [
    # Public
    html.WebStatus(http_port="tcp:12344:interface=127.0.0.1",
                   public_html=PUBLIC_HTML_DIR,
                   allowForce=False),

    # Private
    html.WebStatus(http_port="tcp:12345:interface=127.0.0.1",
                   public_html=PUBLIC_HTML_DIR,
                   allowForce=True),

    # E-mail notifications
    mail.MailNotifier(fromaddr="buildbot@reviewboard.org",
                      mode="problem",
                      extraRecipients=c['administrators'],
                      relayhost="localhost",
                      sendToInterestedUsers=False),

    # IRC
    words.IRC("irc.freenode.net", "RB-BuildBot",
              channels=["#reviewboard-students"],
              notify_events={
                  'exception': 1,
                  'successToFailure': 1,
                  'failureToSuccess': 1,
              }),
]

#from buildbot.status import words
#c['status'].append(words.IRC(host="irc.freenode.net", nick="<NICK>",
#                             password="<PASSWORD>",
#                             channels=["<CHANNEL>"]))

# from buildbot.status import client
# c['status'].append(client.PBListener(9988))


####### DEBUGGING OPTIONS

# if you set 'debugPassword', then you can connect to the buildmaster with
# the diagnostic tool in contrib/debugclient.py . From this tool, you can
# manually force builds and inject changes, which may be useful for testing
# your buildmaster without actually commiting changes to your repository (or
# before you have a functioning 'sources' set up). The debug tool uses the
# same port number as the slaves do: 'slavePortnum'.

#c['debugPassword'] = "debugpassword"

# if you set 'manhole', you can ssh into the buildmaster and get an
# interactive python shell, which may be useful for debugging buildbot
# internals. It is probably only useful for buildbot developers. You can also
# use an authorized_keys file, or plain telnet.
#from buildbot import manhole
#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
#                                       "admin", "password")


####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "Review Board"
c['projectURL'] = "http://www.reviewboard.org/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.Waterfall page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://build.reviewboard.org/"

# vim: set ft=python: