resultSpec optimisation for buildsets

console view starts to be slow if we don't optimize the buildset fetching at the db level

master/buildbot/data/buildsets.py

@@ -60,6 +60,18 @@ def getSs(ssid):
 
         defer.returnValue(buildset)
 
+    fieldMapping = {
+        'buildsetid': 'buildsets.id',
+        'external_idstring': 'buildsets.external_idstring',
+        'reason': 'buildsets.reason',
+        'submitted_at': 'buildsets.submitted_at',
+        'complete': 'buildsets.complete',
+        'complete_at': 'buildsets.complete_at',
+        'results': 'buildsets.results',
+        'parent_buildid': 'buildsets.parent_buildid',
+        'parent_relationship': 'buildsets.parent_relationship'
+    }
+
 
 class BuildsetEndpoint(Db2DataMixin, base.Endpoint):
 
@@ -85,15 +97,11 @@ class BuildsetsEndpoint(Db2DataMixin, base.Endpoint):
 
     def get(self, resultSpec, kwargs):
         complete = resultSpec.popBooleanFilter('complete')
-        d = self.master.db.buildsets.getBuildsets(complete=complete)
+        resultSpec.fieldMapping = self.fieldMapping
+        d = self.master.db.buildsets.getBuildsets(complete=complete, resultSpec=resultSpec)
 
         @d.addCallback
         def db2data(buildsets):
-            # buildset db2data is pretty heavy. for the sake of console_view prefilter the data
-            # should fix it properly in http://trac.buildbot.net/ticket/3537
-            if (resultSpec.order == ['-submitted_at'] and resultSpec.limit and resultSpec.offset is None):
-                buildsets.sort(key=lambda x: x['submitted_at'], reverse=True)
-                buildsets = buildsets[:resultSpec.limit]
             d = defer.DeferredList([self.db2data(bs) for bs in buildsets],
                                    fireOnOneErrback=True, consumeErrors=True)
 

master/buildbot/db/buildsets.py

@@ -164,7 +164,7 @@ def thd(conn):
             return self._thd_row2dict(conn, row)
         return self.db.pool.do(thd)
 
-    def getBuildsets(self, complete=None):
+    def getBuildsets(self, complete=None, resultSpec=None):
         def thd(conn):
             bs_tbl = self.db.model.buildsets
             q = bs_tbl.select()
@@ -174,6 +174,8 @@ def thd(conn):
                 else:
                     q = q.where((bs_tbl.c.complete == 0) |
                                 (bs_tbl.c.complete == NULL))
+            if resultSpec is not None:
+                return resultSpec.thd_execute(conn, q, lambda x: self._thd_row2dict(conn, x))
             res = conn.execute(q)
             return [self._thd_row2dict(conn, row) for row in res.fetchall()]
         return self.db.pool.do(thd)

master/buildbot/test/fake/fakedb.py

@@ -1341,7 +1341,7 @@ def getBuildset(self, bsid):
         row = self.buildsets[bsid]
         return defer.succeed(self._row2dict(row))
 
-    def getBuildsets(self, complete=None):
+    def getBuildsets(self, complete=None, resultSpec=None):
         rv = []
         for bs in itervalues(self.buildsets):
             if complete is not None:
@@ -1351,6 +1351,8 @@ def getBuildsets(self, complete=None):
                     rv.append(self._row2dict(bs))
             else:
                 rv.append(self._row2dict(bs))
+        if resultSpec is not None:
+            rv = self.applyResultSpec(rv, resultSpec)
         return defer.succeed(rv)
 
     @defer.inlineCallbacks

master/buildbot/test/unit/test_db_buildsets.py

@@ -70,7 +70,7 @@ def getBuildset(self, bsid):
 
     def test_signature_getBuildsets(self):
         @self.assertArgSpecMatches(self.db.buildsets.getBuildsets)
-        def getBuildsets(self, complete=None):
+        def getBuildsets(self, complete=None, resultSpec=None):
             pass
 
     def test_signature_getRecentBuildsets(self):

master/docs/developer/database.rst

@@ -108,7 +108,7 @@ buildrequests
         returns ``None`` if there is no such buildrequest.  Note that build
         requests are not cached, as the values in the database are not fixed.
 
-    .. py:method:: getBuildRequests(buildername=None, complete=None, claimed=None, bsid=None, branch=None, repository=None)
+    .. py:method:: getBuildRequests(buildername=None, complete=None, claimed=None, bsid=None, branch=None, repository=None, resultSpec=None)
 
         :param buildername: limit results to buildrequests for this builder
         :type buildername: string
@@ -119,6 +119,8 @@ buildrequests
         :param bsid: see below
         :param repository: the repository associated with the sourcestamps originating the requests
         :param branch: the branch associated with the sourcestamps originating the requests
+        :param resultSpec: resultSpec containing filters sorting and paging request from data/REST API.
+        If possible, the db layer can optimize the SQL query using this information.
         :returns: list of brdicts, via Deferred
 
         Get a list of build requests matching the given characteristics.
@@ -276,11 +278,13 @@ builds
 
         Returns the last successful build from the current build number with the same repository/repository/codebase
 
-    .. py:method:: getBuilds(builderid=None, buildrequestid=None, complete=None)
+    .. py:method:: getBuilds(builderid=None, buildrequestid=None, complete=None, resultSpec=None)
 
         :param integer builderid: builder to get builds for
         :param integer buildrequestid: buildrequest to get builds for
         :param boolean complete: if not None, filters results based on completeness
+        :param resultSpec: resultSpec containing filters sorting and paging request from data/REST API.
+        If possible, the db layer can optimize the SQL query using this information.
         :returns: list of build dictionaries as above, via Deferred
 
         Get a list of builds, in the format described above.
@@ -624,11 +628,14 @@ buildsets
         Note that buildsets are not cached, as the values in the database are
         not fixed.
 
-    .. py:method:: getBuildsets(complete=None)
+    .. py:method:: getBuildsets(complete=None, resultSpec=None)
 
         :param complete: if true, return only complete buildsets; if false,
             return only incomplete buildsets; if ``None`` or omitted, return all
             buildsets
+        :param resultSpec: resultSpec containing filters sorting and paging request from data/REST API.
+        If possible, the db layer can optimize the SQL query using this information.
+
         :returns: list of bsdicts, via Deferred
 
         Get a list of bsdicts matching the given criteria.