improve test coverage

binux · Nov 11, 2014 · 4b33143 · 4b33143
1 parent 77c2d5f
commit 4b33143
Show file tree

Hide file tree

Showing 6 changed files with 151 additions and 6 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,4 +1,17 @@
+[run]
+source =
+    database
+    scheduler
+    fetcher
+    processor
+    result
+    libs
+
 [report]
+omit =
+    libs/sample_handler.py
+    libs/pprint.py
+
 exclude_lines =
     pragma: no cover
     def __repr__
@@ -9,3 +22,4 @@ exclude_lines =
     if 0:
     if __name__ == .__main__.:
     except ImportError:
+    pass
diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,7 @@ install:
     - pip install --allow-all-external -r requirements.txt
     - pip install coveralls
 script:
-    coverage run --source=database,scheduler,fetcher,processor,result,libs --omit=libs/sample_handler.py,libs/pprint.py ./runtest.py
+    coverage run ./runtest.py
 after_success:
     - coverage combine
     - coveralls
diff --git a/database/base/resultdb.py b/database/base/resultdb.py
@@ -22,16 +22,16 @@ class ResultDB(object):
     """
 
     def save(self, project, taskid, url, result):
-        pass
+        raise NotImplementedError
 
     def select(self, project, fields=None, offset=0, limit=None):
-        pass
+        raise NotImplementedError
 
     def count(self, project):
-        pass
+        raise NotImplementedError
 
     def get(self, project, taskid, fields=None):
-        pass
+        raise NotImplementedError
 
     def drop(self, project):
-        pass
+        raise NotImplementedError
diff --git a/test/test_processor.py b/test/test_processor.py
@@ -259,4 +259,27 @@ def test_30_new_task(self):
         self.in_queue.put((task, fetch_result))
         time.sleep(1)
         self.assertFalse(self.status_queue.empty())
+        while not self.status_queue.empty():
+            self.status_queue.get()
         self.assertFalse(self.newtask_queue.empty())
+
+    def test_40_index_page(self):
+        task = None
+        while not self.newtask_queue.empty():
+            task = self.newtask_queue.get()
+        self.assertIsNotNone(task)
+
+        fetch_result = {
+                "orig_url": task['url'],
+                "content": "<html><body><a href='http://binux.me'>binux</a></body></html>",
+                "headers": {},
+                "status_code": 200,
+                "url": task['url'],
+                "time": 0,
+                }
+        self.in_queue.put((task, fetch_result))
+        time.sleep(1)
+        self.assertFalse(self.status_queue.empty())
+        self.assertFalse(self.newtask_queue.empty())
+        task = self.newtask_queue.get()
+        self.assertEqual(task['url'], 'http://binux.me/')
diff --git a/test/test_result_worker.py b/test/test_result_worker.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
+# Author: Binux<[email protected]>
+#         http://binux.me
+# Created on 2014-11-11 20:52:53
+
+import os
+import time
+import unittest2 as unittest
+import logging.config
+logging.config.fileConfig("logging.conf")
+
+import shutil
+from multiprocessing import Queue
+from database.sqlite import resultdb
+from result.result_worker import ResultWorker
+from libs.utils import run_in_subprocess, run_in_thread
+class TestProcessor(unittest.TestCase):
+    resultdb_path = './test/data/result.db'
+
+    @classmethod
+    def setUpClass(self):
+        shutil.rmtree('./test/data/', ignore_errors=True)
+        os.makedirs('./test/data/')
+
+        def get_resultdb():
+            return resultdb.ResultDB(self.resultdb_path)
+        self.resultdb = get_resultdb()
+        self.inqueue = Queue(10)
+
+        def run_result_worker():
+            self.result_worker = ResultWorker(get_resultdb(), self.inqueue)
+            self.result_worker.run()
+        self.process = run_in_thread(run_result_worker)
+        time.sleep(1)
+
+    @classmethod
+    def tearDownClass(self):
+        if self.process.is_alive():
+            self.result_worker.quit()
+            self.process.join(2)
+        assert not self.process.is_alive()
+        shutil.rmtree('./test/data/', ignore_errors=True)
+
+    def test_10_bad_result(self):
+        self.inqueue.put(({'project': 'test_project'}, {}))
+        self.resultdb._list_project()
+        self.assertEqual(len(self.resultdb.projects), 0)
+        self.assertEqual(self.resultdb.count('test_project'), 0)
+
+    def test_20_insert_result(self):
+        data = {
+                'a': 'b'
+                }
+        self.inqueue.put(({
+            'project': 'test_project', 
+            'taskid': 'id1',
+            'url': 'url1'
+            }, data))
+        time.sleep(0.1)
+        self.resultdb._list_project()
+        self.assertEqual(len(self.resultdb.projects), 1)
+        self.assertEqual(self.resultdb.count('test_project'), 1)
+
+        result = self.resultdb.get('test_project', 'id1')
+        self.assertEqual(result['result'], data)
+
+    def test_30_overwrite(self):
+        self.inqueue.put(({
+            'project': 'test_project', 
+            'taskid': 'id1',
+            'url': 'url1'
+            }, "abc"))
+        time.sleep(0.1)
+        result = self.resultdb.get('test_project', 'id1')
+        self.assertEqual(result['result'], "abc")
+
+    def test_40_insert_list(self):
+        self.inqueue.put(({
+            'project': 'test_project', 
+            'taskid': 'id2',
+            'url': 'url1'
+            }, ['a', 'b']))
+        time.sleep(0.1)
+        result = self.resultdb.get('test_project', 'id2')
+        self.assertEqual(result['result'], ['a', 'b'])
diff --git a/test/test_scheduler.py b/test/test_scheduler.py
@@ -128,6 +128,7 @@ def test_10_new_task_ignore(self):
             'url': 'url'
             })
         self.assertEqual(self.rpc.size(), 0)
+        self.assertEqual(len(self.rpc.get_active_tasks()), 0)
 
     def test_20_new_project(self):
         self.projectdb.insert('test_project', {
@@ -145,6 +146,7 @@ def test_30_update_project(self):
         with self.assertRaises(Queue.Empty):
             task = self.scheduler2fetcher.get(timeout=0.1)
         self.projectdb.update('test_project', status="DEBUG")
+        self.rpc.update_project()
 
         task = self.scheduler2fetcher.get(timeout=5)
         self.assertIsNotNone(task)
@@ -173,6 +175,7 @@ def test_35_new_task(self):
         self.assertEqual(self.rpc.counter('all', 'sum')['test_project']['pending'], 1)
 
         task = self.scheduler2fetcher.get(timeout=5)
+        self.assertEqual(len(self.rpc.get_active_tasks()), 1)
         self.assertIsNotNone(task)
         self.assertEqual(task['project'], 'test_project')
         self.assertIn('fetch', task)
@@ -258,6 +261,24 @@ def test_80_newtask_age_ignore(self):
         time.sleep(0.1)
         self.assertEqual(self.rpc.size(), 0)
 
+    def test_82_newtask_via_rpc(self):
+        self.rpc.newtask({
+            'taskid': 'taskid',
+            'project': 'test_project',
+            'url': 'url',
+            'fetch': {
+                'data': 'abc',
+                },
+            'process': {
+                'data': 'abc',
+                },
+            'schedule': {
+                'age': 30,
+                },
+            })
+        time.sleep(0.1)
+        self.assertEqual(self.rpc.size(), 0)
+
     def test_90_newtask_with_itag(self):
         time.sleep(0.1)
         self.newtask_queue.put({