From fbb3c7df0647dc56beb5088fb96721634318671d Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Wed, 2 Aug 2017 13:05:10 +0300 Subject: [PATCH 1/8] Refactored tests --- .../bot/identification/tests/bridge.py | 6 - .../bot/identification/tests/edr_handler.py | 483 ++++++------- .../bot/identification/tests/filter_tender.py | 684 ++++++------------ .../bot/identification/tests/scanner.py | 353 +++------ .../bot/identification/tests/upload_file.py | 230 +++--- 5 files changed, 648 insertions(+), 1108 deletions(-) diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index 7ed2fb9..a9544bc 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -17,7 +17,6 @@ from openprocurement.bot.identification.client import DocServiceClient, ProxyClient from openprocurement.bot.identification.databridge.utils import check_412 - config = { 'main': { @@ -43,7 +42,6 @@ class AlmostAlwaysTrue(object): - def __init__(self, total_iterations=1): self.total_iterations = total_iterations self.current_iteration = 0 @@ -112,7 +110,6 @@ def proxy_response(): class TestBridgeWorker(BaseServersTest): - def test_init(self): self.worker = EdrDataBridge(config) self.assertEqual(self.worker.delay, 15) @@ -280,7 +277,6 @@ def test_check_services_mock(self): self.worker.check_services() self.assertTrue(self.worker.set_sleep.called) - @patch("gevent.sleep") def test_check_log(self, gevent_sleep): gevent_sleep = custom_sleep @@ -289,5 +285,3 @@ def test_check_log(self, gevent_sleep): self.worker.check_services = MagicMock(return_value=True) self.worker.run() self.assertTrue(self.worker.edrpou_codes_queue.qsize.called) - - diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index 41b093f..d56af7d 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -36,12 +36,16 @@ def setUp(self): self.source_date = ["2017-04-25T11:56:36+00:00"] self.gen_req_id = [generate_request_id() for _ in xrange(10)] self.edrpou_codes_queue = Queue(10) + self.check_queue = Queue(10) self.edr_ids_queue = Queue(10) self.upload_to_doc_service_queue = Queue(10) self.filtered_tender_ids_queue = Queue(10) + self.client = MagicMock() self.tender_id = uuid.uuid4().hex - self.document_id = generate_doc_id() + self.document_ids = [generate_doc_id() for _ in xrange(3)] self.award_id = uuid.uuid4().hex + self.bid_id = uuid.uuid4().hex + self.qualification_id = uuid.uuid4().hex self.edr_req_ids = [generate_request_id() for _ in xrange(10)] self.proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') self.uri = "{uri}".format(uri=self.proxy_client.verify_url) @@ -54,7 +58,7 @@ def setUp(self): self.sleep_change_value = APIRateController() def meta(self): - return {'meta': {'id': self.document_id, 'author': author, 'sourceRequests': [ + return {'meta': {'id': self.document_ids[0], 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}} def tearDown(self): @@ -119,10 +123,10 @@ def test_proxy_client(self, mrequest, gevent_sleep): expected_result = [] for i in range(2): self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_id, 1, 1, []))) # data + self.file_con({}, self.document_ids[0], 1, 1, []))) # data expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i]]))) + self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(self.edr_ids)) @@ -143,7 +147,8 @@ def test_proxy_client_429(self, mrequest, gevent_sleep): self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) # result + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[i], self.edr_req_ids[i]]))) # result for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) @@ -152,7 +157,6 @@ def test_proxy_client_429(self, mrequest, gevent_sleep): def test_proxy_client_402(self, mrequest, gevent_sleep): """First request returns Edr API returns to proxy 402 status code with messages.""" gevent_sleep.side_effect = custom_sleep - edr_details_req_id = self.gen_req_id[0:2] mrequest.get(self.uri, [self.stat_c(403, 0, [{'message': 'Payment required.', 'code': 5}], self.edr_req_ids[0]), # pay for me self.stat_200([{}], self.source_date, @@ -164,12 +168,12 @@ def test_proxy_client_402(self, mrequest, gevent_sleep): expected_result = [] for i in xrange(2): mrequest.get(self.url_id(self.local_edr_ids[i]), - [self.stat_200([{}], self.source_date, edr_details_req_id[i])]) + [self.stat_200([{}], self.source_date, self.edr_req_ids[i])]) self.edrpou_codes_queue.put( Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) + self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertIsNotNone(mrequest.request_history[3].headers['X-Client-Request-ID']) @@ -179,19 +183,17 @@ def test_proxy_client_402(self, mrequest, gevent_sleep): def test_retry_get_edr_id(self, mrequest, gevent_sleep): """First and second response returns 403 status code. Tests retry for get_edr_id worker""" gevent_sleep.side_effect = custom_sleep - edr_details_req_ids = self.gen_req_id[0] - local_edr_ids = get_random_edr_ids(1) - mrequest.get(self.uri, [self.stat_c(403, 0, '', edr_details_req_ids), - self.stat_c(403, 0, '', edr_details_req_ids), - self.stat_200([{}], self.source_date, edr_details_req_ids)]) - mrequest.get(self.url_id(local_edr_ids[0]), [self.stat_200([{}], self.source_date, edr_details_req_ids)]) + mrequest.get(self.uri, [self.stat_c(403, 0, '', self.edr_req_ids[0]), + self.stat_c(403, 0, '', self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) + mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) expected_result = [] for i in range(1): self.edrpou_codes_queue.put( Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_id, 1, 1, - [edr_details_req_ids, edr_details_req_ids]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[0]]))) self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result[0]) self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) @@ -201,22 +203,21 @@ def test_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept response with 404 status code and error message 'EDRPOU not found'. Check that tender_data is in self.upload_to_doc_service_queue.""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0] mrequest.get(self.url, json={'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date}}]}]}, - status_code=404, headers={'X-Request-ID': edr_req_id}) + status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, - "id": self.document_id, "version": version, + "id": self.document_ids[0], "version": version, 'author': author, "sourceRequests": ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id]}})) + self.edr_req_ids[0]]}})) # check that data not in self.edr_ids_queue self.assertEqual(mrequest.call_count, 1) # Requests must call proxy once self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -230,21 +231,20 @@ def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): [self.stat_200([{"test": 1}, {"test": 2}], [self.source_date], self.edr_req_ids[0]), self.stat_200([{"test": 1}, {"test": 2}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) expected_result = [] - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 1}, self.document_id, 2, 1, + self.file_con({"test": 1}, self.document_ids[0], 2, 1, [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 2}, self.document_id, 2, 2, + self.file_con({"test": 2}, self.document_ids[0], 2, 2, [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - self.assertEqual(check_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.check_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') @@ -255,21 +255,20 @@ def test_retry_get_edr_data_two_ids(self, mrequest, gevent_sleep): [self.stat_c(403, '10', '', self.edr_req_ids[0]), self.stat_200([{"test": 1}, {"test": 2}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) expected_result = [] - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 1}, self.document_id, 2, 1, + self.file_con({"test": 1}, self.document_ids[0], 2, 1, [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 2}, self.document_id, 2, 2, + self.file_con({"test": 2}, self.document_ids[0], 2, 2, [self.edr_req_ids[0], self.edr_req_ids[1]]))) - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - self.assertEqual(check_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.check_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') @@ -277,26 +276,26 @@ def test_retry_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept 5 times response with status code 403 and error, then accept response with status code 404 and message 'EDRPOU not found'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0] - mrequest.get(self.uri, [self.stat_c(403, 0, '', edr_req_id), self.stat_c(403, 0, '', edr_req_id), - self.stat_c(403, 0, '', edr_req_id), self.stat_c(403, 0, '', edr_req_id), - self.stat_c(403, 0, '', edr_req_id), - {'json': { - 'errors': [ - {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"detailsSourceDate": self.source_date}}]}]}, - 'status_code': 404, 'headers': {'X-Request-ID': edr_req_id}}]) + mrequest.get(self.uri, + [self.stat_c(403, 0, '', self.edr_req_ids[0]), self.stat_c(403, 0, '', self.edr_req_ids[0]), + self.stat_c(403, 0, '', self.edr_req_ids[0]), self.stat_c(403, 0, '', self.edr_req_ids[0]), + self.stat_c(403, 0, '', self.edr_req_ids[0]), + {'json': { + 'errors': [ + {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, + 'status_code': 404, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, - 'id': self.document_id, + 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id]}})) + self.edr_req_ids[0]]}})) # check that data not in self.edr_ids_queue self.assertEqual(mrequest.call_count, 6) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -307,23 +306,22 @@ def test_retry_get_edr_data_empty_response(self, mrequest, gevent_sleep): def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0] - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) expected_result = [] - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, 'author': author, + 'id': self.document_ids[0], "version": version, + 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=403)), - RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, + RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, json=MagicMock(return_value= {"errors": [{"description": @@ -332,33 +330,32 @@ def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): "code": u"notFound"}, "meta": { "detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, + 'id': self.document_ids[0], "version": version, 'author': author }}]}]})))]) for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0] - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) expected_result = [] - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, 'author': author, + 'id': self.document_ids[0], "version": version, + 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result self.worker.get_edr_data_request = MagicMock(side_effect=[ - RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, + RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, json=MagicMock(return_value= {"errors": [{"description": @@ -367,33 +364,32 @@ def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): "code": u"notFound"}, "meta": { "detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, + 'id': self.document_ids[0], "version": version, 'author': author}}]}]})))]) for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0] - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) expected_result = [] - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, 'author': author, + 'id': self.document_ids[0], "version": version, + 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result self.worker.get_edr_data_request = MagicMock(side_effect=[ Exception(), - RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, + RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, json=MagicMock(return_value= {"errors": [{"description": @@ -402,26 +398,26 @@ def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): "code": u"notFound"}, "meta": { "detailsSourceDate": self.source_date, - 'id': self.document_id, "version": version, + 'id': self.document_ids[0], "version": version, 'author': author}}]}]})))]) for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): """Recieve 404 and not valid data (worker dies). Check that worker get up""" gevent_sleep.side_effect = custom_sleep - edr_details_req_id = self.gen_req_id[0] - edr_req_id = self.gen_req_id[0:2] mrequest.get(self.url, [{'json': {'data': [{}]}, - 'headers': {'X-Request-ID': edr_req_id[0]}}, # data contains dict, instead of list - self.stat_200([{}], self.source_date, edr_req_id[1])]) - mrequest.get(self.url_id(321), [self.stat_200([{}], self.source_date, edr_details_req_id)]) + 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + # data contains dict, instead of list + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.url_id(321), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -432,18 +428,16 @@ def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): """Accept dict instead of list in first response to /verify endpoint. Check that worker get up""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:3] - mrequest.get(self.url, [self.stat_c(403, 0, '', edr_req_id[0]), + mrequest.get(self.url, [self.stat_c(403, 0, '', self.edr_req_ids[0]), {'json': {'data': [{}]}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_id[1]}}, - self.stat_200([{}], self.source_date, edr_req_id[2])]) - edrpou_codes_queue = Queue(10) - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, self.upload_to_doc_service_queue, - MagicMock(), MagicMock()) + 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + self.stat_200([{}], self.source_date, self.edr_req_ids[2])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker.process_tracker = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1], edr_req_id[2]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1], self.edr_req_ids[2]]))) self.assertEqual(mrequest.call_count, 3) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -455,22 +449,20 @@ def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): """Accept 6 times errors in response while requesting /verify""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:8] mrequest.get(self.url, - [self.stat_c(403, 0, '', edr_req_id[0]), - self.stat_c(403, 0, '', edr_req_id[1]), - self.stat_c(403, 0, '', edr_req_id[2]), - self.stat_c(403, 0, '', edr_req_id[3]), - self.stat_c(403, 0, '', edr_req_id[4]), - self.stat_c(403, 0, '', edr_req_id[5]), - self.stat_200([{}], self.source_date, edr_req_id[6])]) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock(), MagicMock()) - self.assertEquals(upload_to_doc_service_queue.get(), + [self.stat_c(403, 0, '', self.edr_req_ids[0]), + self.stat_c(403, 0, '', self.edr_req_ids[1]), + self.stat_c(403, 0, '', self.edr_req_ids[2]), + self.stat_c(403, 0, '', self.edr_req_ids[3]), + self.stat_c(403, 0, '', self.edr_req_ids[4]), + self.stat_c(403, 0, '', self.edr_req_ids[5]), + self.stat_200([{}], self.source_date, self.edr_req_ids[6])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker.process_tracker = MagicMock() + self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[6]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[6]]))) self.assertEqual(mrequest.call_count, 7) # processing 7 requests self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) # check first url @@ -483,18 +475,16 @@ def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): def test_timeout(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - edr_details_req_id = [self.gen_req_id[0], self.document_id] - edr_req_id = self.gen_req_id[0:7] - mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], edr_req_id[0]), - self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], - edr_details_req_id[0]), - self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, - [edr_req_id[0], edr_req_id[1]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -504,38 +494,30 @@ def test_timeout(self, mrequest, gevent_sleep): def test_identifier_id_type(self, mrequest, gevent_sleep): """Create filter_tenders and edr_handler workers. Test when identifier.id is type int (not str).""" gevent_sleep.side_effect = custom_sleep - bid_id = uuid.uuid4().hex - edr_req_id = self.gen_req_id[0] - # create queues - filtered_tender_ids_queue = Queue(10) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - filtered_tender_ids_queue.put(self.tender_id) - + self.filtered_tender_ids_queue.put(self.tender_id) # create workers and responses - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220'}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': self.award_id, - 'status': 'pending', - 'bid_id': bid_id, - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 14360570} - # int instead of str type - }]}, ]}})) - mrequest.get(self.url, [self.stat_200([{}], self.source_date, edr_req_id)]) - filter_tenders_worker = FilterTenders.spawn(client, filtered_tender_ids_queue, edrpou_codes_queue, + self.client.request.return_value = ResponseMock({'X-Request-ID': 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220'}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [{'id': self.award_id, + 'status': 'pending', + 'bid_id': self.bid_id, + 'suppliers': [{'identifier': { + 'scheme': 'UA-EDR', + 'id': 14360570} + # int instead of str type + }]}, ]}})) + mrequest.get(self.url, [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) + filter_tenders_worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, + self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, - MagicMock(), MagicMock()) + self.worker.process_tracker = MagicMock() - obj = upload_to_doc_service_queue.get() + obj = self.upload_to_doc_service_queue.get() self.assertEqual(obj.tender_id, self.tender_id) self.assertEqual(obj.item_id, self.award_id) self.assertEqual(obj.code, '14360570') @@ -546,9 +528,9 @@ def test_identifier_id_type(self, mrequest, gevent_sleep): self.assertEqual(obj.file_content['meta']['version'], version) self.assertEqual(obj.file_content['meta']['author'], author) self.assertEqual(obj.file_content['meta']['sourceRequests'], - ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', edr_req_id]) + ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', self.edr_req_ids[0]]) filter_tenders_worker.shutdown() - self.assertEqual(filtered_tender_ids_queue.qsize(), 0) + self.assertEqual(self.filtered_tender_ids_queue.qsize(), 0) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -556,43 +538,35 @@ def test_identifier_id_type(self, mrequest, gevent_sleep): def test_processing_items(self, mrequest, gevent_sleep): """Return list of objects from EDR. Check number of edr_ids in processing_items.""" gevent_sleep.side_effect = custom_sleep - qualification_id = uuid.uuid4().hex - document_ids = [generate_doc_id(), generate_doc_id()] - edr_req_id = self.gen_req_id[0:2] award_key = item_key(self.tender_id, self.award_id) - qualification_key = item_key(self.tender_id, qualification_id) + qualification_key = item_key(self.tender_id, self.qualification_id) data_1 = Data(self.tender_id, self.award_id, '123', "awards", - self.file_con({}, document_ids[0], 2, 1, [edr_req_id[0]])) + self.file_con({}, self.document_ids[0], 2, 1, [self.edr_req_ids[0]])) data_2 = Data(self.tender_id, self.award_id, '123', "awards", - self.file_con({}, document_ids[0], 2, 2, [edr_req_id[0]])) - data_3 = Data(self.tender_id, qualification_id, '124', 'qualifications', - self.file_con({}, document_ids[1], 3, 1, [edr_req_id[1]])) - data_4 = Data(self.tender_id, qualification_id, '124', 'qualifications', - self.file_con({}, document_ids[1], 3, 2, [edr_req_id[1]])) - data_5 = Data(self.tender_id, qualification_id, '124', 'qualifications', - self.file_con({}, document_ids[1], 3, 3, [edr_req_id[1]])) + self.file_con({}, self.document_ids[0], 2, 2, [self.edr_req_ids[0]])) + data_3 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', + self.file_con({}, self.document_ids[1], 3, 1, [self.edr_req_ids[1]])) + data_4 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', + self.file_con({}, self.document_ids[1], 3, 2, [self.edr_req_ids[1]])) + data_5 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', + self.file_con({}, self.document_ids[1], 3, 3, [self.edr_req_ids[1]])) mrequest.get(self.url, [ - self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], edr_req_id[0]), + self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[0]), self.stat_200([{}, {}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00", - "2017-04-25T11:56:36+00:00"], edr_req_id[1])]) + "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) # create queues - edrpou_codes_queue = Queue(10) - upload_to_doc_service = Queue(10) - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, '123', "awards", - {'meta': {'id': document_ids[0], 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - edrpou_codes_queue.put(Data(self.tender_id, qualification_id, '124', 'qualifications', - {'meta': {'id': document_ids[1], 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service, - self.process_tracker, MagicMock()) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, '123', "awards", + {'meta': {'id': self.document_ids[0], 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + self.edrpou_codes_queue.put(Data(self.tender_id, self.qualification_id, '124', 'qualifications', + {'meta': {'id': self.document_ids[1], 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) for data in [data_1, data_2, data_3, data_4, data_5]: - self.assertEquals(upload_to_doc_service.get(), data) - self.assertEqual(edrpou_codes_queue.qsize(), 0) + self.assertEquals(self.upload_to_doc_service_queue.get(), data) + self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertEqual(self.process_tracker.processing_items[award_key], 2) self.assertEqual(self.process_tracker.processing_items[qualification_key], 3) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) @@ -602,16 +576,15 @@ def test_processing_items(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_wrong_ip(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] - edr_details_req_id = self.gen_req_id[0:2] - mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Forbidden'}], edr_req_id[0]), - self.stat_200([{}], self.source_date, edr_req_id[1])]) - mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Forbidden'}], edr_details_req_id[0]), - self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Forbidden'}], self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Forbidden'}], self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -620,31 +593,29 @@ def test_wrong_ip(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for edrpou_codes_queue """ - document_ids = [self.document_id, self.document_id] gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] - edr_details_req_id = self.gen_req_id[0:2] mrequest.get(self.uri, - [self.stat_200([{}], self.source_date, edr_req_id[0]), - self.stat_200([{}], self.source_date, edr_req_id[1])]) + [self.stat_200([{}], self.source_date, self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) mrequest.get(self.url_id(self.local_edr_ids[0]), - [self.stat_200([{}], self.source_date, edr_details_req_id[0])]) + [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) mrequest.get(self.url_id(self.local_edr_ids[1]), - [self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + [self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) edrpou_codes_queue = MagicMock() expected_result = [] edrpou_codes_queue_list = [LoopExit()] for i in range(2): edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - {'meta': {'id': document_ids[i], 'author': author, 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data + {'meta': {'id': self.document_ids[i], 'author': author, + 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, document_ids[i], 1, 1, [edr_req_id[i]]))) # result + self.file_con({}, self.document_ids[i], 1, 1, [self.edr_req_ids[i]]))) # result edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) self.worker.retry_edrpou_codes_queue = MagicMock() - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, self.upload_to_doc_service_queue, - MagicMock(), MagicMock()) + self.worker.process_tracker = MagicMock() + self.worker.edrpou_codes_queue = edrpou_codes_queue for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(expected_result[0].code)) @@ -655,23 +626,21 @@ def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for retry_edrpou_codes_queue """ - document_ids = [self.document_id, self.document_id] gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] - edr_details_req_id = self.gen_req_id[0:2] - mrequest.get(self.uri, [self.stat_200([{}], self.source_date, edr_req_id[0]), - self.stat_200([{}], self.source_date, edr_req_id[1])]) - mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, edr_details_req_id[0])]) - mrequest.get(self.url_id(self.local_edr_ids[1]), [self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + mrequest.get(self.uri, [self.stat_200([{}], self.source_date, self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) + mrequest.get(self.url_id(self.local_edr_ids[1]), [self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) edrpou_codes_queue_list = [LoopExit()] expected_result = [] for i in range(2): edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - {"meta": {"id": document_ids[i], 'author': author, 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data + {"meta": {"id": self.document_ids[i], 'author': author, + 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, document_ids[i], 1, 1, [edr_req_id[i]]))) # result + self.file_con({}, self.document_ids[i], 1, 1, [self.edr_req_ids[i]]))) # result self.worker.retry_edrpou_codes_queue = MagicMock() self.worker.retry_edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) @@ -686,29 +655,28 @@ def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): def test_retry_exception(self, mrequest, gevent_sleep): """ Raise RetryException in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = self.gen_req_id[0:2] retry_response = MagicMock() retry_response.status_code = 500 mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, {'exc': RetryException('Retry Exception', retry_response)}, {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) + 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}]) expected_result = [] for i in range(1): - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", - self.file_con({}, self.document_id, 1, 1, - [edr_req_ids[0], edr_req_ids[1]]))) # result - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) @@ -720,7 +688,6 @@ def test_retry_exception(self, mrequest, gevent_sleep): def test_retry_exception_404(self, mrequest, gevent_sleep): """ Raise RetryException in retry_get_edr_data with status_code 404""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = self.gen_req_id[0] retry_response = MagicMock() retry_response.status_code = 404 retry_response.json = MagicMock() @@ -728,32 +695,31 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, 'meta': {"detailsSourceDate": self.source_date}}]}]} mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, {'exc': RetryException('Retry Exception', retry_response)}]) - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - expected_result = [] for i in range(1): - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", {'error': {'errorDetails': "Couldn't find this code in EDR.", 'code': 'notFound'}, 'meta': {'detailsSourceDate': self.source_date, - 'id': self.document_id, "version": version, 'author': author, + 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids]}})) # result + self.edr_req_ids[0]]}})) # result - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.call_count, 6) @@ -763,27 +729,24 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): def test_exception(self, mrequest, gevent_sleep): """ Raise Exception in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = self.gen_req_id[0:2] mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'exc': Exception()}, self.stat_200([{}], self.source_date, edr_req_ids[1])]) - - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, + {'exc': Exception()}, self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(1): - edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", - self.file_con({}, self.document_id, 1, 1, - [edr_req_ids[0], edr_req_ids[1]]))) # result - self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock(), MagicMock()) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result + self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(check_queue.get(), result) + self.assertEquals(self.check_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) @@ -795,20 +758,20 @@ def test_exception(self, mrequest, gevent_sleep): def test_value_error(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] mrequest.get(self.url, - [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], edr_req_id[0]), - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - self.stat_200([{}], self.source_date, edr_req_id[1])]) + [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0]), + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.worker.retry_edr_ids_queue = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 7) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -820,35 +783,35 @@ def test_value_error(self, mrequest, gevent_sleep): def test_value_error_mock(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] self.worker.retry_edr_ids_queue = MagicMock() self.worker.retry_edrpou_codes_queue.put( Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.worker.get_edr_data_request = MagicMock(side_effect=[RetryException("Exception", Response()), - MagicMock(headers={'X-Request-ID': edr_req_id[0]}, - status_code=200, - json=MagicMock(return_value=munchify( - {'data': [{}], "meta": { - "detailsSourceDate": - self.source_date}})))]) + MagicMock( + headers={'X-Request-ID': self.edr_req_ids[0]}, + status_code=200, + json=MagicMock(return_value=munchify( + {'data': [{}], "meta": { + "detailsSourceDate": + self.source_date}})))]) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0]]))) + self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[0]]))) @requests_mock.Mocker() @patch('gevent.sleep') def test_429_mock(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - edr_req_id = self.gen_req_id[0:2] self.worker.retry_edr_ids_queue = MagicMock() self.worker.retry_edrpou_codes_queue.put( Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) self.worker.get_edr_data_request = MagicMock( - side_effect=[MagicMock(headers={'X-Request-ID': edr_req_id[0], 'Retry-After': '1'}, status_code=429), - MagicMock(headers={'X-Request-ID': edr_req_id[1]}, status_code=200, + side_effect=[MagicMock(headers={'X-Request-ID': self.edr_req_ids[0], 'Retry-After': '1'}, status_code=429), + MagicMock(headers={'X-Request-ID': self.edr_req_ids[1]}, status_code=200, json=MagicMock(return_value=munchify( {'data': [{}], "meta": {"detailsSourceDate": self.source_date}})))]) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) + self.file_con({}, self.document_ids[0], 1, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) diff --git a/openprocurement/bot/identification/tests/filter_tender.py b/openprocurement/bot/identification/tests/filter_tender.py index 9489af9..efaee7f 100644 --- a/openprocurement/bot/identification/tests/filter_tender.py +++ b/openprocurement/bot/identification/tests/filter_tender.py @@ -54,8 +54,8 @@ def response_get_tender(): 'bid_id': '111', 'status': 'pending', 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'}}]}]}}) + 'scheme': 'UA-EDR', + 'id': '14360570'}}]}]}}) def generate_response(): @@ -67,16 +67,34 @@ def generate_response(): class TestFilterWorker(unittest.TestCase): - def setUp(self): self.filtered_tender_ids_queue = Queue(10) self.edrpou_codes_queue = Queue(10) self.process_tracker = ProcessTracker() - self.request_id = generate_request_id() self.tender_id = uuid.uuid4().hex - self.award_id = uuid.uuid4().hex - self.bid_id = uuid.uuid4().hex + self.filtered_tender_ids_queue.put(self.tender_id) self.sleep_change_value = APIRateController() + self.client = MagicMock() + self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, + self.process_tracker, MagicMock(), self.sleep_change_value) + self.bid_ids = [uuid.uuid4().hex for _ in range(5)] + self.qualification_ids = [uuid.uuid4().hex for _ in range(5)] + self.award_ids = [uuid.uuid4().hex for _ in range(5)] + self.request_ids = [generate_request_id() for _ in range(2)] + + def tearDown(self): + self.worker.shutdown() + del self.worker + + def awards(self, counter_id, counter_bid_id, status, sup_id): + return {'id': self.award_ids[counter_id], 'bid_id': self.bid_ids[counter_bid_id], 'status': status, + 'suppliers': [{'identifier': {'scheme': 'UA-EDR', 'id': sup_id}}]} + + def bids(self, counter_id, ten_id): + return {'id': self.bid_ids[counter_id], 'tenderers': [{'identifier': {'scheme': 'UA-EDR', 'id': ten_id}}]} + + def qualifications(self, status, counter_qual_id, counter_bid_id): + return {'status': status, 'id': self.qualification_ids[counter_qual_id], 'bidID': self.bid_ids[counter_bid_id]} def check_data_objects(self, obj, example): """Checks that two data objects are equal, @@ -102,521 +120,298 @@ def test_init(self): self.assertEqual(worker.sleep_change_value.time_between_requests, 0) self.assertEqual(worker.delay, 15) self.assertEqual(worker.exit, False) - worker.shutdown() del worker @patch('gevent.sleep') def test_worker_qualification(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - bid_ids = [uuid.uuid4().hex for i in range(5)] - qualification_ids = [uuid.uuid4().hex for i in range(5)] - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': self.request_id}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'bids': [{'id': bid_ids[0], - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }]}, - {'id': bid_ids[1], - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '0013823'} - }]}, - {'id': bid_ids[2], - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '23494714'} - }]}, - {'id': bid_ids[3], - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '23494714'} - }]}, - {'id': bid_ids[4], - 'tenderers': [{'identifier': { - 'scheme': 'UA-ED', - 'id': '23494714'} - }]}, - ], - 'qualifications': [{'status': 'pending', - 'id': qualification_ids[0], - 'bidID': bid_ids[0]}, - {'status': 'pending', - 'id': qualification_ids[1], - 'bidID': bid_ids[1]}, - {'status': 'pending', - 'id': qualification_ids[2], - 'bidID': bid_ids[2]}, - {'status': 'unsuccessful', - 'id': qualification_ids[3], - 'bidID': bid_ids[3]}, - {'status': 'pending', - 'id': qualification_ids[4], - 'bidID': bid_ids[4]}, - ]}})) - first_data = Data(self.tender_id, qualification_ids[0], '14360570', 'qualifications', {'meta': {'sourceRequests': [self.request_id]}}) - second_data = Data(self.tender_id, qualification_ids[1], '0013823', 'qualifications', {'meta': {'sourceRequests': [self.request_id]}}) - third_data = Data(self.tender_id, qualification_ids[2], '23494714', 'qualifications', {'meta': {'sourceRequests': [self.request_id]}}) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) - + self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify( + {'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'bids': [self.bids(0, '14360570'), + self.bids(1, '0013823'), + self.bids(2, '23494714'), + self.bids(3, '23494714'), + {'id': self.bid_ids[4], + 'tenderers': [{'identifier': { + 'scheme': 'UA-ED', + 'id': '23494714'}}]}], + 'qualifications': [ + self.qualifications('pending', 0, 0), + self.qualifications('pending', 1, 1), + self.qualifications('pending', 2, 2), + self.qualifications('unsuccessful', 3, 3), + self.qualifications('pending', 4, 4)]}})) + first_data = Data(self.tender_id, self.qualification_ids[0], '14360570', 'qualifications', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + second_data = Data(self.tender_id, self.qualification_ids[1], '0013823', 'qualifications', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + third_data = Data(self.tender_id, self.qualification_ids[2], '23494714', 'qualifications', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) for data in [first_data, second_data, third_data]: self.check_data_objects(self.edrpou_codes_queue.get(), data) - - worker.shutdown() - del worker - self.assertItemsEqual(self.process_tracker.processing_items.keys(), - [item_key(self.tender_id, qualification_ids[0]), - item_key(self.tender_id, qualification_ids[1]), - item_key(self.tender_id, qualification_ids[2])]) + [item_key(self.tender_id, self.qualification_ids[i]) for i in range(3)]) @patch('gevent.sleep') def test_worker_award(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - bid_ids = [uuid.uuid4().hex for i in range(5)] - award_ids = [uuid.uuid4().hex for i in range(5)] - client = MagicMock() - client.request.side_effect = [ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': award_ids[0], - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }]}, - {'id': award_ids[1], - 'bid_id': bid_ids[1], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '0013823'} - }]}, - {'id': award_ids[2], - 'bid_id': bid_ids[2], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '23494714'} - }]}, - {'id': award_ids[3], - 'bid_id': bid_ids[3], - 'status': 'unsuccessful', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '23494714'} - }]}, - {'id': award_ids[4], - 'bid_id': bid_ids[4], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-ED', - 'id': '23494714'} - }]}, - ] - }}))] - - first_data = Data(self.tender_id, award_ids[0], '14360570', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - second_data = Data(self.tender_id, award_ids[1], '0013823', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - third_data = Data(self.tender_id, award_ids[2], '23494714', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) + self.client.request.side_effect = [ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [ + self.awards(0, 0, 'pending', '14360570'), + self.awards(1, 1, 'pending', '0013823'), + self.awards(2, 2, 'pending', '23494714'), + self.awards(3, 3, 'unsuccessful', + '23494714'), + {'id': self.bid_ids[4], + 'tenderers': [{'identifier': { + 'scheme': 'UA-ED', + 'id': '23494714'}}]}]}}))] + first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + second_data = Data(self.tender_id, self.award_ids[1], '0013823', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + third_data = Data(self.tender_id, self.award_ids[2], '23494714', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) for edrpou in [first_data, second_data, third_data]: self.check_data_objects(self.edrpou_codes_queue.get(), edrpou) - - worker.shutdown() - del worker - self.assertItemsEqual(self.process_tracker.processing_items.keys(), - [item_key(self.tender_id, award_ids[0]), item_key(self.tender_id, award_ids[1]), - item_key(self.tender_id, award_ids[2])]) + [item_key(self.tender_id, self.award_ids[0]), item_key(self.tender_id, self.award_ids[1]), + item_key(self.tender_id, self.award_ids[2])]) @patch('gevent.sleep') def test_get_tender_exception(self, gevent_sleep): """ We must not lose tender after restart filter worker """ gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - client = MagicMock() - client.request.side_effect = [Exception(), - ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': self.award_id, - 'bid_id': self.bid_id, - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }]} - ] - }}))] - data = Data(self.tender_id, self.award_id, '14360570', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) + self.client.request.side_effect = [Exception(), + ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(0, 0, 'pending', + '14360570')]}}))] + data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) - worker.shutdown() - self.assertEqual(worker.sleep_change_value.time_between_requests, 0) - del worker + self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0) gevent_sleep.assert_called_with_once(1) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[0])]) self.assertEqual(self.edrpou_codes_queue.qsize(), 0) @patch('gevent.sleep') def test_get_tender_429(self, gevent_sleep): """ We must not lose tender after restart filter worker """ gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - client = MagicMock() - client.request.side_effect = [ - ResourceError(http_code=429), - ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': self.award_id, - 'status': 'pending', - 'bid_id': self.bid_id, - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }]} - ] - }}))] - data = Data(self.tender_id, self.award_id, '14360570', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) + self.client.request.side_effect = [ + ResourceError(http_code=429), + ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(0, 0, 'pending', '14360570')]}}))] + data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) self.check_data_objects(self.edrpou_codes_queue.get(), data) - worker.shutdown() - self.assertEqual(worker.sleep_change_value.time_between_requests, 1) - del worker + self.assertEqual(self.worker.sleep_change_value.time_between_requests, 1) gevent_sleep.assert_called_with_once(1) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[0])]) self.assertEqual(self.edrpou_codes_queue.qsize(), 0) @patch('gevent.sleep') def test_worker_restart(self, gevent_sleep): """ Process tender after catch Unauthorized exception """ gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - award_ids = [uuid.uuid4().hex for i in range(2)] - bid_ids = [uuid.uuid4().hex for i in range(2)] - client = MagicMock() - client.request.side_effect = [ + self.client.request.side_effect = [ Unauthorized(http_code=403), Unauthorized(http_code=403), Unauthorized(http_code=403), - ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': award_ids[0], - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }]}, - {'id': award_ids[1], - 'bid_id': bid_ids[1], - 'status': 'unsuccessful', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '23494714'} - }]}, - ] - }})) - ] - data = Data(self.tender_id, award_ids[0], '14360570', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) - + ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(0, 0, 'pending', '14360570'), + self.awards(1, 1, 'unsuccessful', '23494714')]}}))] + data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) - - worker.shutdown() - del worker - - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, award_ids[0])]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[0])]) @patch('gevent.sleep') def test_worker_dead(self, gevent_sleep): """ Test that worker will process tender after exception """ gevent_sleep.side_effect = custom_sleep self.filtered_tender_ids_queue.put(self.tender_id) - self.filtered_tender_ids_queue.put(self.tender_id) - bid_ids = [uuid.uuid4().hex for i in range(2)] - award_ids = [uuid.uuid4().hex for i in range(2)] - request_ids = [generate_request_id() for i in range(2)] - client = MagicMock() - client.request.side_effect = [ - ResponseMock({'X-Request-ID': request_ids[0]}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': { - 'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [ - {'id': award_ids[0], - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [ - {'identifier': {'scheme': 'UA-EDR', - 'id': '14360570'}}] - }]}})), - ResponseMock({'X-Request-ID': request_ids[1]}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': { - 'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [ - {'id': award_ids[1], - 'bid_id': bid_ids[1], - 'status': 'pending', - 'suppliers': [ - {'identifier': {'scheme': 'UA-EDR', - 'id': '14360570'} - }]}]}}))] - first_data = Data(self.tender_id, award_ids[0], '14360570', 'awards', {'meta': {'sourceRequests': [request_ids[0]]}}) - second_data = Data(self.tender_id, award_ids[1], '14360570', 'awards', {'meta': {'sourceRequests': [request_ids[1]]}}) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) + self.client.request.side_effect = [ + ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify( + {'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': { + 'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(0, 0, 'pending', '14360570')]}})), + ResponseMock({'X-Request-ID': self.request_ids[1]}, + munchify( + {'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': { + 'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(1, 1, 'pending', '14360570')]}}))] + first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + second_data = Data(self.tender_id, self.award_ids[1], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[1]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), first_data) - worker.job.kill(timeout=1) + self.worker.job.kill(timeout=1) self.check_data_objects(self.edrpou_codes_queue.get(), second_data) - - worker.shutdown() - del worker - - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, award_ids[0]), - item_key(self.tender_id, award_ids[1])]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[i]) for i in range(2)]) @patch('gevent.sleep') def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep filtered_tender_ids_queue = MagicMock() filtered_tender_ids_queue.peek.side_effect = [LoopExit(), self.tender_id] - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': self.request_id}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': { - 'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [ - {'id': self.award_id, - 'bid_id': self.bid_id, - 'status': 'pending', - 'suppliers': [ - {'identifier': {'scheme': 'UA-EDR', - 'id': '14360570'}}] - }]}})) - first_data = Data(self.tender_id, self.award_id, '14360570', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - worker = FilterTenders.spawn(client, filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) + self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify( + {'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': { + 'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [ + self.awards(0, 0, 'pending', '14360570')]}})) + first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) + self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue self.check_data_objects(self.edrpou_codes_queue.get(), first_data) - - worker.shutdown() - del worker - - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[0])]) @patch('gevent.sleep') def test_worker_award_with_cancelled_lot(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - award_ids = [uuid.uuid4().hex for i in range(2)] - bid_ids = [uuid.uuid4().hex for i in range(2)] - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'lots': [{'status': 'cancelled', - 'id': '123456789'}, - {'status': 'active', - 'id': '12345678'} - ], - 'awards': [{'id': award_ids[0], - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '14360570'} - }], - 'lotID': '123456789'}, - {'id': award_ids[1], - 'bid_id': bid_ids[1], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': '0013823'} - }], - 'lotID': '12345678'} - ] - }})) - - data = Data(self.tender_id, award_ids[1], '0013823', 'awards', {'meta': {'sourceRequests': [self.request_id]}}) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) - + self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'lots': [{'status': 'cancelled', + 'id': '123456789'}, + {'status': 'active', + 'id': '12345678'}], + 'awards': [{'id': self.award_ids[0], + 'bid_id': self.bid_ids[0], + 'status': 'pending', + 'suppliers': [{'identifier': { + 'scheme': 'UA-EDR', + 'id': '14360570'}}], + 'lotID': '123456789'}, + {'id': self.award_ids[1], + 'bid_id': self.bid_ids[1], + 'status': 'pending', + 'suppliers': [{'identifier': { + 'scheme': 'UA-EDR', + 'id': '0013823'}}], + 'lotID': '12345678'}]}})) + data = Data(self.tender_id, self.award_ids[1], '0013823', 'awards', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) for edrpou in [data]: self.check_data_objects(self.edrpou_codes_queue.get(), edrpou) - - worker.shutdown() - del worker - - self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, award_ids[1])]) + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + [item_key(self.tender_id, self.award_ids[1])]) @patch('gevent.sleep') def test_qualification_not_valid_identifier_id(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - first_bid_id, second_bid_id = [uuid.uuid4().hex for i in range(2)] - first_qualification_id, second_qualification_id = [uuid.uuid4().hex for i in range(2)] - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': self.request_id}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'bids': [{'id': first_bid_id, - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 'test@test.com'} - }]}, - {'id': second_bid_id, - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': ''} - }]}, - {'id': second_bid_id, - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 'абв'} - }]}, - {'id': second_bid_id, - 'tenderers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': u'абв'} - }]} - ], - 'qualifications': [{'status': 'pending', - 'id': first_qualification_id, - 'bidID': first_bid_id}, - {'status': 'pending', - 'id': second_qualification_id, - 'bidID': second_bid_id}]}})) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) - + self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify( + {'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'bids': [self.bids(0, 'test@test.com'), + self.bids(1, ''), + self.bids(1, 'абв'), + self.bids(1, u'абв')], + 'qualifications': [ + self.qualifications('pending', 0, 0), + self.qualifications('pending', 1, 1)]}})) sleep(1) self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertItemsEqual(self.process_tracker.processing_items, {}) - worker.shutdown() - del worker - @patch('gevent.sleep') def test_award_not_valid_identifier_id(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - first_award_id, second_award_id = [uuid.uuid4().hex for i in range(2)] - bid_ids = [uuid.uuid4().hex for i in range(2)] - client = MagicMock() - client.request.return_value = ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': first_award_id, - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': ''} - }]}, - {'id': first_award_id, - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': u'абв'} - }]}, - {'id': first_award_id, - 'bid_id': bid_ids[1], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 'абв'} - }]}, - {'id': second_award_id, - 'bid_id': bid_ids[1], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 'test@test.com'} - }]}] - }})) - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) - + self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(0, 0, 'pending', ''), + self.awards(0, 0, 'pending', + u'абв'), + self.awards(0, 1, 'pending', + 'абв'), + self.awards(1, 1, 'pending', + 'test@test.com')]}})) sleep(1) - self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertItemsEqual(self.process_tracker.processing_items, {}) - worker.shutdown() - del worker - @patch('gevent.sleep') def test_412(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put('123') + self.worker.kill() + filtered_tender_ids_queue = Queue(10) + filtered_tender_ids_queue.put('123') api_server_bottle = Bottle() api_server = WSGIServer(('127.0.0.1', 20604), api_server_bottle, log=None) setup_routing(api_server_bottle, response_spore) setup_routing(api_server_bottle, generate_response, path='/api/2.3/tenders/123') api_server.start() client = TendersClientSync('', host_url='http://127.0.0.1:20604', api_version='2.3') - self.assertEqual(client.headers['Cookie'], 'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) + self.assertEqual(client.headers['Cookie'], + 'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies + worker = FilterTenders.spawn(client, filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, + MagicMock(), self.sleep_change_value) data = Data('123', '124', '14360570', 'awards', {'meta': {'sourceRequests': ['125']}}) for i in [data]: self.check_data_objects(self.edrpou_codes_queue.get(), i) - self.assertEqual(client.headers['Cookie'], 'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies + self.assertEqual(client.headers['Cookie'], + 'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertItemsEqual(self.process_tracker.processing_items.keys(), ['123_124']) @@ -627,31 +422,16 @@ def test_412(self, gevent_sleep): @patch('gevent.sleep') def test_request_failed(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.filtered_tender_ids_queue.put(self.tender_id) - bid_ids = [uuid.uuid4().hex for i in range(2)] - client = MagicMock() - client.request.side_effect = [RequestFailed(http_code=401, msg=RequestFailed()), - ResponseMock({'X-Request-ID': self.request_id}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': self.award_id, - 'bid_id': bid_ids[0], - 'status': 'pending', - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': ''} - }]}] - }}))] - worker = FilterTenders.spawn(client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) - + self.client.request.side_effect = [RequestFailed(http_code=401, msg=RequestFailed()), + ResponseMock({'X-Request-ID': self.request_ids[0]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [ + self.awards(0, 0, 'pending', '')]}}))] sleep(1) - - self.assertEqual(client.request.call_count, 2) + self.assertEqual(self.client.request.call_count, 2) self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertItemsEqual(self.process_tracker.processing_items, {}) - - worker.shutdown() - del worker diff --git a/openprocurement/bot/identification/tests/scanner.py b/openprocurement/bot/identification/tests/scanner.py index 55e26a1..c73047b 100644 --- a/openprocurement/bot/identification/tests/scanner.py +++ b/openprocurement/bot/identification/tests/scanner.py @@ -11,9 +11,7 @@ from mock import patch, MagicMock from time import sleep from munch import munchify -from restkit.errors import ( - Unauthorized, RequestFailed, ResourceError -) +from restkit.errors import (Unauthorized, RequestFailed, ResourceError) from openprocurement.bot.identification.databridge.scanner import Scanner from openprocurement.bot.identification.tests.utils import custom_sleep @@ -24,324 +22,160 @@ class TestScannerWorker(unittest.TestCase): def setUp(self): self.process_tracker = ProcessTracker(MagicMock(has=MagicMock(return_value=False))) + self.tenders_id = [uuid.uuid4().hex for _ in range(4)] self.sleep_change_value = APIRateController() + self.client = MagicMock() + self.tender_queue = Queue(10) + self.worker = Scanner.spawn(self.client, self.tender_queue, MagicMock(), self.process_tracker, + self.sleep_change_value) + + def tearDown(self): + self.worker.shutdown() + del self.worker + + @staticmethod + def munchify(prev_p, next_p, status, id, procurementMethodType, data=True): + if data: + return munchify({'prev_page': {'offset': prev_p}, + 'next_page': {'offset': next_p}, + 'data': [{'status': status, + "id": id, + 'procurementMethodType': 'aboveThreshold{}'.format(procurementMethodType)}]}) + else: + return munchify({'prev_page': {'offset': prev_p}, + 'next_page': {'offset': next_p}, + 'data': []}) def test_init(self): - client = MagicMock() - tender_queue = Queue(10) - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - self.assertGreater(datetime.datetime.now().isoformat(), worker.start_time.isoformat()) - self.assertEqual(worker.tenders_sync_client, client) - self.assertEqual(worker.filtered_tender_ids_queue, tender_queue) - self.assertEqual(worker.sleep_change_value.time_between_requests, 0) - self.assertEqual(worker.delay, 15) - self.assertEqual(worker.exit, False) - - worker.shutdown() - del worker + self.assertGreater(datetime.datetime.now().isoformat(), self.worker.start_time.isoformat()) + self.assertEqual(self.worker.tenders_sync_client, self.client) + self.assertEqual(self.worker.filtered_tender_ids_queue, self.tender_queue) + self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0) + self.assertEqual(self.worker.delay, 15) + self.assertEqual(self.worker.exit, False) @patch('gevent.sleep') def test_worker(self, gevent_sleep): """ Returns tenders, check queue elements after filtering """ gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex, uuid.uuid4().hex] - client.sync_tenders.side_effect = [ + self.client.sync_tenders.side_effect = [ RequestFailed(), # worker must restart - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdUA'}]}), + self.munchify('123', '1234', "active.qualification", self.tenders_id[0], 'UA'), Unauthorized(), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.tendering", - "id": uuid.uuid4().hex, - 'procurementMethodType': 'aboveThresholdUA'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]}) - ] - - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU')] + for tender_id in self.tenders_id[0:2]: + self.assertEqual(self.tender_queue.get(), tender_id) @patch('gevent.sleep') def test_429(self, gevent_sleep): """Receive 429 status, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(3)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1235'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '1235'}, - 'next_page': {'offset': '1236'}, - 'data': [{'status': "active.tendering", - "id": uuid.uuid4().hex, - 'procurementMethodType': 'aboveThresholdUA'}]}), - munchify({'prev_page': {'offset': '1236'}, - 'next_page': {'offset': '1237'}, - 'data': [{'status': "active.qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdUA'}]}), + self.client.sync_tenders.side_effect = [ + self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify('1235', '1236', "active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify('1236', '1237', "active.qualification", self.tenders_id[1], 'UA'), ResourceError(http_code=429), - munchify({'prev_page': {'offset': '1237'}, - 'next_page': {'offset': '1238'}, - 'data': [{'status': "active.qualification", - "id": tenders_id[2], - 'procurementMethodType': 'aboveThresholdUA'}]})] - + self.munchify('1237', '1238', "active.qualification", self.tenders_id[2], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + for tender_id in self.tenders_id[0:3]: + self.assertEqual(self.tender_queue.get(), tender_id) self.assertEqual(self.sleep_change_value.time_between_requests, 1) @patch('gevent.sleep') def test_429_sleep_change_value(self, gevent_sleep): """Three times receive 429, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(2)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.tendering", - "id": uuid.uuid4().hex, - 'procurementMethodType': 'aboveThresholdUA'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.tendering", - "id": uuid.uuid4().hex, - 'procurementMethodType': 'aboveThresholdUA'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.tendering", - "id": uuid.uuid4().hex, - 'procurementMethodType': 'aboveThresholdUA'}]}), + self.client.sync_tenders.side_effect = [ + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), ResourceError(http_code=429), ResourceError(http_code=429), ResourceError(http_code=429), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]})] - + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - + for tender_id in self.tenders_id[0:2]: + self.assertEqual(self.tender_queue.get(), tender_id) self.assertEqual(self.sleep_change_value.time_between_requests, 2.5) - worker.shutdown() - del worker @patch('gevent.sleep') def test_backward_dead(self, gevent_sleep): """Test when backward dies """ gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(3)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1235'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]}), + self.client.sync_tenders.side_effect = [ + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU'), ResourceError(http_code=403), - munchify({'prev_page': {'offset': '1235'}, - 'next_page': {'offset': '1236'}, - 'data': []}), - munchify({'prev_page': {'offset': '1236'}, - 'next_page': {'offset': '1237'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[2], - 'procurementMethodType': 'aboveThresholdEU'}]})] - + self.munchify('1235', '1236', None, None, False), + self.munchify('1236', '1237', "active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + for tender_id in self.tenders_id[0:3]: + self.assertEqual(self.tender_queue.get(), tender_id) @patch('gevent.sleep') def test_forward_dead(self, gevent_sleep): """ Test when forward dies""" gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(2)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '123'}, - 'data': []}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '123'}, - 'data': []}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1235'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdEU'}]}), + self.client.sync_tenders.side_effect = [ + self.munchify('1234', '123', None, None, None, False), + self.munchify('1234', '123', None, None, None, False), + self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[0], 'EU'), ResourceError(http_code=403), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1235'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]})] - + self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + for tender_id in self.tenders_id[0:2]: + self.assertEqual(self.tender_queue.get(), tender_id) @patch('gevent.sleep') def test_forward_run(self, gevent_sleep): """ Run forward when backward get empty response and prev_page.offset is equal to next_page.offset """ gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(3)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[0], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1235'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '1235'}, - 'next_page': {'offset': '1235'}, - 'data': []}), - munchify({'prev_page': {'offset': '1236'}, - 'next_page': {'offset': '1237'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[2], - 'procurementMethodType': 'aboveThresholdEU'}]})] - + self.client.sync_tenders.side_effect = [ + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU'), + self.munchify('1235', '1235', None, None, None, False), + self.munchify('1236', '1237', "active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + for tender_id in self.tenders_id[0:3]: + self.assertEqual(self.tender_queue.get(), tender_id) @patch('gevent.sleep') def test_get_tenders_exception(self, gevent_sleep): """ Catch exception in backward worker and after that put 2 tenders to process.Then catch exception for forward and after that put tender to process.""" gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tenders_id = [uuid.uuid4().hex for _ in range(4)] - client.sync_tenders.side_effect = [ - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - 'procurementMethodType': 'aboveThresholdEU', - 'id': tenders_id[0]}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': []}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[1], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[2], - 'procurementMethodType': 'aboveThresholdEU'}]}), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': []}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': None}, - 'data': []}), - munchify({'prev_page': {'offset': '1234'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tenders_id[3], - 'procurementMethodType': 'aboveThresholdEU'}]}) - ] - + self.client.sync_tenders.side_effect = [ + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify('123', '1234', None, None, None, False), + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU'), + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[2], 'EU'), + self.munchify('123', '1234', None, None, None, False), + self.munchify('1234', None, None, None, None, False), + self.munchify('1234', '1234', "active.pre-qualification", self.tenders_id[3], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - - for tender_id in tenders_id: - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + for tender_id in self.tenders_id: + self.assertEqual(self.tender_queue.get(), tender_id) @patch('gevent.sleep') def test_resource_error(self, gevent_sleep): """Raise Resource error, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tender_id = uuid.uuid4().hex - client.sync_tenders.side_effect = [ + self.client.sync_tenders.side_effect = [ ResourceError(http_code=429), - munchify({'prev_page': {'offset': '1237'}, - 'next_page': {'offset': '1238'}, - 'data': [{'status': "active.qualification", - "id": tender_id, - 'procurementMethodType': 'aboveThresholdUA'}]})] - + self.munchify('1237', '1238', "active.qualification", self.tenders_id[0], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - self.assertEqual(tender_queue.get(), tender_id) - worker.shutdown() - del worker + self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) self.assertEqual(self.sleep_change_value.time_between_requests, 0) @patch('gevent.sleep') @@ -362,21 +196,10 @@ def test_forward_exception(self, gevent_sleep): """ Run forward when backward get empty response and prev_page.offset is equal to next_page.offset """ gevent_sleep.side_effect = custom_sleep - tender_queue = Queue(10) - client = MagicMock() - tender_id = uuid.uuid4().hex self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - worker = Scanner.spawn(client, tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value) - worker.initialize_sync = MagicMock(side_effect=[ + self.worker.initialize_sync = MagicMock(side_effect=[ ResourceError(msg=RequestFailed()), - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': [{'status': "active.pre-qualification", - "id": tender_id, - 'procurementMethodType': 'aboveThresholdEU'}]}) - ]) - self.assertEqual(tender_queue.get(), tender_id) - self.assertEqual(worker.initialize_sync.call_count, 2) - worker.shutdown() - del worker + self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU')]) + self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) + self.assertEqual(self.worker.initialize_sync.call_count, 2) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 4673c04..83b3413 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from gevent import monkey + monkey.patch_all() import uuid @@ -67,7 +68,6 @@ def generate_response(): class TestUploadFileWorker(unittest.TestCase): - def setUp(self): self.tender_id = uuid.uuid4().hex self.award_id = uuid.uuid4().hex @@ -95,10 +95,10 @@ def is_working(self, worker): worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()) def shutdown_when_done(self, worker): - self.worker.start() - while self.is_working(self.worker): + worker.start() + while self.is_working(worker): sleep(0.1) - self.worker.shutdown() + worker.shutdown() def test_init(self): worker = UploadFile.spawn(None, None, None, None, None, None, None) @@ -122,15 +122,16 @@ def test_init(self): def test_successful_upload(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) @@ -141,7 +142,8 @@ def test_successful_upload(self, mrequest, gevent_sleep): self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) # test that item removed from processing_items + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + []) # test that item removed from processing_items self.assertEqual(self.client._create_tender_resource_item.call_count, 1) # check upload to tender @requests_mock.Mocker() @@ -150,21 +152,16 @@ def test_retry_doc_service(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post('{url}'.format(url=doc_service_client.url), - [{'text': '', 'status_code': 401}, - {'text': '', 'status_code': 401}, - {'text': '', 'status_code': 401}, - {'text': '', 'status_code': 401}, - {'text': '', 'status_code': 401}, - {'text': '', 'status_code': 401}, - {'json': {'data': {'url': 'test url', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) + [{'text': '', 'status_code': 401} for _ in range(6)] + [ + {'json': {'data': {'url': 'test url', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + 'status_code': 200}]) self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) @@ -174,7 +171,8 @@ def test_retry_doc_service(self, mrequest, gevent_sleep): self.assertEqual(mrequest.call_count, 7) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) # test that item removed from processing_items + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + []) # test that item removed from processing_items self.assertEqual(self.client._create_tender_resource_item.call_count, 1) # check upload to tender @patch('gevent.sleep') @@ -212,18 +210,19 @@ def test_upload_to_tender_exception_status_int_none(self, gevent_sleep): def test_retry_upload_to_tender(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) self.client._create_tender_resource_item.side_effect = [Unauthorized(http_code=401), - Unauthorized(http_code=403), - Unauthorized(http_code=429), - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + Unauthorized(http_code=403), + Unauthorized(http_code=429), + {'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) @@ -249,7 +248,8 @@ def test_retry_upload_to_tender_422(self, gevent_sleep): @patch('gevent.sleep') def test_retry_upload_to_tender_429(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.client.client_upload_to_tender = MagicMock(side_effect=[ResourceError(http_code=429), ResourceError(http_code=403)]) + self.client.client_upload_to_tender = MagicMock( + side_effect=[ResourceError(http_code=429), ResourceError(http_code=403)]) self.worker.retry_upload_to_tender_queue = Queue(10) self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) @@ -270,10 +270,11 @@ def test_retry_upload_to_tender_exception(self, gevent_sleep): def test_request_failed(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) self.client._create_tender_resource_item.side_effect = ResourceError(http_code=422) self.upload_to_doc_service_queue.put(self.data) @@ -291,13 +292,13 @@ def test_request_failed(self, mrequest, gevent_sleep): def test_request_failed_item_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) - self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=403), - ResourceError(http_code=403)] + self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=403) for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.qualification_id, 1) self.upload_to_doc_service_queue.put(self.data) self.upload_to_doc_service_queue.put(self.qualification_data) @@ -313,15 +314,9 @@ def test_request_failed_item_status_change(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_request_failed_in_retry(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep - self.worker = UploadFile.spawn(self.client, self.upload_to_doc_service_queue, self.upload_to_tender_queue, - self.process_tracker, self.doc_service_client, MagicMock(), self.sleep_change_value) self.worker.client_upload_to_tender = MagicMock() - self.worker.client_upload_to_tender.side_effect = [ResourceError(http_code=429), - ResourceError(http_code=429), - ResourceError(http_code=429), - ResourceError(http_code=429), - ResourceError(http_code=429), - ResourceError(http_code=403)] + self.worker.client_upload_to_tender.side_effect = [ResourceError(http_code=429) for _ in range(5)] + [ + ResourceError(http_code=403)] self.sleep_change_value.increment_step = 3 self.sleep_change_value.decrement_step = 1.5 self.worker.retry_upload_to_tender_queue.put(self.data) @@ -338,17 +333,14 @@ def test_request_failed_in_retry(self, gevent_sleep): def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) - self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=429), - ResourceError(http_code=403), - ResourceError(http_code=403), - ResourceError(http_code=403), - ResourceError(http_code=403) - ] + self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=429)] + [ + ResourceError(http_code=403) for _ in range(4)] self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') @@ -361,25 +353,26 @@ def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): def test_processing_items(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - [{'json': {'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}, - {'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) + [{'json': {'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + 'status_code': 200}, + {'json': {'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + 'status_code': 200}]) self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}, + {'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}}] self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.upload_to_doc_service_queue.put(self.data) self.upload_to_doc_service_queue.put(self.data) @@ -405,25 +398,19 @@ def test_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): 'format': 'application/yaml', 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', 'title': file_name}}, - 'status_code': 200}, - {'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) - self.client._create_tender_resource_item.side_effect = [ - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + 'status_code': 200}, + {'json': {'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + 'status_code': 200}]) + self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}} for _ in range(2)] self.worker.start() sleep(1) - self.worker.shutdown() self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) @@ -470,7 +457,7 @@ def test_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): default=LoopExit()) self.worker.start() sleep(1) - self.worker.shutdown() + self.shutdown_when_done(self.worker) self.assertEqual(self.process_tracker.processing_items, {}) self.assertIsNotNone(self.client.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone(self.client.request_history[1].headers['X-Client-Request-ID']) @@ -531,22 +518,11 @@ def test_retry_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_slee 'format': 'application/yaml', 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', 'title': file_name}}, - 'status_code': 200}, - {'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) - self.client._create_tender_resource_item.side_effect = [ - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + 'status_code': 200} for _ in range(2)]) + self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}} for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.worker.retry_upload_to_doc_service_queue = MagicMock() self.worker.retry_upload_to_doc_service_queue.peek.side_effect = generate_answers( @@ -565,10 +541,11 @@ def test_retry_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_slee def test_412(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, + json={'data': { + 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, status_code=200) api_server_bottle = Bottle() api_server = WSGIServer(('127.0.0.1', 20604), api_server_bottle, log=None) @@ -579,16 +556,19 @@ def test_412(self, mrequest, gevent_sleep): self.worker.client = TendersClientSync('', host_url='http://127.0.0.1:20604', api_version='2.3') setup_routing(api_server_bottle, generate_response, path='/api/2.3/tenders/{}/awards/{}/documents'.format( self.tender_id, self.award_id), method='POST') - self.assertEqual(self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies + self.assertEqual(self.worker.client.headers['Cookie'], + 'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.worker.start() self.shutdown_when_done(self.worker) - self.assertEqual(self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies + self.assertEqual(self.worker.client.headers['Cookie'], + 'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) # test that item removed from processing_items \ No newline at end of file + self.assertItemsEqual(self.process_tracker.processing_items.keys(), + []) # test that item removed from processing_items From 91b5bebab8a070b2c5d786fccb27ed6511356088 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Fri, 4 Aug 2017 15:50:57 +0300 Subject: [PATCH 2/8] Refactored tests --- .../identification/databridge/constants.py | 2 +- .../bot/identification/tests/edr_handler.py | 292 ++++++++---------- .../bot/identification/tests/filter_tender.py | 115 +++---- .../bot/identification/tests/scanner.py | 117 ++++--- .../bot/identification/tests/upload_file.py | 234 ++++---------- 5 files changed, 297 insertions(+), 463 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index b51eed3..f3e6b49 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,4 +4,4 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1000 +retry_mult = 1 diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index d56af7d..eac6f3d 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -36,7 +36,6 @@ def setUp(self): self.source_date = ["2017-04-25T11:56:36+00:00"] self.gen_req_id = [generate_request_id() for _ in xrange(10)] self.edrpou_codes_queue = Queue(10) - self.check_queue = Queue(10) self.edr_ids_queue = Queue(10) self.upload_to_doc_service_queue = Queue(10) self.filtered_tender_ids_queue = Queue(10) @@ -82,7 +81,7 @@ def stat_200(data_info, det_source_date, x_req_id): 'status_code': 200, 'headers': {'X-Request-ID': x_req_id}} @staticmethod - def file_con(data_info, doc_id, suf_1, suf_2, source_req): + def file_con(data_info={}, doc_id=generate_request_id(), suf_1=1, suf_2=1, source_req=generate_request_id()): if suf_1 == 1 and suf_2 == 1: return {'data': data_info, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", "id": "{}".format(doc_id), @@ -122,11 +121,11 @@ def test_proxy_client(self, mrequest, gevent_sleep): self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(2): - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[0], 1, 1, []))) # data + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(source_req=[]))) # data expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[i]]))) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(source_req=[self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(self.edr_ids)) @@ -144,11 +143,10 @@ def test_proxy_client_429(self, mrequest, gevent_sleep): self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(2): - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[i], self.edr_req_ids[i]]))) # result + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], source_req=[self.edr_req_ids[i], self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) @@ -170,10 +168,10 @@ def test_proxy_client_402(self, mrequest, gevent_sleep): mrequest.get(self.url_id(self.local_edr_ids[i]), [self.stat_200([{}], self.source_date, self.edr_req_ids[i])]) self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], source_req=[self.edr_req_ids[i], self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertIsNotNone(mrequest.request_history[3].headers['X-Client-Request-ID']) @@ -190,10 +188,10 @@ def test_retry_get_edr_id(self, mrequest, gevent_sleep): expected_result = [] for i in range(1): self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[0]]))) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[0]]))) self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result[0]) self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) @@ -208,7 +206,7 @@ def test_get_edr_data_empty_response(self, mrequest, gevent_sleep): "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date}}]}]}, status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {"error": {"errorDetails": "Couldn't find this code in EDR.", @@ -232,19 +230,19 @@ def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): self.stat_200([{"test": 1}, {"test": 2}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) expected_result = [] - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 1}, self.document_ids[0], 2, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 2}, self.document_ids[0], 2, 2, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - self.worker.upload_to_doc_service_queue = self.check_queue + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(data_info={"test": 1}, doc_id=self.document_ids[0], suf_1=2, suf_2=1, + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(data_info={"test": 2}, doc_id=self.document_ids[0], suf_1=2, suf_2=2, + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - self.assertEqual(self.check_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') @@ -256,19 +254,19 @@ def test_retry_get_edr_data_two_ids(self, mrequest, gevent_sleep): self.stat_200([{"test": 1}, {"test": 2}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) expected_result = [] - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 1}, self.document_ids[0], 2, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({"test": 2}, self.document_ids[0], 2, 2, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) - self.worker.upload_to_doc_service_queue = self.check_queue + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(data_info={"test": 1}, doc_id=self.document_ids[0], suf_1=2, suf_2=1, + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(data_info={"test": 2}, doc_id=self.document_ids[0], suf_1=2, suf_2=2, + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - self.assertEqual(self.check_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') @@ -277,16 +275,14 @@ def test_retry_get_edr_data_empty_response(self, mrequest, gevent_sleep): message 'EDRPOU not found'""" gevent_sleep.side_effect = custom_sleep mrequest.get(self.uri, - [self.stat_c(403, 0, '', self.edr_req_ids[0]), self.stat_c(403, 0, '', self.edr_req_ids[0]), - self.stat_c(403, 0, '', self.edr_req_ids[0]), self.stat_c(403, 0, '', self.edr_req_ids[0]), - self.stat_c(403, 0, '', self.edr_req_ids[0]), - {'json': { - 'errors': [ - {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"detailsSourceDate": self.source_date}}]}]}, - 'status_code': 404, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + [self.stat_c(403, 0, '', self.edr_req_ids[0]) for _ in range(5)] + [ + {'json': { + 'errors': [ + {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, + 'status_code': 404, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {"error": {"errorDetails": "Couldn't find this code in EDR.", @@ -307,18 +303,18 @@ def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep expected_result = [] - self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=403)), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, @@ -334,7 +330,7 @@ def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): 'author': author }}]}]})))]) for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') @@ -342,18 +338,18 @@ def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep expected_result = [] - self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, json=MagicMock(return_value= @@ -367,7 +363,7 @@ def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): 'id': self.document_ids[0], "version": version, 'author': author}}]}]})))]) for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') @@ -375,18 +371,18 @@ def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep expected_result = [] - self.worker.upload_to_doc_service_queue = self.check_queue + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() self.worker.retry_edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, "meta": {"detailsSourceDate": self.source_date, 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) self.worker.get_edr_data_request = MagicMock(side_effect=[ Exception(), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, @@ -401,7 +397,7 @@ def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): 'id': self.document_ids[0], "version": version, 'author': author}}]}]})))]) for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') @@ -413,11 +409,11 @@ def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): # data contains dict, instead of list self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) mrequest.get(self.url_id(321), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -432,12 +428,13 @@ def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): {'json': {'data': [{}]}, 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, self.stat_200([{}], self.source_date, self.edr_req_ids[2])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.process_tracker = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1], self.edr_req_ids[2]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1], + self.edr_req_ids[2]]))) self.assertEqual(mrequest.call_count, 3) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -450,19 +447,14 @@ def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): """Accept 6 times errors in response while requesting /verify""" gevent_sleep.side_effect = custom_sleep mrequest.get(self.url, - [self.stat_c(403, 0, '', self.edr_req_ids[0]), - self.stat_c(403, 0, '', self.edr_req_ids[1]), - self.stat_c(403, 0, '', self.edr_req_ids[2]), - self.stat_c(403, 0, '', self.edr_req_ids[3]), - self.stat_c(403, 0, '', self.edr_req_ids[4]), - self.stat_c(403, 0, '', self.edr_req_ids[5]), - self.stat_200([{}], self.source_date, self.edr_req_ids[6])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + [self.stat_c(403, 0, '', self.edr_req_ids[i]) for i in range(6)] + [ + self.stat_200([{}], self.source_date, self.edr_req_ids[6])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.process_tracker = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[6]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[6]]))) self.assertEqual(mrequest.call_count, 7) # processing 7 requests self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) # check first url @@ -480,11 +472,11 @@ def test_timeout(self, mrequest, gevent_sleep): mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0]), self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -540,16 +532,16 @@ def test_processing_items(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep award_key = item_key(self.tender_id, self.award_id) qualification_key = item_key(self.tender_id, self.qualification_id) - data_1 = Data(self.tender_id, self.award_id, '123', "awards", - self.file_con({}, self.document_ids[0], 2, 1, [self.edr_req_ids[0]])) - data_2 = Data(self.tender_id, self.award_id, '123', "awards", - self.file_con({}, self.document_ids[0], 2, 2, [self.edr_req_ids[0]])) - data_3 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', - self.file_con({}, self.document_ids[1], 3, 1, [self.edr_req_ids[1]])) - data_4 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', - self.file_con({}, self.document_ids[1], 3, 2, [self.edr_req_ids[1]])) - data_5 = Data(self.tender_id, self.qualification_id, '124', 'qualifications', - self.file_con({}, self.document_ids[1], 3, 3, [self.edr_req_ids[1]])) + data_1 = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], suf_1=2, suf_2=1, source_req=[self.edr_req_ids[0]])) + data_2 = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], suf_1=2, suf_2=2, source_req=[self.edr_req_ids[0]])) + data_3 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', + self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=1, source_req=[self.edr_req_ids[1]])) + data_4 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', + self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=2, source_req=[self.edr_req_ids[1]])) + data_5 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', + self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=3, source_req=[self.edr_req_ids[1]])) mrequest.get(self.url, [ self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[0]), self.stat_200([{}, {}, {}], @@ -557,10 +549,10 @@ def test_processing_items(self, mrequest, gevent_sleep): "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) # create queues - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, '123', "awards", + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {'meta': {'id': self.document_ids[0], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - self.edrpou_codes_queue.put(Data(self.tender_id, self.qualification_id, '124', 'qualifications', + self.edrpou_codes_queue.put(Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', {'meta': {'id': self.document_ids[1], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) @@ -580,11 +572,11 @@ def test_wrong_ip(self, mrequest, gevent_sleep): self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Forbidden'}], self.edr_req_ids[0]), self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -605,13 +597,13 @@ def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): expected_result = [] edrpou_codes_queue_list = [LoopExit()] for i in range(2): - edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {'meta': {'id': self.document_ids[i], 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[i], 1, 1, [self.edr_req_ids[i]]))) # result + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) self.worker.retry_edrpou_codes_queue = MagicMock() self.worker.process_tracker = MagicMock() @@ -634,13 +626,13 @@ def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): edrpou_codes_queue_list = [LoopExit()] expected_result = [] for i in range(2): - edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {"meta": {"id": self.document_ids[i], 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", - self.file_con({}, self.document_ids[i], 1, 1, [self.edr_req_ids[i]]))) # result + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) self.worker.retry_edrpou_codes_queue = MagicMock() self.worker.retry_edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) @@ -658,25 +650,21 @@ def test_retry_exception(self, mrequest, gevent_sleep): retry_response = MagicMock() retry_response.status_code = 500 mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'exc': RetryException('Retry Exception', retry_response)}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, - 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}]) + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ + {'exc': RetryException('Retry Exception', retry_response)}, + {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, + 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}]) expected_result = [] for i in range(1): self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - self.worker.upload_to_doc_service_queue = self.check_queue + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) @@ -695,31 +683,25 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, 'meta': {"detailsSourceDate": self.source_date}}]}]} mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'exc': RetryException('Retry Exception', retry_response)}]) + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ + {'exc': RetryException('Retry Exception', retry_response)}]) expected_result = [] for i in range(1): self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {'error': {'errorDetails': "Couldn't find this code in EDR.", 'code': 'notFound'}, 'meta': {'detailsSourceDate': self.source_date, 'id': self.document_ids[0], "version": version, 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - self.edr_req_ids[0]]}})) # result - - self.worker.upload_to_doc_service_queue = self.check_queue + self.edr_req_ids[0]]}})) + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() - for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.call_count, 6) @@ -730,23 +712,20 @@ def test_exception(self, mrequest, gevent_sleep): """ Raise Exception in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - {'exc': Exception()}, self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ + {'exc': Exception()}, self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(1): self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result - self.worker.upload_to_doc_service_queue = self.check_queue + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: - self.assertEquals(self.check_queue.get(), result) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) @@ -759,19 +738,17 @@ def test_value_error(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep mrequest.get(self.url, - [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0]), - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}, - self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) - self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0])] + [ + {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}} for _ in + range(5)] + [ + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.retry_edr_ids_queue = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.assertEqual(mrequest.call_count, 7) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @@ -785,7 +762,7 @@ def test_value_error_mock(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.worker.retry_edr_ids_queue = MagicMock() self.worker.retry_edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.get_edr_data_request = MagicMock(side_effect=[RetryException("Exception", Response()), MagicMock( headers={'X-Request-ID': self.edr_req_ids[0]}, @@ -796,7 +773,8 @@ def test_value_error_mock(self, mrequest, gevent_sleep): self.source_date}})))]) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, [self.edr_req_ids[0]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0]]))) @requests_mock.Mocker() @patch('gevent.sleep') @@ -805,7 +783,7 @@ def test_429_mock(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.worker.retry_edr_ids_queue = MagicMock() self.worker.retry_edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.get_edr_data_request = MagicMock( side_effect=[MagicMock(headers={'X-Request-ID': self.edr_req_ids[0], 'Retry-After': '1'}, status_code=429), MagicMock(headers={'X-Request-ID': self.edr_req_ids[1]}, status_code=200, @@ -813,5 +791,5 @@ def test_429_mock(self, mrequest, gevent_sleep): {'data': [{}], "meta": {"detailsSourceDate": self.source_date}})))]) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con({}, self.document_ids[0], 1, 1, - [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) diff --git a/openprocurement/bot/identification/tests/filter_tender.py b/openprocurement/bot/identification/tests/filter_tender.py index efaee7f..f5cc4f4 100644 --- a/openprocurement/bot/identification/tests/filter_tender.py +++ b/openprocurement/bot/identification/tests/filter_tender.py @@ -25,6 +25,7 @@ COOKIES_412 = ("b7afc9b1fc79e640f2487ba48243ca071c07a823d27" "8cf9b7adf0fae467a524747e3c6c6973262130fac2b" "96a11693fa8bd38623e4daee121f60b4301aef012c") +CODES = ('14360570', '0013823', '23494714') def setup_routing(app, func, path='/api/2.3/spore', method='GET'): @@ -55,7 +56,7 @@ def response_get_tender(): 'status': 'pending', 'suppliers': [{'identifier': { 'scheme': 'UA-EDR', - 'id': '14360570'}}]}]}}) + 'id': CODES[0]}}]}]}}) def generate_response(): @@ -76,7 +77,7 @@ def setUp(self): self.sleep_change_value = APIRateController() self.client = MagicMock() self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, - self.process_tracker, MagicMock(), self.sleep_change_value) + self.process_tracker, MagicMock(), self.sleep_change_value) self.bid_ids = [uuid.uuid4().hex for _ in range(5)] self.qualification_ids = [uuid.uuid4().hex for _ in range(5)] self.award_ids = [uuid.uuid4().hex for _ in range(5)] @@ -90,8 +91,8 @@ def awards(self, counter_id, counter_bid_id, status, sup_id): return {'id': self.award_ids[counter_id], 'bid_id': self.bid_ids[counter_bid_id], 'status': status, 'suppliers': [{'identifier': {'scheme': 'UA-EDR', 'id': sup_id}}]} - def bids(self, counter_id, ten_id): - return {'id': self.bid_ids[counter_id], 'tenderers': [{'identifier': {'scheme': 'UA-EDR', 'id': ten_id}}]} + def bids(self, counter_id, edr_id): + return {'id': self.bid_ids[counter_id], 'tenderers': [{'identifier': {'scheme': 'UA-EDR', 'id': edr_id}}]} def qualifications(self, status, counter_qual_id, counter_bid_id): return {'status': status, 'id': self.qualification_ids[counter_qual_id], 'bidID': self.bid_ids[counter_bid_id]} @@ -133,27 +134,23 @@ def test_worker_qualification(self, gevent_sleep): 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', - 'bids': [self.bids(0, '14360570'), - self.bids(1, '0013823'), - self.bids(2, '23494714'), - self.bids(3, '23494714'), + 'bids': [self.bids(0, CODES[0]), + self.bids(1, CODES[1]), + self.bids(2, CODES[2]), + self.bids(3, CODES[2]), {'id': self.bid_ids[4], 'tenderers': [{'identifier': { 'scheme': 'UA-ED', - 'id': '23494714'}}]}], + 'id': CODES[2]}}]}], 'qualifications': [ self.qualifications('pending', 0, 0), self.qualifications('pending', 1, 1), self.qualifications('pending', 2, 2), self.qualifications('unsuccessful', 3, 3), self.qualifications('pending', 4, 4)]}})) - first_data = Data(self.tender_id, self.qualification_ids[0], '14360570', 'qualifications', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - second_data = Data(self.tender_id, self.qualification_ids[1], '0013823', 'qualifications', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - third_data = Data(self.tender_id, self.qualification_ids[2], '23494714', 'qualifications', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - for data in [first_data, second_data, third_data]: + for i in range(3): + data = Data(self.tender_id, self.qualification_ids[i], CODES[i], 'qualifications', + {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.qualification_ids[i]) for i in range(3)]) @@ -168,25 +165,19 @@ def test_worker_award(self, gevent_sleep): 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [ - self.awards(0, 0, 'pending', '14360570'), - self.awards(1, 1, 'pending', '0013823'), - self.awards(2, 2, 'pending', '23494714'), + self.awards(0, 0, 'pending', CODES[0]), + self.awards(1, 1, 'pending', CODES[1]), + self.awards(2, 2, 'pending', CODES[2]), self.awards(3, 3, 'unsuccessful', - '23494714'), + CODES[2]), {'id': self.bid_ids[4], 'tenderers': [{'identifier': { 'scheme': 'UA-ED', - 'id': '23494714'}}]}]}}))] - first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + 'id': CODES[2]}}]}]}}))] + for i in range(3): + data = Data(self.tender_id, self.award_ids[i], CODES[i], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) - second_data = Data(self.tender_id, self.award_ids[1], '0013823', 'awards', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - third_data = Data(self.tender_id, self.award_ids[2], '23494714', 'awards', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - self.sleep_change_value.increment_step = 2 - self.sleep_change_value.decrement_step = 1 - for edrpou in [first_data, second_data, third_data]: - self.check_data_objects(self.edrpou_codes_queue.get(), edrpou) + self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0]), item_key(self.tender_id, self.award_ids[1]), item_key(self.tender_id, self.award_ids[2])]) @@ -203,8 +194,8 @@ def test_get_tender_exception(self, gevent_sleep): 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(0, 0, 'pending', - '14360570')]}}))] - data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + CODES[0])]}}))] + data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0) @@ -225,8 +216,8 @@ def test_get_tender_429(self, gevent_sleep): 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', - 'awards': [self.awards(0, 0, 'pending', '14360570')]}}))] - data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + 'awards': [self.awards(0, 0, 'pending', CODES[0])]}}))] + data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 @@ -251,9 +242,9 @@ def test_worker_restart(self, gevent_sleep): 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', - 'awards': [self.awards(0, 0, 'pending', '14360570'), - self.awards(1, 1, 'unsuccessful', '23494714')]}}))] - data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + 'awards': [self.awards(0, 0, 'pending', CODES[0]), + self.awards(1, 1, 'unsuccessful', CODES[2])]}}))] + data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), @@ -265,31 +256,19 @@ def test_worker_dead(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.filtered_tender_ids_queue.put(self.tender_id) self.client.request.side_effect = [ - ResponseMock({'X-Request-ID': self.request_ids[0]}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': { - 'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [self.awards(0, 0, 'pending', '14360570')]}})), - ResponseMock({'X-Request-ID': self.request_ids[1]}, - munchify( - {'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': { - 'status': "active.pre-qualification", - 'id': self.tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [self.awards(1, 1, 'pending', '14360570')]}}))] - first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', - {'meta': {'sourceRequests': [self.request_ids[0]]}}) - second_data = Data(self.tender_id, self.award_ids[1], '14360570', 'awards', - {'meta': {'sourceRequests': [self.request_ids[1]]}}) - self.check_data_objects(self.edrpou_codes_queue.get(), first_data) + ResponseMock({'X-Request-ID': self.request_ids[i]}, + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': { + 'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [self.awards(i, i, 'pending', CODES[0])]}})) for i in range(2)] + for i in range(2): + data = Data(self.tender_id, self.award_ids[i], CODES[0], 'awards', + {'meta': {'sourceRequests': [self.request_ids[i]]}}) + self.check_data_objects(self.edrpou_codes_queue.get(), data) self.worker.job.kill(timeout=1) - self.check_data_objects(self.edrpou_codes_queue.get(), second_data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[i]) for i in range(2)]) @@ -307,8 +286,8 @@ def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep): 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [ - self.awards(0, 0, 'pending', '14360570')]}})) - first_data = Data(self.tender_id, self.award_ids[0], '14360570', 'awards', + self.awards(0, 0, 'pending', CODES[0])]}})) + first_data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue self.check_data_objects(self.edrpou_codes_queue.get(), first_data) @@ -333,16 +312,16 @@ def test_worker_award_with_cancelled_lot(self, gevent_sleep): 'status': 'pending', 'suppliers': [{'identifier': { 'scheme': 'UA-EDR', - 'id': '14360570'}}], + 'id': CODES[0]}}], 'lotID': '123456789'}, {'id': self.award_ids[1], 'bid_id': self.bid_ids[1], 'status': 'pending', 'suppliers': [{'identifier': { 'scheme': 'UA-EDR', - 'id': '0013823'}}], + 'id': CODES[1]}}], 'lotID': '12345678'}]}})) - data = Data(self.tender_id, self.award_ids[1], '0013823', 'awards', + data = Data(self.tender_id, self.award_ids[1], CODES[1], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) for edrpou in [data]: self.check_data_objects(self.edrpou_codes_queue.get(), edrpou) @@ -406,15 +385,13 @@ def test_412(self, gevent_sleep): 'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies worker = FilterTenders.spawn(client, filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) - data = Data('123', '124', '14360570', 'awards', {'meta': {'sourceRequests': ['125']}}) - + data = Data('123', '124', CODES[0], 'awards', {'meta': {'sourceRequests': ['125']}}) for i in [data]: self.check_data_objects(self.edrpou_codes_queue.get(), i) self.assertEqual(client.headers['Cookie'], 'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies self.assertEqual(self.edrpou_codes_queue.qsize(), 0) self.assertItemsEqual(self.process_tracker.processing_items.keys(), ['123_124']) - worker.shutdown() del worker api_server.stop() diff --git a/openprocurement/bot/identification/tests/scanner.py b/openprocurement/bot/identification/tests/scanner.py index c73047b..681d4ad 100644 --- a/openprocurement/bot/identification/tests/scanner.py +++ b/openprocurement/bot/identification/tests/scanner.py @@ -11,7 +11,7 @@ from mock import patch, MagicMock from time import sleep from munch import munchify -from restkit.errors import (Unauthorized, RequestFailed, ResourceError) +from restkit.errors import Unauthorized, RequestFailed, ResourceError from openprocurement.bot.identification.databridge.scanner import Scanner from openprocurement.bot.identification.tests.utils import custom_sleep @@ -34,16 +34,16 @@ def tearDown(self): del self.worker @staticmethod - def munchify(prev_p, next_p, status, id, procurementMethodType, data=True): + def munchify(status, id, procurementMethodType, data=True): if data: - return munchify({'prev_page': {'offset': prev_p}, - 'next_page': {'offset': next_p}, + return munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, 'data': [{'status': status, "id": id, 'procurementMethodType': 'aboveThreshold{}'.format(procurementMethodType)}]}) else: - return munchify({'prev_page': {'offset': prev_p}, - 'next_page': {'offset': next_p}, + return munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, 'data': []}) def test_init(self): @@ -58,13 +58,12 @@ def test_init(self): def test_worker(self, gevent_sleep): """ Returns tenders, check queue elements after filtering """ gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - RequestFailed(), - # worker must restart - self.munchify('123', '1234', "active.qualification", self.tenders_id[0], 'UA'), - Unauthorized(), - self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU')] + self.client.sync_tenders.side_effect = [RequestFailed(), + # worker must restart + self.munchify("active.qualification", self.tenders_id[0], 'UA'), + Unauthorized(), + self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] for tender_id in self.tenders_id[0:2]: self.assertEqual(self.tender_queue.get(), tender_id) @@ -72,12 +71,11 @@ def test_worker(self, gevent_sleep): def test_429(self, gevent_sleep): """Receive 429 status, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify('1235', '1236', "active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify('1236', '1237', "active.qualification", self.tenders_id[1], 'UA'), - ResourceError(http_code=429), - self.munchify('1237', '1238', "active.qualification", self.tenders_id[2], 'UA')] + self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify("active.qualification", self.tenders_id[1], 'UA'), + ResourceError(http_code=429), + self.munchify("active.qualification", self.tenders_id[2], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 for tender_id in self.tenders_id[0:3]: @@ -88,15 +86,14 @@ def test_429(self, gevent_sleep): def test_429_sleep_change_value(self, gevent_sleep): """Three times receive 429, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify('123', '1234', "active.tendering", uuid.uuid4().hex, 'UA'), - ResourceError(http_code=429), - ResourceError(http_code=429), - ResourceError(http_code=429), - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU')] + self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + ResourceError(http_code=429), + ResourceError(http_code=429), + ResourceError(http_code=429), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:2]: @@ -107,12 +104,11 @@ def test_429_sleep_change_value(self, gevent_sleep): def test_backward_dead(self, gevent_sleep): """Test when backward dies """ gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU'), - ResourceError(http_code=403), - self.munchify('1235', '1236', None, None, False), - self.munchify('1236', '1237', "active.pre-qualification", self.tenders_id[2], 'EU')] + self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), + ResourceError(http_code=403), + self.munchify(None, None, False), + self.munchify("active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:3]: @@ -122,12 +118,11 @@ def test_backward_dead(self, gevent_sleep): def test_forward_dead(self, gevent_sleep): """ Test when forward dies""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('1234', '123', None, None, None, False), - self.munchify('1234', '123', None, None, None, False), - self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[0], 'EU'), - ResourceError(http_code=403), - self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU')] + self.client.sync_tenders.side_effect = [self.munchify(None, None, None, False), + self.munchify(None, None, None, False), + self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + ResourceError(http_code=403), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:2]: @@ -138,11 +133,10 @@ def test_forward_run(self, gevent_sleep): """ Run forward when backward get empty response and prev_page.offset is equal to next_page.offset """ gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify('1234', '1235', "active.pre-qualification", self.tenders_id[1], 'EU'), - self.munchify('1235', '1235', None, None, None, False), - self.munchify('1236', '1237', "active.pre-qualification", self.tenders_id[2], 'EU')] + self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), + self.munchify(None, None, None, False), + self.munchify("active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:3]: @@ -153,14 +147,13 @@ def test_get_tenders_exception(self, gevent_sleep): """ Catch exception in backward worker and after that put 2 tenders to process.Then catch exception for forward and after that put tender to process.""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify('123', '1234', None, None, None, False), - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[1], 'EU'), - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[2], 'EU'), - self.munchify('123', '1234', None, None, None, False), - self.munchify('1234', None, None, None, None, False), - self.munchify('1234', '1234', "active.pre-qualification", self.tenders_id[3], 'EU')] + self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.munchify(None, None, None, False), + self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), + self.munchify("active.pre-qualification", self.tenders_id[2], 'EU'), + self.munchify(None, None, None, False), + self.munchify(None, None, None, False), + self.munchify("active.pre-qualification", self.tenders_id[3], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id: @@ -170,9 +163,8 @@ def test_get_tenders_exception(self, gevent_sleep): def test_resource_error(self, gevent_sleep): """Raise Resource error, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [ - ResourceError(http_code=429), - self.munchify('1237', '1238', "active.qualification", self.tenders_id[0], 'UA')] + self.client.sync_tenders.side_effect = [ResourceError(http_code=429), + self.munchify("active.qualification", self.tenders_id[0], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) @@ -184,12 +176,13 @@ def test_kill_jobs_with_exception(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 - worker = Scanner.spawn(MagicMock(), MagicMock(), MagicMock(), self.process_tracker, self.sleep_change_value) + self.worker.tenders_sync_client = MagicMock() + self.worker.filtered_tender_ids_queue = MagicMock() sleep(1) - for job in worker.jobs: + for job in self.worker.jobs: job.kill(exception=Exception) sleep(4) - self.assertFalse(worker.ready()) + self.assertFalse(self.worker.ready()) @patch('gevent.sleep') def test_forward_exception(self, gevent_sleep): @@ -198,8 +191,8 @@ def test_forward_exception(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 - self.worker.initialize_sync = MagicMock(side_effect=[ - ResourceError(msg=RequestFailed()), - self.munchify('123', '1234', "active.pre-qualification", self.tenders_id[0], 'EU')]) + self.worker.initialize_sync = MagicMock(side_effect=[ResourceError(msg=RequestFailed()), + self.munchify("active.pre-qualification", + self.tenders_id[0], 'EU')]) self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) self.assertEqual(self.worker.initialize_sync.call_count, 2) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 83b3413..64e4442 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -87,6 +87,22 @@ def setUp(self): self.worker = UploadFile(self.client, self.upload_to_doc_service_queue, self.upload_to_tender_queue, self.process_tracker, self.doc_service_client, MagicMock(), self.sleep_change_value) + self.url = '{url}'.format(url=self.doc_service_client.url) + + @staticmethod + def stat_200(): + return {'data': {'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}} + + @staticmethod + def get_tender(): + return {'data': {'id': uuid.uuid4().hex, + 'documentOf': 'tender', + 'documentType': 'registerExtract', + 'url': 'url'}} + def tearDown(self): del self.worker @@ -112,7 +128,6 @@ def test_init(self): self.assertEqual(worker.sleep_change_value, None) self.assertEqual(worker.delay, 15) self.assertEqual(worker.exit, False) - worker.shutdown() self.assertEqual(worker.exit, True) del worker @@ -121,29 +136,18 @@ def test_init(self): @patch('gevent.sleep') def test_successful_upload(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) - self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + mrequest.post(self.url, json=self.stat_200(), status_code=200) + self.client._create_tender_resource_item.side_effect = [self.get_tender()] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.shutdown_when_done(self.worker) - self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), - []) # test that item removed from processing_items + self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) self.assertEqual(self.client._create_tender_resource_item.call_count, 1) # check upload to tender @requests_mock.Mocker() @@ -151,17 +155,13 @@ def test_successful_upload(self, mrequest, gevent_sleep): def test_retry_doc_service(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') - mrequest.post('{url}'.format(url=doc_service_client.url), - [{'text': '', 'status_code': 401} for _ in range(6)] + [ - {'json': {'data': {'url': 'test url', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) - self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + mrequest.post(self.url, [{'text': '', 'status_code': 401} for _ in range(6)] + [ + {'json': {'data': {'url': 'test url', + 'format': 'application/yaml', + 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', + 'title': file_name}}, + 'status_code': 200}]) + self.client._create_tender_resource_item.side_effect = [self.get_tender()] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) @@ -209,20 +209,10 @@ def test_upload_to_tender_exception_status_int_none(self, gevent_sleep): @patch('gevent.sleep') def test_retry_upload_to_tender(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) + mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = [Unauthorized(http_code=401), Unauthorized(http_code=403), - Unauthorized(http_code=429), - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + Unauthorized(http_code=429), self.get_tender()] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) @@ -269,13 +259,7 @@ def test_retry_upload_to_tender_exception(self, gevent_sleep): @patch('gevent.sleep') def test_request_failed(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) + mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = ResourceError(http_code=422) self.upload_to_doc_service_queue.put(self.data) self.shutdown_when_done(self.worker) @@ -291,13 +275,7 @@ def test_request_failed(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_request_failed_item_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) + mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=403) for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.qualification_id, 1) self.upload_to_doc_service_queue.put(self.data) @@ -332,13 +310,7 @@ def test_request_failed_in_retry(self, gevent_sleep): @patch('gevent.sleep') def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) + mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = [ResourceError(http_code=429)] + [ ResourceError(http_code=403) for _ in range(4)] self.worker.retry_upload_to_tender_queue.put(self.data) @@ -352,27 +324,8 @@ def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_processing_items(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - [{'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}, - {'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) - self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + mrequest.post(self.url, [{'json': self.stat_200(), 'status_code': 200} for _ in range(2)]) + self.client._create_tender_resource_item.side_effect = [self.get_tender() for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.upload_to_doc_service_queue.put(self.data) self.upload_to_doc_service_queue.put(self.data) @@ -392,23 +345,8 @@ def test_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): self.worker.upload_to_doc_service_queue = MagicMock() self.worker.upload_to_doc_service_queue.peek.side_effect = generate_answers( answers=[LoopExit(), self.data, self.data], default=LoopExit()) - mrequest.post('{url}'.format(url=self.doc_service_client.url), - [{'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}, - {'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200}]) - self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}} for _ in range(2)] + mrequest.post(self.url, [{'json': self.stat_200(), 'status_code': 200} for _ in range(2)]) + self.client._create_tender_resource_item.side_effect = [self.get_tender() for _ in range(2)] self.worker.start() sleep(1) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') @@ -422,42 +360,24 @@ def test_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): def test_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for upload_to_tender_queue """ gevent_sleep.side_effect = custom_sleep - self.client._create_tender_resource_item.side_effect = [ - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + self.client._create_tender_resource_item.side_effect = [self.get_tender() for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.worker.upload_to_doc_service_queue = Queue(1) self.worker.upload_to_tender_queue = MagicMock() self.worker.upload_to_tender_queue.peek.side_effect = generate_answers( - answers=[LoopExit(), - Data(tender_id=self.tender_id, - item_id=self.award_id, - code='123', item_name='awards', - file_content={ - u'meta': {u'id': self.document_id}, - u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - u'format': u'application/yaml', - u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', - u'title': file_name}), - Data(tender_id=self.tender_id, - item_id=self.award_id, - code='123', item_name='awards', - file_content={ - u'meta': {u'id': self.document_id}, - u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - u'format': u'application/yaml', - u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', - u'title': file_name})], + answers=[LoopExit()] + [Data(tender_id=self.tender_id, + item_id=self.award_id, + code='123', item_name='awards', + file_content={ + u'meta': {u'id': self.document_id}, + u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + u'format': u'application/yaml', + u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', + u'title': file_name}) for _ in range(2)], default=LoopExit()) self.worker.start() sleep(1) - self.shutdown_when_done(self.worker) + self.worker.shutdown() self.assertEqual(self.process_tracker.processing_items, {}) self.assertIsNotNone(self.client.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone(self.client.request_history[1].headers['X-Client-Request-ID']) @@ -467,36 +387,18 @@ def test_upload_to_tender_queue_loop_exit(self, mrequest, gevent_sleep): def test_retry_upload_to_tender_queue_loop_exit(self, gevent_sleep): """ Test LoopExit for retry_upload_to_tender_queue """ gevent_sleep.side_effect = custom_sleep - self.client._create_tender_resource_item.side_effect = [ - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}, - {'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}}] + self.client._create_tender_resource_item.side_effect = [self.get_tender() for _ in range(2)] self.worker.retry_upload_to_tender_queue = MagicMock() self.worker.retry_upload_to_tender_queue.peek.side_effect = generate_answers( - answers=[LoopExit(), - Data(tender_id=self.tender_id, - item_id=self.award_id, - code='123', item_name='awards', - file_content={ - u'meta': {u'id': self.document_id}, - u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - u'format': u'application/yaml', - u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', - u'title': file_name}), - Data(tender_id=self.tender_id, - item_id=self.award_id, - code='123', item_name='awards', - file_content={ - u'meta': {u'id': self.document_id}, - u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - u'format': u'application/yaml', - u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', - u'title': file_name})], + answers=[LoopExit()] + [Data(tender_id=self.tender_id, + item_id=self.award_id, + code='123', item_name='awards', + file_content={ + u'meta': {u'id': self.document_id}, + u'url': u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', + u'format': u'application/yaml', + u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555', + u'title': file_name}) for _ in range(2)], default=LoopExit()) self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.worker.start() @@ -512,17 +414,8 @@ def test_retry_upload_to_tender_queue_loop_exit(self, gevent_sleep): def test_retry_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for retry_upload_to_doc_service_queue """ gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - [{'json': {'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - 'status_code': 200} for _ in range(2)]) - self.client._create_tender_resource_item.side_effect = [{'data': {'id': uuid.uuid4().hex, - 'documentOf': 'tender', - 'documentType': 'registerExtract', - 'url': 'url'}} for _ in range(2)] + mrequest.post(self.url, [{'json': self.stat_200(), 'status_code': 200} for _ in range(2)]) + self.client._create_tender_resource_item.side_effect = [self.get_tender() for _ in range(2)] self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.worker.retry_upload_to_doc_service_queue = MagicMock() self.worker.retry_upload_to_doc_service_queue.peek.side_effect = generate_answers( @@ -540,13 +433,7 @@ def test_retry_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_slee @patch('gevent.sleep') def test_412(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - mrequest.post('{url}'.format(url=self.doc_service_client.url), - json={'data': { - 'url': 'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f', - 'format': 'application/yaml', - 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', - 'title': file_name}}, - status_code=200) + mrequest.post(self.url, json=self.stat_200(), status_code=200) api_server_bottle = Bottle() api_server = WSGIServer(('127.0.0.1', 20604), api_server_bottle, log=None) setup_routing(api_server_bottle, response_spore) @@ -570,5 +457,4 @@ def test_412(self, mrequest, gevent_sleep): self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertItemsEqual(self.process_tracker.processing_items.keys(), - []) # test that item removed from processing_items + self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) From 7fb6cfd22bbd03126ba62ad39d76926f4b8d7a72 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Fri, 4 Aug 2017 16:06:32 +0300 Subject: [PATCH 3/8] Refactored tests --- openprocurement/bot/identification/databridge/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index f3e6b49..b51eed3 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,4 +4,4 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1 +retry_mult = 1000 From 93e1b7f3d73765c0070f49542d05ae20f5070387 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Mon, 7 Aug 2017 12:15:42 +0300 Subject: [PATCH 4/8] refactored upload_file --- .../databridge/journal_msg_ids.py | 7 +- .../identification/databridge/upload_file.py | 234 ++++++++---------- 2 files changed, 109 insertions(+), 132 deletions(-) diff --git a/openprocurement/bot/identification/databridge/journal_msg_ids.py b/openprocurement/bot/identification/databridge/journal_msg_ids.py index 3e0e4e0..06e8279 100644 --- a/openprocurement/bot/identification/databridge/journal_msg_ids.py +++ b/openprocurement/bot/identification/databridge/journal_msg_ids.py @@ -17,10 +17,8 @@ DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE= 'edr_databridge_success_upload_to_doc_service' DATABRIDGE_RESTART_WORKER = 'edr_databridge_restart_worker' DATABRIDGE_PROCESSING_TENDER = 'edr_databridge_processing_tender' -DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE = 'edr_databridge_unsuccess_upload_to_doc_service' -DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE = 'edr_databridge_unsuccess_retry_upload_to_doc_service' -DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER = 'edr_databridge_unsuccess_upload_to_tender' -DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER = 'edr_databridge_unsuccess_retry_upload_to_tender' +DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE = 'edr_databridge_unsuccess{}_upload_to_doc_service' +DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER = 'edr_databridge_unsuccess{}_upload_to_tender' DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER = 'edr_databridge_success_upload_to_tender' DATABRIDGE_RESTART_UPLOAD_TO_DOC_SERVICE = 'edr_databridge_restart_upload_to_doc_service' DATABRIDGE_START_UPLOAD = 'edr_databridge_start_upload' @@ -39,5 +37,4 @@ DATABRIDGE_DOC_SERVICE_CONN_ERROR = 'edr_databridge_doc_service_conn_error' DATABRIDGE_TENDERS_SERVER_CONN_ERROR = 'edr_databridge_tenders_server_conn_error' DATABRIDGE_PROXY_SERVER_CONN_ERROR = 'edr_databridge_proxy_server_conn_error' -DATABRIDGE_422_UPLOAD_TO_TENDER = 'edr_databridge_422_upload_to_tender' DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING = 'edr_databridge_item_status_changed_while_processing' diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 0f60725..16b224e 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -12,10 +12,9 @@ from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, data_string from openprocurement.bot.identification.databridge.journal_msg_ids import ( - DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, - DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, - DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, - DATABRIDGE_422_UPLOAD_TO_TENDER, DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) + DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, + DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, + DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) from openprocurement.bot.identification.databridge.constants import file_name, retry_mult logger = logging.getLogger(__name__) @@ -25,9 +24,12 @@ class UploadFile(Greenlet): """ Upload file with details """ pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') - qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') + qualification_procurementMethodType = ( + 'aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', + 'competitiveDialogueEU.stage2') - def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, process_tracker, doc_service_client, services_not_available, sleep_change_value, delay=15): + def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, process_tracker, doc_service_client, + services_not_available, sleep_change_value, delay=15): super(UploadFile, self).__init__() self.exit = False self.start_time = datetime.now() @@ -51,6 +53,67 @@ def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, # blockers self.services_not_available = services_not_available + def resource_errors_403_422_none(self, retry, re, tender_data, document_id, item_name_id): + logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( + re.status_int, data_string(tender_data), document_id, re.msg), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) + self.sleep_change_value.decrement() + eval('self.{}upload_to_tender_queue.get()'.format(retry)) + + def resource_error_429(self, re, tender_data, document_id, item_name_id): + logger.info("Accept 429 while uploading to tender {} {} {} doc_id: {}. Message {}".format( + tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + self.sleep_change_value.increment() + + def resource_errors_else(self, retry, re, tender_data, document_id, item_name_id): + logger.info( + 'ResourceError while{} uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format( + '' if retry == '' else ' ' + retry, data_string(tender_data), document_id, re.status_int, re.message), + extra=journal_context( + {"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER.format('' if retry == '' else '_' + retry)}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + self.sleep_change_value.decrement() + + def exception_errors(self, retry, e, tender_data, document_id, item_name_id): + logger.info('Exception while{} uploading file to {} doc_id: {}. Message: {}'.format( + '' if retry == '' else ' ' + retry, data_string(tender_data), document_id, e.message), + extra=journal_context( + {"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER.format('' if retry == '' else '_' + retry)}, + params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + logger.exception("Message: {}".format(e.message)) + self.sleep_change_value.decrement() + + def succesfully_uploaded_to_tender(self, retry, tender_data, document_id, item_name_id): + logger.info('Successfully uploaded file to {} doc_id: {} in retry'.format( + data_string(tender_data), document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER}, + params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + # delete current tender after successful upload file (to avoid reloading file) + self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) + eval('self.{}upload_to_tender_queue.get()'.format(retry)) + self.sleep_change_value.decrement() + + def st_code_200(self, retry, tender_data, response, document_id, item_name_id): + data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, + tender_data.item_name, dict(response.json(), **{'meta': {'id': document_id}})) + self.upload_to_tender_queue.put(data) + eval('self.{}upload_to_doc_service_queue.get()'.format(retry)) + logger.info('Successfully uploaded file to doc service {} doc_id: {}'.format( + data_string(tender_data), document_id), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + def upload_to_doc_service(self): """Get data from upload_to_doc_service_queue; Create file of the Data.file_content data; If upload successful put Data object to upload_file_to_tender, otherwise put Data to retry_upload_file_queue.""" @@ -64,34 +127,29 @@ def upload_to_doc_service(self): continue item_name_id = tender_data.item_name[:-1].upper() + "_ID" try: - response = self.doc_service_client.upload(file_name, create_file(tender_data.file_content), 'application/yaml', + response = self.doc_service_client.upload(file_name, create_file(tender_data.file_content), + 'application/yaml', headers={'X-Client-Request-ID': document_id}) except Exception as e: logger.warning('Exception while uploading file to doc service {}. Message: {}. ' 'Put tender_data to retry queue '.format(data_string(tender_data), e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE}, params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) logger.exception("Message: {}".format(e.message)) self.retry_upload_to_doc_service_queue.put(tender_data) self.upload_to_doc_service_queue.get() else: if response.status_code == 200: - data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, - tender_data.item_name, dict(response.json(), **{'meta': {'id': document_id}})) - self.upload_to_tender_queue.put(data) - self.upload_to_doc_service_queue.get() - logger.info('Successfully uploaded file to doc service {} doc_id: {}'.format( - data_string(tender_data), document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) + self.st_code_200('', tender_data, response, document_id, item_name_id) else: - logger.info('Not successful response from document service while uploading {} doc_id: {}. Response {}'. - format(data_string(tender_data), document_id, response.status_code), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE}, - params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + logger.info( + 'Not successful response from document service while uploading {} doc_id: {}. Response {}'. + format(data_string(tender_data), document_id, response.status_code), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) self.retry_upload_to_doc_service_queue.put(tender_data) self.upload_to_doc_service_queue.get() gevent.sleep(0) @@ -117,35 +175,32 @@ def retry_upload_to_doc_service(self): 'Lost tender_data'.format(data_string(tender_data), document_id, e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE}, params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) logger.exception("Message: {}".format(e.message)) self.retry_upload_to_doc_service_queue.get() self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) raise e else: if response.status_code == 200: - data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, - tender_data.item_name, dict(response.json(), **{'meta': {'id': document_id}})) - self.upload_to_tender_queue.put(data) - self.retry_upload_to_doc_service_queue.get() - logger.info('Successfully uploaded file to doc service {} doc_id: {}'.format( - data_string(tender_data), document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) + self.st_code_200('retry_', tender_data, response, document_id, item_name_id) else: - logger.info('Not successful response in retry from document service while uploading {} {} {} {}. Response {}'. - format(tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, response.status_code), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_DOC_SERVICE}, - params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + logger.info( + 'Not successful response in retry from document service while uploading {} {} {} {}. Response {}'. + format(tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, + response.status_code), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE.format('_retry')}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) gevent.sleep(0) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult) def client_upload_to_doc_service(self, tender_data): """Process upload request for retry queue objects.""" return self.doc_service_client.upload(file_name, create_file(tender_data.file_content), 'application/yaml', - headers={'X-Client-Request-ID': tender_data.file_content.get('meta', {}).get('id')}) + headers={ + 'X-Client-Request-ID': tender_data.file_content.get('meta', {}).get( + 'id')}) def upload_to_tender(self): """Get data from upload_to_tender_queue; Upload get_Url and documentType; @@ -169,60 +224,20 @@ def upload_to_tender(self): '{}/{}/documents'.format(tender_data.item_name, tender_data.item_id)) except ResourceError as re: - if re.status_int == 422: # WARNING and don't retry - logger.warning("Accept 422, skip {} doc_id: {}. Message: {}".format(data_string(tender_data), document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.upload_to_tender_queue.get() - self.sleep_change_value.decrement() - continue - elif re.status_int == 403 or re.status_int is None: - logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( - re.status_int, data_string(tender_data), document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id}) - ) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.sleep_change_value.decrement() - self.upload_to_tender_queue.get() + if re.status_int == 403 or re.status_int == 422 or re.status_int is None: + self.resource_errors_403_422_none('', re, tender_data, document_id, item_name_id) elif re.status_int == 429: - logger.info("Accept 429 while uploading to tender {} {} {} doc_id: {}. Message {}".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) - self.sleep_change_value.increment() + self.resource_error_429(re, tender_data, document_id, item_name_id) else: - logger.warning('ResourceError while uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, re.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) + self.resource_errors_else('', re, tender_data, document_id, item_name_id) self.retry_upload_to_tender_queue.put(tender_data) self.upload_to_tender_queue.get() - self.sleep_change_value.decrement() except Exception as e: - logger.info('Exception while uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, e.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) + self.exception_errors('', e, tender_data, document_id, item_name_id) self.retry_upload_to_tender_queue.put(tender_data) self.upload_to_tender_queue.get() - self.sleep_change_value.decrement() else: - logger.info('Successfully uploaded file to {} doc_id: {}'.format( - data_string(tender_data), document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) - # delete current tender after successful upload file (to avoid reloading file) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.upload_to_tender_queue.get() - self.sleep_change_value.decrement() + self.succesfully_uploaded_to_tender('', tender_data, document_id, item_name_id) gevent.sleep(self.sleep_change_value.time_between_requests) def retry_upload_to_tender(self): @@ -241,53 +256,17 @@ def retry_upload_to_tender(self): try: self.client_upload_to_tender(tender_data) except ResourceError as re: - if re.status_int == 422: # WARNING and don't retry - logger.warning("Accept 422, skip {} doc_id: {}. Message {}".format(data_string(tender_data), document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.retry_upload_to_tender_queue.get() - self.sleep_change_value.decrement() + if re.status_int == 403 or re.status_int == 422 or re.status_int is None: + self.resource_errors_403_422_none('retry_', re, tender_data, document_id, item_name_id) continue elif re.status_int == 429: - logger.info("Accept 429 while uploading to tender {} {} {} doc_id: {}. Message {}".format( - tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) - self.sleep_change_value.increment() - elif re.status_int == 403 or re.status_int is None: - logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( - re.status_int, data_string(tender_data), document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id}) - ) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.retry_upload_to_tender_queue.get() - self.sleep_change_value.decrement() - continue + self.resource_error_429(re, tender_data, document_id, item_name_id) else: - logger.info('ResourceError while retry uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format( - data_string(tender_data), document_id, re.status_int, re.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) - self.sleep_change_value.decrement() + self.resource_errors_else('retry', re, tender_data, document_id, item_name_id) except Exception as e: - logger.info('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, e.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) - logger.exception("Message: {}".format(e.message)) - self.sleep_change_value.decrement() + self.exception_errors('retry', e, tender_data, document_id, item_name_id) else: - logger.info('Successfully uploaded file to {} doc_id: {} in retry'.format( - data_string(tender_data), document_id), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) - # delete current tender after successful upload file (to avoid reloading file) - self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - self.retry_upload_to_tender_queue.get() - self.sleep_change_value.decrement() + self.succesfully_uploaded_to_tender('retry_', tender_data, document_id, item_name_id) gevent.sleep(self.sleep_change_value.time_between_requests) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult) @@ -313,7 +292,8 @@ def _run(self): gevent.sleep(self.delay) for name, job in self.immortal_jobs.items(): if job.dead: - logger.warning("{} worker dead try restart".format(name), extra=journal_context({"MESSAGE_ID": 'DATABRIDGE_RESTART_{}'.format(name.lower())}, {})) + logger.warning("{} worker dead try restart".format(name), extra=journal_context( + {"MESSAGE_ID": 'DATABRIDGE_RESTART_{}'.format(name.lower())}, {})) self.immortal_jobs[name] = gevent.spawn(getattr(self, name)) logger.info("{} worker is up".format(name)) From 4ce0c6c045ed804140d18c924c223dfb7d2c80d3 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Tue, 8 Aug 2017 10:41:59 +0300 Subject: [PATCH 5/8] =?UTF-8?q?=D0=A1orrection=20of=20inaccuracies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../identification/databridge/constants.py | 5 +- .../bot/identification/databridge/scanner.py | 44 ++++++------ .../identification/databridge/upload_file.py | 54 +++++++------- .../bot/identification/tests/edr_handler.py | 29 ++++---- .../bot/identification/tests/scanner.py | 70 +++++++++---------- .../bot/identification/tests/upload_file.py | 1 - 6 files changed, 104 insertions(+), 99 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index b51eed3..ae24506 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,4 +4,7 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1000 +retry_mult = 1 +pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') +qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', + 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') diff --git a/openprocurement/bot/identification/databridge/scanner.py b/openprocurement/bot/identification/databridge/scanner.py index 237b48e..e537a8a 100644 --- a/openprocurement/bot/identification/databridge/scanner.py +++ b/openprocurement/bot/identification/databridge/scanner.py @@ -5,16 +5,14 @@ import gevent from gevent.event import Event from gevent import Greenlet, spawn -from openprocurement.bot.identification.databridge.constants import retry_mult +from openprocurement.bot.identification.databridge.constants import retry_mult, \ + pre_qualification_procurementMethodType, qualification_procurementMethodType from retrying import retry from restkit import ResourceError -from openprocurement.bot.identification.databridge.journal_msg_ids import ( - DATABRIDGE_INFO, DATABRIDGE_SYNC_SLEEP, DATABRIDGE_TENDER_PROCESS, - DATABRIDGE_WORKER_DIED, DATABRIDGE_RESTART, DATABRIDGE_START_SCANNER) -from openprocurement.bot.identification.databridge.utils import ( - journal_context, generate_req_id -) +from openprocurement.bot.identification.databridge.journal_msg_ids import DATABRIDGE_INFO, DATABRIDGE_SYNC_SLEEP, \ + DATABRIDGE_TENDER_PROCESS, DATABRIDGE_WORKER_DIED, DATABRIDGE_RESTART, DATABRIDGE_START_SCANNER +from openprocurement.bot.identification.databridge.utils import journal_context, generate_req_id logger = logging.getLogger(__name__) @@ -22,10 +20,8 @@ class Scanner(Greenlet): """ Edr API Data Bridge """ - pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') - qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') - - def __init__(self, tenders_sync_client, filtered_tender_ids_queue, services_not_available, process_tracker, sleep_change_value, delay=15): + def __init__(self, tenders_sync_client, filtered_tender_ids_queue, services_not_available, process_tracker, + sleep_change_value, delay=15): super(Scanner, self).__init__() self.exit = False self.start_time = datetime.now() @@ -48,7 +44,8 @@ def initialize_sync(self, params=None, direction=None): if direction == "backward": self.initialization_event.clear() assert params['descending'] - response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) + response = self.tenders_sync_client.sync_tenders(params, + extra_headers={'X-Client-Request-ID': generate_req_id()}) # set values in reverse order due to 'descending' option self.initial_sync_point = {'forward_offset': response.prev_page.offset, 'backward_offset': response.next_page.offset} @@ -60,14 +57,15 @@ def initialize_sync(self, params=None, direction=None): self.initialization_event.wait() params['offset'] = self.initial_sync_point['forward_offset'] logger.info("Starting forward sync from offset {}".format(params['offset'])) - return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) + return self.tenders_sync_client.sync_tenders(params, + extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tenders(self, params={}, direction=""): response = self.initialize_sync(params=params, direction=direction) while not (params.get('descending') and - not len(response.data) and - params.get('offset') == response.next_page.offset): + not len(response.data) and + params.get('offset') == response.next_page.offset): tenders = response.data if response else [] params['offset'] = response.next_page.offset for tender in tenders: @@ -75,13 +73,15 @@ def get_tenders(self, params={}, direction=""): yield tender else: logger.info('Skipping tender {} with status {} with procurementMethodType {}'.format( - tender['id'], tender['status'], tender['procurementMethodType']), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, - params={"TENDER_ID": tender['id']})) - logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) + tender['id'], tender['status'], tender['procurementMethodType']), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, + params={"TENDER_ID": tender['id']})) + logger.info('Sleep {} sync...'.format(direction), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) gevent.sleep(self.delay + self.sleep_change_value.time_between_requests) try: - response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) + response = self.tenders_sync_client.sync_tenders(params, extra_headers={ + 'X-Client-Request-ID': generate_req_id()}) self.sleep_change_value.decrement() except ResourceError as re: if re.status_int == 429: @@ -96,11 +96,11 @@ def should_process_tender(self, tender): def valid_qualification_tender(self, tender): return (tender['status'] == "active.qualification" and - tender['procurementMethodType'] in self.qualification_procurementMethodType) + tender['procurementMethodType'] in qualification_procurementMethodType) def valid_prequal_tender(self, tender): return (tender['status'] == 'active.pre-qualification' and - tender['procurementMethodType'] in self.pre_qualification_procurementMethodType) + tender['procurementMethodType'] in pre_qualification_procurementMethodType) def get_tenders_forward(self): logger.info('Start forward data sync worker...') diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 16b224e..d17cf1a 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -11,10 +11,9 @@ from restkit import ResourceError from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, data_string -from openprocurement.bot.identification.databridge.journal_msg_ids import ( - DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, - DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, - DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING) +from openprocurement.bot.identification.databridge.journal_msg_ids import DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE, \ + DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE, DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER, \ + DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER, DATABRIDGE_START_UPLOAD, DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING from openprocurement.bot.identification.databridge.constants import file_name, retry_mult logger = logging.getLogger(__name__) @@ -23,11 +22,6 @@ class UploadFile(Greenlet): """ Upload file with details """ - pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') - qualification_procurementMethodType = ( - 'aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', - 'competitiveDialogueEU.stage2') - def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, process_tracker, doc_service_client, services_not_available, sleep_change_value, delay=15): super(UploadFile, self).__init__() @@ -53,7 +47,7 @@ def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, # blockers self.services_not_available = services_not_available - def resource_errors_403_422_none(self, retry, re, tender_data, document_id, item_name_id): + def removing_data(self, retry, re, tender_data, document_id, item_name_id): logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( re.status_int, data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -61,9 +55,12 @@ def resource_errors_403_422_none(self, retry, re, tender_data, document_id, item "DOCUMENT_ID": document_id})) self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) self.sleep_change_value.decrement() - eval('self.{}upload_to_tender_queue.get()'.format(retry)) + if retry == '': + self.upload_to_tender_queue.get() + elif retry == 'retry': + self.retry_upload_to_tender_queue.get() - def resource_error_429(self, re, tender_data, document_id, item_name_id): + def decrease_request_frequency(self, re, tender_data, document_id, item_name_id): logger.info("Accept 429 while uploading to tender {} {} {} doc_id: {}. Message {}".format( tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -99,14 +96,20 @@ def succesfully_uploaded_to_tender(self, retry, tender_data, document_id, item_n "DOCUMENT_ID": document_id})) # delete current tender after successful upload file (to avoid reloading file) self.process_tracker.update_items_and_tender(tender_data.tender_id, tender_data.item_id) - eval('self.{}upload_to_tender_queue.get()'.format(retry)) self.sleep_change_value.decrement() + if retry == '': + self.upload_to_tender_queue.get() + elif retry == 'retry': + self.retry_upload_to_tender_queue.get() - def st_code_200(self, retry, tender_data, response, document_id, item_name_id): + def move_to_tender_queue(self, retry, tender_data, response, document_id, item_name_id): data = Data(tender_data.tender_id, tender_data.item_id, tender_data.code, tender_data.item_name, dict(response.json(), **{'meta': {'id': document_id}})) self.upload_to_tender_queue.put(data) - eval('self.{}upload_to_doc_service_queue.get()'.format(retry)) + if retry == '': + self.upload_to_doc_service_queue.get() + elif retry == 'retry': + self.retry_upload_to_doc_service_queue.get() logger.info('Successfully uploaded file to doc service {} doc_id: {}'.format( data_string(tender_data), document_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE}, @@ -142,7 +145,7 @@ def upload_to_doc_service(self): self.upload_to_doc_service_queue.get() else: if response.status_code == 200: - self.st_code_200('', tender_data, response, document_id, item_name_id) + self.move_to_tender_queue('', tender_data, response, document_id, item_name_id) else: logger.info( 'Not successful response from document service while uploading {} doc_id: {}. Response {}'. @@ -183,15 +186,16 @@ def retry_upload_to_doc_service(self): raise e else: if response.status_code == 200: - self.st_code_200('retry_', tender_data, response, document_id, item_name_id) + self.move_to_tender_queue('retry', tender_data, response, document_id, item_name_id) else: logger.info( 'Not successful response in retry from document service while uploading {} {} {} {}. Response {}'. format(tender_data.tender_id, tender_data.item_name, tender_data.item_id, document_id, response.status_code), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE.format('_retry')}, - params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + extra=journal_context( + {"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE.format('_retry')}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) gevent.sleep(0) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult) @@ -225,9 +229,9 @@ def upload_to_tender(self): tender_data.item_id)) except ResourceError as re: if re.status_int == 403 or re.status_int == 422 or re.status_int is None: - self.resource_errors_403_422_none('', re, tender_data, document_id, item_name_id) + self.removing_data('', re, tender_data, document_id, item_name_id) elif re.status_int == 429: - self.resource_error_429(re, tender_data, document_id, item_name_id) + self.decrease_request_frequency(re, tender_data, document_id, item_name_id) else: self.resource_errors_else('', re, tender_data, document_id, item_name_id) self.retry_upload_to_tender_queue.put(tender_data) @@ -257,16 +261,16 @@ def retry_upload_to_tender(self): self.client_upload_to_tender(tender_data) except ResourceError as re: if re.status_int == 403 or re.status_int == 422 or re.status_int is None: - self.resource_errors_403_422_none('retry_', re, tender_data, document_id, item_name_id) + self.removing_data('retry', re, tender_data, document_id, item_name_id) continue elif re.status_int == 429: - self.resource_error_429(re, tender_data, document_id, item_name_id) + self.decrease_request_frequency(re, tender_data, document_id, item_name_id) else: self.resource_errors_else('retry', re, tender_data, document_id, item_name_id) except Exception as e: self.exception_errors('retry', e, tender_data, document_id, item_name_id) else: - self.succesfully_uploaded_to_tender('retry_', tender_data, document_id, item_name_id) + self.succesfully_uploaded_to_tender('retry', tender_data, document_id, item_name_id) gevent.sleep(self.sleep_change_value.time_between_requests) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=retry_mult) diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index eac6f3d..b689278 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -232,11 +232,11 @@ def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): expected_result = [] self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(data_info={"test": 1}, doc_id=self.document_ids[0], suf_1=2, suf_2=1, - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con({"test": 1}, self.document_ids[0], 2, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(data_info={"test": 2}, doc_id=self.document_ids[0], suf_1=2, suf_2=2, - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con({"test": 2}, self.document_ids[0], 2, 2, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: @@ -256,11 +256,11 @@ def test_retry_get_edr_data_two_ids(self, mrequest, gevent_sleep): expected_result = [] self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(data_info={"test": 1}, doc_id=self.document_ids[0], suf_1=2, suf_2=1, - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con({"test": 1}, self.document_ids[0], 2, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(data_info={"test": 2}, doc_id=self.document_ids[0], suf_1=2, suf_2=2, - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.file_con({"test": 2}, self.document_ids[0], 2, 2, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() for result in expected_result: @@ -405,8 +405,7 @@ def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): """Recieve 404 and not valid data (worker dies). Check that worker get up""" gevent_sleep.side_effect = custom_sleep mrequest.get(self.url, [{'json': {'data': [{}]}, - 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, - # data contains dict, instead of list + 'headers': {'X-Request-ID': self.edr_req_ids[0]}}, # contains dict instead of list self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) mrequest.get(self.url_id(321), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) @@ -533,15 +532,15 @@ def test_processing_items(self, mrequest, gevent_sleep): award_key = item_key(self.tender_id, self.award_id) qualification_key = item_key(self.tender_id, self.qualification_id) data_1 = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[0], suf_1=2, suf_2=1, source_req=[self.edr_req_ids[0]])) + self.file_con({}, self.document_ids[0], 2, 1, [self.edr_req_ids[0]])) data_2 = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[0], suf_1=2, suf_2=2, source_req=[self.edr_req_ids[0]])) + self.file_con({}, self.document_ids[0], 2, 2, [self.edr_req_ids[0]])) data_3 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', - self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=1, source_req=[self.edr_req_ids[1]])) + self.file_con({}, self.document_ids[1], 3, 1, [self.edr_req_ids[1]])) data_4 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', - self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=2, source_req=[self.edr_req_ids[1]])) + self.file_con({}, self.document_ids[1], 3, 2, [self.edr_req_ids[1]])) data_5 = Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', - self.file_con(doc_id=self.document_ids[1], suf_1=3, suf_2=3, source_req=[self.edr_req_ids[1]])) + self.file_con({}, self.document_ids[1], 3, 3, [self.edr_req_ids[1]])) mrequest.get(self.url, [ self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[0]), self.stat_200([{}, {}, {}], diff --git a/openprocurement/bot/identification/tests/scanner.py b/openprocurement/bot/identification/tests/scanner.py index 681d4ad..a5d366d 100644 --- a/openprocurement/bot/identification/tests/scanner.py +++ b/openprocurement/bot/identification/tests/scanner.py @@ -34,7 +34,7 @@ def tearDown(self): del self.worker @staticmethod - def munchify(status, id, procurementMethodType, data=True): + def mock_tenders(status, id, procurementMethodType, data=True): if data: return munchify({'prev_page': {'offset': '123'}, 'next_page': {'offset': '1234'}, @@ -60,10 +60,10 @@ def test_worker(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client.sync_tenders.side_effect = [RequestFailed(), # worker must restart - self.munchify("active.qualification", self.tenders_id[0], 'UA'), + self.mock_tenders("active.qualification", self.tenders_id[0], 'UA'), Unauthorized(), - self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] + self.mock_tenders("active.tendering", uuid.uuid4().hex, 'UA'), + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU')] for tender_id in self.tenders_id[0:2]: self.assertEqual(self.tender_queue.get(), tender_id) @@ -71,11 +71,11 @@ def test_worker(self, gevent_sleep): def test_429(self, gevent_sleep): """Receive 429 status, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify("active.qualification", self.tenders_id[1], 'UA'), + self.client.sync_tenders.side_effect = [self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), + self.mock_tenders("active.tendering", uuid.uuid4().hex, 'UA'), + self.mock_tenders("active.qualification", self.tenders_id[1], 'UA'), ResourceError(http_code=429), - self.munchify("active.qualification", self.tenders_id[2], 'UA')] + self.mock_tenders("active.qualification", self.tenders_id[2], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 for tender_id in self.tenders_id[0:3]: @@ -86,14 +86,14 @@ def test_429(self, gevent_sleep): def test_429_sleep_change_value(self, gevent_sleep): """Three times receive 429, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), - self.munchify("active.tendering", uuid.uuid4().hex, 'UA'), + self.client.sync_tenders.side_effect = [self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), + self.mock_tenders("active.tendering", uuid.uuid4().hex, 'UA'), + self.mock_tenders("active.tendering", uuid.uuid4().hex, 'UA'), + self.mock_tenders("active.tendering", uuid.uuid4().hex, 'UA'), ResourceError(http_code=429), ResourceError(http_code=429), ResourceError(http_code=429), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:2]: @@ -104,11 +104,11 @@ def test_429_sleep_change_value(self, gevent_sleep): def test_backward_dead(self, gevent_sleep): """Test when backward dies """ gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), + self.client.sync_tenders.side_effect = [self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU'), ResourceError(http_code=403), - self.munchify(None, None, False), - self.munchify("active.pre-qualification", self.tenders_id[2], 'EU')] + self.mock_tenders(None, None, False), + self.mock_tenders("active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:3]: @@ -118,11 +118,11 @@ def test_backward_dead(self, gevent_sleep): def test_forward_dead(self, gevent_sleep): """ Test when forward dies""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify(None, None, None, False), - self.munchify(None, None, None, False), - self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), + self.client.sync_tenders.side_effect = [self.mock_tenders(None, None, None, False), + self.mock_tenders(None, None, None, False), + self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), ResourceError(http_code=403), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU')] + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:2]: @@ -133,10 +133,10 @@ def test_forward_run(self, gevent_sleep): """ Run forward when backward get empty response and prev_page.offset is equal to next_page.offset """ gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), - self.munchify(None, None, None, False), - self.munchify("active.pre-qualification", self.tenders_id[2], 'EU')] + self.client.sync_tenders.side_effect = [self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU'), + self.mock_tenders(None, None, None, False), + self.mock_tenders("active.pre-qualification", self.tenders_id[2], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id[0:3]: @@ -147,13 +147,13 @@ def test_get_tenders_exception(self, gevent_sleep): """ Catch exception in backward worker and after that put 2 tenders to process.Then catch exception for forward and after that put tender to process.""" gevent_sleep.side_effect = custom_sleep - self.client.sync_tenders.side_effect = [self.munchify("active.pre-qualification", self.tenders_id[0], 'EU'), - self.munchify(None, None, None, False), - self.munchify("active.pre-qualification", self.tenders_id[1], 'EU'), - self.munchify("active.pre-qualification", self.tenders_id[2], 'EU'), - self.munchify(None, None, None, False), - self.munchify(None, None, None, False), - self.munchify("active.pre-qualification", self.tenders_id[3], 'EU')] + self.client.sync_tenders.side_effect = [self.mock_tenders("active.pre-qualification", self.tenders_id[0], 'EU'), + self.mock_tenders(None, None, None, False), + self.mock_tenders("active.pre-qualification", self.tenders_id[1], 'EU'), + self.mock_tenders("active.pre-qualification", self.tenders_id[2], 'EU'), + self.mock_tenders(None, None, None, False), + self.mock_tenders(None, None, None, False), + self.mock_tenders("active.pre-qualification", self.tenders_id[3], 'EU')] self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 for tender_id in self.tenders_id: @@ -164,7 +164,7 @@ def test_resource_error(self, gevent_sleep): """Raise Resource error, check queue, check sleep_change_value""" gevent_sleep.side_effect = custom_sleep self.client.sync_tenders.side_effect = [ResourceError(http_code=429), - self.munchify("active.qualification", self.tenders_id[0], 'UA')] + self.mock_tenders("active.qualification", self.tenders_id[0], 'UA')] self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) @@ -192,7 +192,7 @@ def test_forward_exception(self, gevent_sleep): self.sleep_change_value.increment_step = 1 self.sleep_change_value.decrement_step = 0.5 self.worker.initialize_sync = MagicMock(side_effect=[ResourceError(msg=RequestFailed()), - self.munchify("active.pre-qualification", - self.tenders_id[0], 'EU')]) + self.mock_tenders("active.pre-qualification", + self.tenders_id[0], 'EU')]) self.assertEqual(self.tender_queue.get(), self.tenders_id[0]) self.assertEqual(self.worker.initialize_sync.call_count, 2) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 64e4442..2e231da 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from gevent import monkey - monkey.patch_all() import uuid From ecbe2f2af611a180e3d029c397c6847578d0d2ea Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Thu, 10 Aug 2017 15:58:05 +0300 Subject: [PATCH 6/8] Changed retry_mult value --- openprocurement/bot/identification/databridge/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index ae24506..835ca7a 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,7 +4,7 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1 +retry_mult = 1000 pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') From acff7f741b36ca51690d32a6685cdbae3d4d5637 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Thu, 10 Aug 2017 17:47:24 +0300 Subject: [PATCH 7/8] added some changes --- .../identification/databridge/constants.py | 2 +- .../databridge/filter_tender.py | 89 +++--- .../identification/databridge/upload_file.py | 1 - .../bot/identification/tests/edr_handler.py | 257 ++++++++---------- 4 files changed, 168 insertions(+), 181 deletions(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index 835ca7a..ae24506 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,7 +4,7 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1000 +retry_mult = 1 pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') diff --git a/openprocurement/bot/identification/databridge/filter_tender.py b/openprocurement/bot/identification/databridge/filter_tender.py index 526574e..8c8675f 100644 --- a/openprocurement/bot/identification/databridge/filter_tender.py +++ b/openprocurement/bot/identification/databridge/filter_tender.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from gevent import monkey + monkey.patch_all() import logging.config @@ -21,7 +22,6 @@ from openprocurement.bot.identification.databridge.constants import author from restkit import ResourceError - logger = logging.getLogger(__name__) @@ -29,7 +29,8 @@ class FilterTenders(Greenlet): """ Edr API Data Bridge """ identification_scheme = u'UA-EDR' - def __init__(self, tenders_sync_client, filtered_tender_ids_queue, edrpou_codes_queue, process_tracker, services_not_available, sleep_change_value, delay=15): + def __init__(self, tenders_sync_client, filtered_tender_ids_queue, edrpou_codes_queue, process_tracker, + services_not_available, sleep_change_value, delay=15): super(FilterTenders, self).__init__() self.exit = False self.start_time = datetime.now() @@ -58,12 +59,15 @@ def prepare_data(self): gevent.sleep(0) continue try: - response = self.tenders_sync_client.request("GET", path='{}/{}'.format(self.tenders_sync_client.prefix_path, tender_id), + response = self.tenders_sync_client.request("GET", + path='{}/{}'.format(self.tenders_sync_client.prefix_path, + tender_id), headers={'X-Client-Request-ID': generate_req_id()}) except ResourceError as re: if re.status_int == 429: self.sleep_change_value.increment() - logger.info("Waiting tender {} for sleep_change_value: {} seconds".format(tender_id, self.sleep_change_value.time_between_requests)) + logger.info("Waiting tender {} for sleep_change_value: {} seconds".format(tender_id, + self.sleep_change_value.time_between_requests)) else: logger.warning('Fail to get tender info {}'.format(tender_id), extra=journal_context(params={"TENDER_ID": tender_id})) @@ -84,12 +88,14 @@ def prepare_data(self): tender = munchify(loads(response.body_string()))['data'] logger.info('Get tender {} from filtered_tender_ids_queue'.format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_TENDER_FROM_QUEUE}, - params={"TENDER_ID": tender['id']})) + params={"TENDER_ID": tender['id']})) if 'awards' in tender: for award in tender['awards']: - logger.info('Processing tender {} bid {} award {}'.format(tender['id'], award['bid_id'], award['id']), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, - params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], "AWARD_ID": award['id']})) + logger.info( + 'Processing tender {} bid {} award {}'.format(tender['id'], award['bid_id'], award['id']), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, + params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], + "AWARD_ID": award['id']})) if self.should_process_item(award): for supplier in award['suppliers']: code = supplier['identifier']['id'] @@ -98,29 +104,40 @@ def prepare_data(self): logger.info(u'Tender {} bid {} award {} identifier id {} is not valid.'.format( tender['id'], award['bid_id'], award['id'], code), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, - params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], "AWARD_ID": award['id']})) + params={"TENDER_ID": tender['id'], + "BID_ID": award['bid_id'], + "AWARD_ID": award['id']})) continue - # quick check if item was already processed + # quick check if item was already processed if self.process_tracker.check_processed_item(tender['id'], award['id']): logger.info('Tender {} bid {} award {} was already processed.'.format( tender['id'], award['bid_id'], award['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, - params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], "AWARD_ID": award['id']})) + params={"TENDER_ID": tender['id'], + "BID_ID": award['bid_id'], + "AWARD_ID": award['id']})) elif self.should_process_award(supplier, tender, award): self.process_tracker.set_item(tender['id'], award['id']) document_id = generate_doc_id() tender_data = Data(tender['id'], award['id'], str(code), - 'awards', {'meta': {'id': document_id, 'author': author, 'sourceRequests': [response.headers['X-Request-ID']]}}) + 'awards', {'meta': {'id': document_id, 'author': author, + 'sourceRequests': [ + response.headers['X-Request-ID']]}}) self.edrpou_codes_queue.put(tender_data) else: - logger.info('Tender {} bid {} award {} identifier schema isn\'t UA-EDR or tender is already in process.'.format( - tender['id'], award['bid_id'], award['id']), + logger.info( + 'Tender {} bid {} award {} identifier schema isn\'t UA-EDR or tender is already in process.'.format( + tender['id'], award['bid_id'], award['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, - params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], "AWARD_ID": award['id']})) + params={"TENDER_ID": tender['id'], + "BID_ID": award['bid_id'], + "AWARD_ID": award['id']})) else: - logger.info('Tender {} bid {} award {} is not in status pending or award has already document ' - 'with documentType registerExtract.'.format(tender_id, award['bid_id'], award['id']), - extra=journal_context(params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], "AWARD_ID": award['id']})) + logger.info( + 'Tender {} bid {} award {} is not in status pending or award has already document ' + 'with documentType registerExtract.'.format(tender_id, award['bid_id'], award['id']), + extra=journal_context(params={"TENDER_ID": tender['id'], "BID_ID": award['bid_id'], + "AWARD_ID": award['id']})) elif 'qualifications' in tender: for qualification in tender['qualifications']: if self.should_process_item(qualification): @@ -132,38 +149,48 @@ def prepare_data(self): tender['id'], qualification['bidID'], qualification['id'], code), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, params={"TENDER_ID": tender['id'], "BID_ID": - qualification['bidID'], "QUALIFICATION_ID": qualification['id']})) + qualification['bidID'], + "QUALIFICATION_ID": qualification['id']})) continue # quick check if item was already processed if self.process_tracker.check_processed_item(tender['id'], qualification['id']): logger.info('Tender {} bid {} qualification {} was already processed.'.format( - tender['id'], qualification['bidID'], qualification['id']), + tender['id'], qualification['bidID'], qualification['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, params={"TENDER_ID": tender['id'], - "BID_ID": qualification['bidID'], "QUALIFICATION_ID": qualification['id']})) + "BID_ID": qualification['bidID'], + "QUALIFICATION_ID": qualification['id']})) # check first identification scheme, if yes then check if item is already in process or not elif self.should_process_qualification(appropriate_bid, tender, qualification): self.process_tracker.set_item(tender['id'], qualification['id']) document_id = generate_doc_id() tender_data = Data(tender['id'], qualification['id'], str(code), - 'qualifications', {'meta': {'id': document_id, 'author': author, 'sourceRequests': [response.headers['X-Request-ID']]}}) + 'qualifications', {'meta': {'id': document_id, 'author': author, + 'sourceRequests': [ + response.headers['X-Request-ID']]}}) self.edrpou_codes_queue.put(tender_data) logger.info('Processing tender {} bid {} qualification {}'.format( tender['id'], qualification['bidID'], qualification['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS}, - params={"TENDER_ID": tender['id'], "BID_ID": qualification['bidID'], "QUALIFICATION_ID": qualification['id']})) + params={"TENDER_ID": tender['id'], + "BID_ID": qualification['bidID'], + "QUALIFICATION_ID": qualification['id']})) else: - logger.info('Tender {} bid {} qualification {} identifier schema is not UA-EDR or tender is already in process.'.format( - tender['id'], qualification['bidID'], qualification['id']), + logger.info( + 'Tender {} bid {} qualification {} identifier schema is not UA-EDR or tender is already in process.'.format( + tender['id'], qualification['bidID'], qualification['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS}, params={"TENDER_ID": tender['id'], - "BID_ID": qualification['bidID'], "QUALIFICATION_ID": qualification['id']})) + "BID_ID": qualification['bidID'], + "QUALIFICATION_ID": qualification['id']})) else: - logger.info('Tender {} bid {} qualification {} is not in status pending or qualification has ' - 'already document with documentType registerExtract.'.format( - tender_id, qualification['bidID'], qualification['id']), - extra=journal_context(params={"TENDER_ID": tender['id'], - "BID_ID": qualification['bidID'], "QUALIFICATION_ID": qualification['id']})) + logger.info( + 'Tender {} bid {} qualification {} is not in status pending or qualification has ' + 'already document with documentType registerExtract.'.format( + tender_id, qualification['bidID'], qualification['id']), + extra=journal_context(params={"TENDER_ID": tender['id'], + "BID_ID": qualification['bidID'], + "QUALIFICATION_ID": qualification['id']})) self.filtered_tender_ids_queue.get() # Remove elem from queue gevent.sleep(self.sleep_change_value.time_between_requests) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index d17cf1a..7854ea9 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -262,7 +262,6 @@ def retry_upload_to_tender(self): except ResourceError as re: if re.status_int == 403 or re.status_int == 422 or re.status_int is None: self.removing_data('retry', re, tender_data, document_id, item_name_id) - continue elif re.status_int == 429: self.decrease_request_frequency(re, tender_data, document_id, item_name_id) else: diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index b689278..11c2eb5 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -123,9 +123,8 @@ def test_proxy_client(self, mrequest, gevent_sleep): for i in range(2): self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.file_con(source_req=[]))) # data - expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(source_req=[self.edr_req_ids[i]]))) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(source_req=[self.edr_req_ids[i]]))) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(self.edr_ids)) @@ -157,18 +156,15 @@ def test_proxy_client_402(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.get(self.uri, [self.stat_c(403, 0, [{'message': 'Payment required.', 'code': 5}], self.edr_req_ids[0]), # pay for me - self.stat_200([{}], self.source_date, - self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[0]), self.stat_c(403, 0, [{'message': 'Payment required.', 'code': 5}], self.edr_req_ids[1]), # pay for me - self.stat_200([{}], self.source_date, - self.edr_req_ids[1])]) + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in xrange(2): mrequest.get(self.url_id(self.local_edr_ids[i]), [self.stat_200([{}], self.source_date, self.edr_req_ids[i])]) - self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) expected_result.append( Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.file_con(doc_id=self.document_ids[0], source_req=[self.edr_req_ids[i], self.edr_req_ids[i]]))) @@ -185,14 +181,11 @@ def test_retry_get_edr_id(self, mrequest, gevent_sleep): self.stat_c(403, 0, '', self.edr_req_ids[0]), self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) - expected_result = [] - for i in range(1): - self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[0], - source_req=[self.edr_req_ids[0], self.edr_req_ids[0]]))) - self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result[0]) + expected_result = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[0]])) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result) self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -201,11 +194,10 @@ def test_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept response with 404 status code and error message 'EDRPOU not found'. Check that tender_data is in self.upload_to_doc_service_queue.""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.url, json={'errors': [{'description': - [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"detailsSourceDate": self.source_date}}]}]}, - status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}) + mrequest.get(self.url, json={'errors': [{'description': [ + {"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, status_code=404, + headers={'X-Request-ID': self.edr_req_ids[0]}) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', @@ -249,10 +241,10 @@ def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): def test_retry_get_edr_data_two_ids(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.uri, - [self.stat_c(403, '10', '', self.edr_req_ids[0]), - self.stat_200([{"test": 1}, {"test": 2}], - ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) + mrequest.get(self.uri, [self.stat_c(403, '10', '', self.edr_req_ids[0]), + self.stat_200([{"test": 1}, {"test": 2}], + ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], + self.edr_req_ids[1])]) expected_result = [] self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', @@ -274,14 +266,12 @@ def test_retry_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept 5 times response with status code 403 and error, then accept response with status code 404 and message 'EDRPOU not found'""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.uri, - [self.stat_c(403, 0, '', self.edr_req_ids[0]) for _ in range(5)] + [ - {'json': { - 'errors': [ - {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"detailsSourceDate": self.source_date}}]}]}, - 'status_code': 404, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}]) + mrequest.get(self.uri, [self.stat_c(403, 0, '', self.edr_req_ids[0]) for _ in range(5)] + + [{'json': { + 'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, + 'status_code': 404, 'headers': {'X-Request-ID': self.edr_req_ids[0]}}]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', @@ -318,17 +308,16 @@ def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=403)), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, - json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": { - "errorDetails": "Couldn't find this code in EDR.", - "code": u"notFound"}, - "meta": { - "detailsSourceDate": self.source_date, - 'id': self.document_ids[0], "version": version, - 'author': author - }}]}]})))]) + json=MagicMock(return_value={ + "errors": + [{"description": + [{"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_ids[0], "version": version, + 'author': author}}]}]})))]) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) @@ -352,16 +341,16 @@ def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, - json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": { - "errorDetails": "Couldn't find this code in EDR.", - "code": u"notFound"}, - "meta": { - "detailsSourceDate": self.source_date, - 'id': self.document_ids[0], "version": version, - 'author': author}}]}]})))]) + json=MagicMock(return_value={ + "errors": [ + {"description": [ + {"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_ids[0], "version": version, + 'author': author}}]}]})))]) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) @@ -386,16 +375,16 @@ def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): self.worker.get_edr_data_request = MagicMock(side_effect=[ Exception(), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': self.edr_req_ids[0]}, - json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": { - "errorDetails": "Couldn't find this code in EDR.", - "code": u"notFound"}, - "meta": { - "detailsSourceDate": self.source_date, - 'id': self.document_ids[0], "version": version, - 'author': author}}]}]})))]) + json=MagicMock(return_value={ + "errors": [ + {"description": [ + {"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_ids[0], "version": version, + 'author': author}}]}]})))]) for result in expected_result: self.assertEquals(self.upload_to_doc_service_queue.get(), result) @@ -445,9 +434,8 @@ def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): """Accept 6 times errors in response while requesting /verify""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.url, - [self.stat_c(403, 0, '', self.edr_req_ids[i]) for i in range(6)] + [ - self.stat_200([{}], self.source_date, self.edr_req_ids[6])]) + mrequest.get(self.url, [self.stat_c(403, 0, '', self.edr_req_ids[i]) for i in range(6)] + + [self.stat_200([{}], self.source_date, self.edr_req_ids[6])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.process_tracker = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), @@ -455,10 +443,8 @@ def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): self.file_con(doc_id=self.document_ids[0], source_req=[self.edr_req_ids[0], self.edr_req_ids[6]]))) self.assertEqual(mrequest.call_count, 7) # processing 7 requests - self.assertEqual(mrequest.request_history[0].url, - self.urls('verify?id={}'.format(self.edr_ids))) # check first url - self.assertEqual(mrequest.request_history[6].url, - self.urls('verify?id={}'.format(self.edr_ids))) # check 7th url + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -543,10 +529,8 @@ def test_processing_items(self, mrequest, gevent_sleep): self.file_con({}, self.document_ids[1], 3, 3, [self.edr_req_ids[1]])) mrequest.get(self.url, [ self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[0]), - self.stat_200([{}, {}, {}], - ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00", - "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) - + self.stat_200([{}, {}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00", + "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) # create queues self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', {'meta': {'id': self.document_ids[0], 'author': author, @@ -554,7 +538,6 @@ def test_processing_items(self, mrequest, gevent_sleep): self.edrpou_codes_queue.put(Data(self.tender_id, self.qualification_id, self.edr_ids, 'qualifications', {'meta': {'id': self.document_ids[1], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - for data in [data_1, data_2, data_3, data_4, data_5]: self.assertEquals(self.upload_to_doc_service_queue.get(), data) self.assertEqual(self.edrpou_codes_queue.qsize(), 0) @@ -585,13 +568,10 @@ def test_wrong_ip(self, mrequest, gevent_sleep): def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for edrpou_codes_queue """ gevent_sleep.side_effect = custom_sleep - mrequest.get(self.uri, - [self.stat_200([{}], self.source_date, self.edr_req_ids[0]), - self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) - mrequest.get(self.url_id(self.local_edr_ids[0]), - [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) - mrequest.get(self.url_id(self.local_edr_ids[1]), - [self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.uri, [self.stat_200([{}], self.source_date, self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, self.edr_req_ids[0])]) + mrequest.get(self.url_id(self.local_edr_ids[1]), [self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) edrpou_codes_queue = MagicMock() expected_result = [] edrpou_codes_queue_list = [LoopExit()] @@ -600,9 +580,8 @@ def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): {'meta': {'id': self.document_ids[i], 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) self.worker.retry_edrpou_codes_queue = MagicMock() self.worker.process_tracker = MagicMock() @@ -629,9 +608,8 @@ def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): {"meta": {"id": self.document_ids[i], 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[i], source_req=[self.edr_req_ids[i]]))) self.worker.retry_edrpou_codes_queue = MagicMock() self.worker.retry_edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) @@ -648,25 +626,21 @@ def test_retry_exception(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep retry_response = MagicMock() retry_response.status_code = 500 - mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ - {'exc': RetryException('Retry Exception', retry_response)}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, - 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}]) - expected_result = [] - for i in range(1): - self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[0], - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) + mrequest.get(self.url, [{'status_code': 500, + 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + + [{'exc': RetryException('Retry Exception', retry_response)}, + {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, + 'status_code': 200, 'headers': {'X-Request-ID': self.edr_req_ids[1]}}]) + expected_result = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]])) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() - for result in expected_result: - self.assertEquals(self.upload_to_doc_service_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) + self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result.code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) + self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result.code))) self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) self.assertEqual(mrequest.call_count, 7) @@ -681,27 +655,20 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): retry_response.json.return_value = {'errors': [ {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, 'meta': {"detailsSourceDate": self.source_date}}]}]} - mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ - {'exc': RetryException('Retry Exception', retry_response)}]) - - expected_result = [] - for i in range(1): - self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - {'error': {'errorDetails': "Couldn't find this code in EDR.", - 'code': 'notFound'}, - 'meta': {'detailsSourceDate': self.source_date, - 'id': self.document_ids[0], "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - self.edr_req_ids[0]]}})) - self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue - self.worker.process_tracker = MagicMock() - for result in expected_result: - self.assertEquals(self.upload_to_doc_service_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) + mrequest.get(self.url, [{'status_code': 500, + 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + + [{'exc': RetryException('Retry Exception', retry_response)}]) + expected_result = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + {'error': {'errorDetails': "Couldn't find this code in EDR.", 'code': 'notFound'}, + 'meta': {'detailsSourceDate': self.source_date, + 'id': self.document_ids[0], "version": version, 'author': author, + 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', self.edr_req_ids[0]]}}) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data + self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue + self.worker.process_tracker = MagicMock() + self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result.code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(mrequest.call_count, 6) @@ -710,24 +677,20 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): def test_exception(self, mrequest, gevent_sleep): """ Raise Exception in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.url, - [{'status_code': 500, 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + [ - {'exc': Exception()}, self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) - expected_result = [] - for i in range(1): - self.edrpou_codes_queue.put( - Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data - expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', - self.file_con(doc_id=self.document_ids[0], - source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) - + mrequest.get(self.url, [{'status_code': 500, + 'headers': {'X-Request-ID': self.edr_req_ids[0]}} for _ in range(5)] + + [{'exc': Exception()}, self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + expected_result = Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con(doc_id=self.document_ids[0], + source_req=[self.edr_req_ids[0], self.edr_req_ids[1]])) + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) # data self.worker.upload_to_doc_service_queue = self.upload_to_doc_service_queue self.worker.process_tracker = MagicMock() - for result in expected_result: - self.assertEquals(self.upload_to_doc_service_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) + self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result.code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) + self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result.code))) self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) self.assertEqual(mrequest.call_count, 7) @@ -736,18 +699,16 @@ def test_exception(self, mrequest, gevent_sleep): def test_value_error(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - mrequest.get(self.url, - [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0])] + [ - {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': self.edr_req_ids[1]}} for _ in - range(5)] + [ - self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) + mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], self.edr_req_ids[0])] + + [{"text": "resp", 'status_code': 403, + 'headers': {'X-Request-ID': self.edr_req_ids[1]}} for _ in range(5)] + + [self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.meta())) self.worker.retry_edr_ids_queue = MagicMock() self.assertEquals(self.upload_to_doc_service_queue.get(), Data(self.tender_id, self.award_id, self.edr_ids, 'awards', self.file_con(doc_id=self.document_ids[0], source_req=[self.edr_req_ids[0], self.edr_req_ids[1]]))) - self.assertEqual(mrequest.call_count, 7) self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) From 5bc1c77b1e0e98167e67e70dcbdf498f59f8b447 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Thu, 10 Aug 2017 17:56:29 +0300 Subject: [PATCH 8/8] Changed retry_mult value --- openprocurement/bot/identification/databridge/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openprocurement/bot/identification/databridge/constants.py b/openprocurement/bot/identification/databridge/constants.py index ae24506..835ca7a 100644 --- a/openprocurement/bot/identification/databridge/constants.py +++ b/openprocurement/bot/identification/databridge/constants.py @@ -4,7 +4,7 @@ version = '{}.{}.{}'.format(major, minor, bugfix) # major.minor.bugfix file_name = 'edr_identification.yaml' author = "IdentificationBot" -retry_mult = 1 +retry_mult = 1000 pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2')