From 11978e35ac48921d4588e8b32f5120eca496ef0a Mon Sep 17 00:00:00 2001 From: Shevchenko Vladislav Date: Tue, 18 Jul 2017 16:00:26 +0300 Subject: [PATCH 1/7] Handled ResourceError in upload_file.py --- .../identification/databridge/upload_file.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index 69d174d..da2797c 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -9,6 +9,7 @@ from gevent import Greenlet, spawn from gevent.hub import LoopExit from restkit import ResourceError +import re as regex from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, data_string from openprocurement.bot.identification.databridge.journal_msg_ids import ( @@ -264,10 +265,21 @@ def retry_upload_to_tender(self): UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 continue else: - logger.info('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, re.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) + err_desc = re.message["errors"][0]["description"] + curr_ten_stat = regex.findall(r'\((.*?)\)', err_desc) + if err_desc == "Can't add document in current ({}) tender status".format(curr_ten_stat[0]): + logger.info('ResourceError while retry uploading file to {} doc_id: {}. Message: {}'.format( + data_string(tender_data), document_id, re.message)) + logger.info('Trying to update processing items') + self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.retry_upload_to_tender_queue.get() + else: + logger.info('ResourceError while retry uploading file to {} doc_id: {}. Message: {}'.format( + data_string(tender_data), document_id, re.message), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, + params={"TENDER_ID": tender_data.tender_id, + item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) except Exception as e: logger.info('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, e.message), From b198b9edf49fffc97b95df1f829450710a04ea32 Mon Sep 17 00:00:00 2001 From: Shevchenko Vladislav Date: Wed, 19 Jul 2017 16:35:44 +0300 Subject: [PATCH 2/7] Handled ResourceError status_int as 403 --- .../identification/databridge/upload_file.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index da2797c..c3ea6ca 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -28,6 +28,7 @@ class UploadFile(Greenlet): pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') sleep_change_value = 0 + start_of_retry_uploading_error_message = "Can't add document" def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, processed_items, doc_service_client, increment_step=1, decrement_step=1, delay=15): super(UploadFile, self).__init__() @@ -195,7 +196,7 @@ def upload_to_tender(self): "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step else: - logger.warning('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format( + logger.warning('ResourceError while uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, re.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, @@ -204,7 +205,6 @@ def upload_to_tender(self): self.upload_to_tender_queue.get() UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 except Exception as e: - logger.info('Exception while uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, e.message), extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER}, @@ -254,7 +254,7 @@ def retry_upload_to_tender(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step - elif re.status_int == 403: + elif re.status_int == 403 or self.start_of_retry_uploading_error_message in re.message["errors"][0]["description"]: logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format( data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -265,21 +265,10 @@ def retry_upload_to_tender(self): UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 continue else: - err_desc = re.message["errors"][0]["description"] - curr_ten_stat = regex.findall(r'\((.*?)\)', err_desc) - if err_desc == "Can't add document in current ({}) tender status".format(curr_ten_stat[0]): - logger.info('ResourceError while retry uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, re.message)) - logger.info('Trying to update processing items') - self.update_processing_items(tender_data.tender_id, tender_data.item_id) - self.retry_upload_to_tender_queue.get() - else: - logger.info('ResourceError while retry uploading file to {} doc_id: {}. Message: {}'.format( - data_string(tender_data), document_id, re.message), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, - params={"TENDER_ID": tender_data.tender_id, - item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) + logger.info('ResourceError while retry uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format( + data_string(tender_data), document_id, re.status_int, re.message), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER}, + params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) except Exception as e: logger.info('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, e.message), From e5ef5efe5e47c8db5b38d4e8b2a8cfe4227589fe Mon Sep 17 00:00:00 2001 From: Shevchenko Vladislav Date: Wed, 19 Jul 2017 16:43:24 +0300 Subject: [PATCH 3/7] Merging --- openprocurement/bot/identification/client.py | 5 +- .../bot/identification/databridge/bridge.py | 6 +- .../bot/identification/tests/bridge.py | 14 +- .../bot/identification/tests/edr_handler.py | 1276 ++++++----------- 4 files changed, 487 insertions(+), 814 deletions(-) diff --git a/openprocurement/bot/identification/client.py b/openprocurement/bot/identification/client.py index 8246175..5ab8f50 100644 --- a/openprocurement/bot/identification/client.py +++ b/openprocurement/bot/identification/client.py @@ -20,9 +20,10 @@ def verify(self, param, code, headers): return response - def health(self): + def health(self, sandbox_mode): """Send request to the Proxy server to get whether its active""" - response = self.session.get(url=self.health_url, auth=(self.user, self.password), timeout=self.timeout) + response = self.session.get(url=self.health_url, auth=(self.user, self.password), + headers={"sandbox-mode": sandbox_mode}, timeout=self.timeout) if response.status_code == 200: return response raise requests.RequestException("{} {} {}".format(response.url, response.status_code, response.reason), response=response) diff --git a/openprocurement/bot/identification/databridge/bridge.py b/openprocurement/bot/identification/databridge/bridge.py index 45bbd46..07a01f7 100644 --- a/openprocurement/bot/identification/databridge/bridge.py +++ b/openprocurement/bot/identification/databridge/bridge.py @@ -58,6 +58,7 @@ def __init__(self, config): self.decrement_step = self.config_get('decrement_step') or 1 self.doc_service_host = self.config_get('doc_service_server') self.doc_service_port = self.config_get('doc_service_port') or 6555 + self.sandbox_mode = os.environ.get('SANDBOX_MODE', 'False') # init clients self.tenders_sync_client = TendersClientSync('', host_url=ro_api_server, api_version=api_version) @@ -137,10 +138,11 @@ def check_doc_service(self): return True def check_proxy(self): + """Check whether proxy is up and has the same sandbox mode (to prevent launching wrong pair of bot-proxy)""" try: - self.proxyClient.health() + self.proxyClient.health(self.sandbox_mode) except RequestException as e: - logger.info('Proxy server connection error, message {}'.format(e), + logger.info('Proxy server connection error, message {} {}'.format(e, self.sandbox_mode), extra=journal_context({"MESSAGE_ID": DATABRIDGE_PROXY_SERVER_CONN_ERROR}, {})) raise e else: diff --git a/openprocurement/bot/identification/tests/bridge.py b/openprocurement/bot/identification/tests/bridge.py index e3a8020..146cab5 100644 --- a/openprocurement/bot/identification/tests/bridge.py +++ b/openprocurement/bot/identification/tests/bridge.py @@ -9,7 +9,7 @@ from restkit import RequestError, ResourceError from gevent.pywsgi import WSGIServer -from bottle import Bottle, response +from bottle import Bottle, response, request from openprocurement.bot.identification.databridge.bridge import EdrDataBridge from openprocurement_client.client import TendersClientSync, TendersClient @@ -105,6 +105,8 @@ def doc_response(): def proxy_response(): + if request.headers.get("sandbox-mode") != "True": # Imitation of health comparison + response.status = 400 return response @@ -198,14 +200,24 @@ def test_run(self, sleep): def test_proxy_server_failure(self): self.worker = EdrDataBridge(config) + self.worker.sandbox_mode = "True" self.proxy_server.stop() with self.assertRaises(RequestException): self.worker.check_proxy() self.proxy_server.start() self.assertTrue(self.worker.check_proxy()) + def test_proxy_sandmox_mismatch(self): + self.worker = EdrDataBridge(config) + self.worker.sandbox_mode = "False" + with self.assertRaises(RequestException): + self.worker.check_proxy() + self.worker.sandbox_mode = "True" + self.assertTrue(self.worker.check_proxy()) + def test_proxy_server_success(self): self.worker = EdrDataBridge(config) + self.worker.sandbox_mode = "True" self.assertTrue(self.worker.check_proxy()) def test_doc_service_failure(self): diff --git a/openprocurement/bot/identification/tests/edr_handler.py b/openprocurement/bot/identification/tests/edr_handler.py index f46e732..5344afe 100644 --- a/openprocurement/bot/identification/tests/edr_handler.py +++ b/openprocurement/bot/identification/tests/edr_handler.py @@ -19,72 +19,111 @@ from openprocurement.bot.identification.databridge.edr_handler import EdrHandler from openprocurement.bot.identification.databridge.filter_tender import FilterTenders from openprocurement.bot.identification.databridge.utils import Data, generate_doc_id, RetryException -from openprocurement.bot.identification.tests.utils import custom_sleep, generate_answers, generate_request_id, ResponseMock +from openprocurement.bot.identification.tests.utils import custom_sleep, generate_answers, generate_request_id, \ + ResponseMock from openprocurement.bot.identification.client import ProxyClient from openprocurement.bot.identification.databridge.constants import version, author +def get_random_edr_ids(count=1): + return [str(random.randrange(10000000, 99999999)) for _ in range(count)] + + class TestEdrHandlerWorker(unittest.TestCase): + def setUp(self): + self.source_date = ["2017-04-25T11:56:36+00:00"] + self.gen_req_id = [generate_request_id() for _ in xrange(10)] + self.edrpou_codes_queue = Queue(10) + self.edr_ids_queue = Queue(10) + self.upload_to_doc_service_queue = Queue(10) + self.filtered_tender_ids_queue = Queue(10) + self.tender_id = uuid.uuid4().hex + self.document_id = generate_doc_id() + self.award_id = uuid.uuid4().hex + self.edr_req_ids = [generate_request_id() for _ in xrange(10)] + self.proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + self.uri = "{uri}".format(uri=self.proxy_client.verify_url) + self.url = "{url}".format(url=self.proxy_client.verify_url) + self.local_edr_ids = get_random_edr_ids(2) + self.edr_ids = get_random_edr_ids(1)[0] + self.processing_items = {} + self.worker = EdrHandler.spawn(self.proxy_client, self.edrpou_codes_queue, + self.upload_to_doc_service_queue, self.processing_items) + + def meta(self): + return {'meta': {'id': self.document_id, 'author': author, 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}} + + def tearDown(self): + self.worker.shutdown() + self.assertEqual(self.edrpou_codes_queue.qsize(), 0) + self.assertEqual(self.edr_ids_queue.qsize(), 0) + del self.worker + + @staticmethod + def stat_c(st_code, ret_aft, err_desc, x_req_id): + if ret_aft == 0: + return {'json': {'errors': [{'description': err_desc}]}, 'status_code': st_code, + 'headers': {'X-Request-ID': x_req_id}} + else: + return {'json': {'errors': [{'description': err_desc}]}, 'status_code': st_code, + 'headers': {'Retry-After': ret_aft, 'X-Request-ID': x_req_id}} + + @staticmethod + def stat_200(data_info, det_source_date, x_req_id): + return {'json': {'data': data_info, "meta": {"detailsSourceDate": det_source_date}}, + 'status_code': 200, 'headers': {'X-Request-ID': x_req_id}} + + @staticmethod + def file_con(data_info, doc_id, suf_1, suf_2, source_req): + if suf_1 == 1 and suf_2 == 1: + return {'data': data_info, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", "id": "{}".format(doc_id), + "version": version, 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220'] + source_req}} + else: + return {'data': data_info, + "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", "id": "{}.{}.{}".format(doc_id, suf_1, suf_2), + "version": version, 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220'] + source_req}} + + @staticmethod + def urls(inf): + return u'127.0.0.1:80/api/1.0/{}'.format(inf) + + def url_id(self, u_id): + return "{url}/{id}".format(url=self.proxy_client.verify_url, id=u_id) def test_init(self): worker = EdrHandler.spawn(None, None, None, None) self.assertGreater(datetime.datetime.now().isoformat(), worker.start_time.isoformat()) - self.assertEqual(worker.proxyClient, None) self.assertEqual(worker.edrpou_codes_queue, None) self.assertEqual(worker.upload_to_doc_service_queue, None) self.assertEqual(worker.delay, 15) self.assertEqual(worker.exit, False) - worker.shutdown() self.assertEqual(worker.exit, True) - del worker @requests_mock.Mocker() @patch('gevent.sleep') def test_proxy_client(self, mrequest, gevent_sleep): """ Test that proxy return json with id """ gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_ids = [generate_request_id(), generate_request_id()] - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_ids[1]}}]) - - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - + mrequest.get(self.url, [self.stat_200([{}], self.source_date, self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(2): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {"data": {}, 'meta': {'sourceDate': '2017-04-25T11:56:36+00:00', - 'id': document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[i]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, self.document_id, 1, 1, []))) # data + expected_result.append( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i]]))) for result in expected_result: - self.assertEquals(check_queue.get(), result) - - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[1].code)) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(self.edr_ids)) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(mrequest.call_count, 2) @requests_mock.Mocker() @@ -92,214 +131,116 @@ def test_proxy_client(self, mrequest, gevent_sleep): def test_proxy_client_429(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 429, - 'headers': {'Retry-After': '3', 'X-Request-ID': edr_req_ids[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 429, 'headers': - {'Retry-After': '3', 'X-Request-ID': edr_req_ids[1]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) - - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - + mrequest.get(self.uri, [self.stat_c(429, '1', '', self.edr_req_ids[0]), + self.stat_200([{}], self.source_date, self.edr_req_ids[0]), + self.stat_c(429, '1', '', self.edr_req_ids[1]), + self.stat_200([{}], self.source_date, self.edr_req_ids[1])]) expected_result = [] for i in range(2): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {"data": {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[i], edr_req_ids[i]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + expected_result.append( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) # result for result in expected_result: - self.assertEquals(check_queue.get(), result) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') + self.assertEquals(self.upload_to_doc_service_queue.get(), result) @requests_mock.Mocker() @patch('gevent.sleep') def test_proxy_client_402(self, mrequest, gevent_sleep): """First request returns Edr API returns to proxy 402 status code with messages.""" gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_id = [generate_request_id(), generate_request_id()] - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': [{'message': 'Payment required.', 'code': 5}]}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, # pay for me - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'errors': [{'description': [{'message': 'Payment required.', 'code': 5}]}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, # pay for me - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}]) - - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - + edr_details_req_id = self.gen_req_id[0:2] + mrequest.get(self.uri, [self.stat_c(403, 0, [{'message': 'Payment required.', 'code': 5}], self.edr_req_ids[0]), + # pay for me + self.stat_200([{}], self.source_date, + self.edr_req_ids[0]), + self.stat_c(403, 0, [{'message': 'Payment required.', 'code': 5}], self.edr_req_ids[1]), + # pay for me + self.stat_200([{}], self.source_date, + self.edr_req_ids[1])]) expected_result = [] - for i in range(2): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[i], edr_req_id[i]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + for i in xrange(2): + mrequest.get(self.url_id(self.local_edr_ids[i]), + [self.stat_200([{}], self.source_date, edr_details_req_id[i])]) + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + expected_result.append( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, self.document_id, 1, 1, [self.edr_req_ids[i], self.edr_req_ids[i]]))) for result in expected_result: - self.assertEquals(check_queue.get(), result) - self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) self.assertIsNotNone(mrequest.request_history[3].headers['X-Client-Request-ID']) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - @requests_mock.Mocker() @patch('gevent.sleep') - def test_retry_get_edr_data(self, mrequest, gevent_sleep): - """First and second response returns 403 status code. Tests retry for get_edr_data worker""" + def test_retry_get_edr_id(self, mrequest, gevent_sleep): + """First and second response returns 403 status code. Tests retry for get_edr_id worker""" gevent_sleep.side_effect = custom_sleep - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id}}]) - - edrpou_codes_queue = Queue(10) - check_queue = Queue(10) - + edr_details_req_ids = self.gen_req_id[0] + local_edr_ids = get_random_edr_ids(1) + mrequest.get(self.uri, [self.stat_c(403, 0, '', edr_details_req_ids), + self.stat_c(403, 0, '', edr_details_req_ids), + self.stat_200([{}], self.source_date, edr_details_req_ids)]) + mrequest.get(self.url_id(local_edr_ids[0]), [self.stat_200([{}], self.source_date, edr_details_req_ids)]) expected_result = [] for i in range(1): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {"meta": {"id": document_id, "author": author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id, edr_req_id]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - self.assertEquals(check_queue.get(), expected_result[0]) + self.edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, self.document_id, 1, 1, + [edr_details_req_ids, edr_details_req_ids]))) + self.assertEquals(self.upload_to_doc_service_queue.get(), expected_result[0]) self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') def test_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept response with 404 status code and error message 'EDRPOU not found'. Check that tender_data - is in upload_to_doc_service_queue.""" + is in self.upload_to_doc_service_queue.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - edr_req_id = generate_request_id() - document_id = generate_doc_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - json={'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}]}]}, + edr_req_id = self.gen_req_id[0] + mrequest.get(self.url, json={'errors': [{'description': + [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, status_code=404, headers={'X-Request-ID': edr_req_id}) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', edr_req_id]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + {"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date, + "id": self.document_id, "version": version, + 'author': author, + "sourceRequests": ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', + edr_req_id]}})) + # check that data not in self.edr_ids_queue self.assertEqual(mrequest.call_count, 1) # Requests must call proxy once - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) @requests_mock.Mocker() @patch('gevent.sleep') def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): """Accept wrong format in first request, put to retry, check the results""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = [generate_request_id() for _ in range(2)] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'data': [{"test": 1}, {"test": 2}], - "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'json': {'data': [{"test": 1}, {"test": 2}], - "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}} - ]) - + mrequest.get(self.uri, + [self.stat_200([{"test": 1}, {"test": 2}], [self.source_date], self.edr_req_ids[0]), + self.stat_200([{"test": 1}, {"test": 2}], + ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) edrpou_codes_queue = Queue(10) check_queue = Queue(10) expected_result = [] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_id = str(random.randrange(10000000, 99999999)) - edrpou_codes_queue.put(Data(tender_id, award_id, edr_id, "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_id, "awards", - {"data": {"test": 1}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id+".2.1", - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - expected_result.append(Data(tender_id, award_id, edr_id, "awards", - {"data": {"test": 2}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id+".2.2", - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({"test": 1}, self.document_id, 2, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({"test": 2}, self.document_id, 2, 2, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) for result in expected_result: self.assertEquals(check_queue.get(), result) - - self.assertEqual(worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(check_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @@ -307,50 +248,24 @@ def test_get_edr_data_two_ids(self, mrequest, gevent_sleep): def test_retry_get_edr_data_two_ids(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_ids = [generate_request_id(), generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, - 'headers': {'Retry-After': '10', 'X-Request-ID': edr_req_ids[0]}}, - {'json': {'data': [{"test": 1}, {"test": 2}], - "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}} - ]) - + mrequest.get(self.uri, + [self.stat_c(403, '10', '', self.edr_req_ids[0]), + self.stat_200([{"test": 1}, {"test": 2}], + ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], self.edr_req_ids[1])]) edrpou_codes_queue = Queue(10) check_queue = Queue(10) - expected_result = [] - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_id = str(random.randrange(10000000, 99999999)) - edrpou_codes_queue.put(Data(tender_id, award_id, edr_id, "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_id, "awards", - {"data": {"test": 1}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id+".2.1", - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - expected_result.append(Data(tender_id, award_id, edr_id, "awards", - {"data": {"test": 2}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id+".2.2", - "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({"test": 1}, self.document_id, 2, 1, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) # result + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({"test": 2}, self.document_id, 2, 2, + [self.edr_req_ids[0], self.edr_req_ids[1]]))) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) for result in expected_result: self.assertEquals(check_queue.get(), result) - - self.assertEqual(worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') + self.assertEqual(self.worker.retry_edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(check_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @@ -359,201 +274,154 @@ def test_retry_get_edr_data_empty_response(self, mrequest, gevent_sleep): """Accept 5 times response with status code 403 and error, then accept response with status code 404 and message 'EDRPOU not found'""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - document_id = generate_doc_id() - award_id = uuid.uuid4().hex - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id}}, - {'json': {'errors': [{'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00"}}]}]}, - 'status_code': 404, 'headers': {'X-Request-ID': edr_req_id}}]) - - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", - "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", 'id': document_id, "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_req_id = self.gen_req_id[0] + mrequest.get(self.uri, [self.stat_c(403, 0, '', edr_req_id), self.stat_c(403, 0, '', edr_req_id), + self.stat_c(403, 0, '', edr_req_id), self.stat_c(403, 0, '', edr_req_id), + self.stat_c(403, 0, '', edr_req_id), + {'json': { + 'errors': [ + {'description': [{"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date}}]}]}, + 'status_code': 404, 'headers': {'X-Request-ID': edr_req_id}}]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + {"error": {"errorDetails": "Couldn't find this code in EDR.", + "code": "notFound"}, + "meta": {"detailsSourceDate": self.source_date, + 'id': self.document_id, + "version": version, 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', + edr_req_id]}})) + # check that data not in self.edr_ids_queue self.assertEqual(mrequest.call_count, 6) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[5].url, u'127.0.0.1:80/api/1.0/verify?id=123') + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[5].url, self.urls('verify?id={}'.format(self.edr_ids))) @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_data_mock_403(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + edr_req_id = self.gen_req_id[0] edrpou_codes_queue = Queue(10) check_queue = Queue(10) expected_result = [] - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - worker.retry_edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, '123', "awards", + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) + self.worker.retry_edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, 'author': author, + "meta": {"detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result - worker.get_edr_data_request = MagicMock(side_effect=[ + self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=403)), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": {"errorDetails": "Couldn't find this code in EDR.", "code": u"notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, 'author': author - }}]}] - }))) - ]) + {"errors": + [{"description": + [{"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, + 'author': author + }}]}]})))]) for result in expected_result: self.assertEquals(check_queue.get(), result) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_data_mock_404(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + edr_req_id = self.gen_req_id[0] edrpou_codes_queue = Queue(10) check_queue = Queue(10) expected_result = [] - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - worker.retry_edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, '123', "awards", + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) + self.worker.retry_edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, 'author': author, + "meta": {"detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result - worker.get_edr_data_request = MagicMock(side_effect=[ + self.worker.get_edr_data_request = MagicMock(side_effect=[ RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": {"errorDetails": "Couldn't find this code in EDR.", "code": u"notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, 'author': author - }}]}] - }))) - ]) + {"errors": + [{"description": + [{"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, + 'author': author}}]}]})))]) for result in expected_result: self.assertEquals(check_queue.get(), result) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_get_edr_data_mock_exception(self, mrequest, gevent_sleep): """Accept 429 status code in first request with header 'Retry-After'""" gevent_sleep.side_effect = custom_sleep - edr_req_id = generate_request_id() - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') + edr_req_id = self.gen_req_id[0] edrpou_codes_queue = Queue(10) check_queue = Queue(10) expected_result = [] - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - worker.retry_edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, '123', "awards", + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) + self.worker.retry_edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, '123', "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, '123', "awards", file_content={"error": {"errorDetails": "Couldn't find this code in EDR.", "code": "notFound"}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, 'author': author, + "meta": {"detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, 'author': author, 'sourceRequests': [ 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # result - worker.get_edr_data_request = MagicMock(side_effect=[ + self.worker.get_edr_data_request = MagicMock(side_effect=[ Exception(), RetryException("test", MagicMock(status_code=404, headers={'X-Request-ID': edr_req_id}, json=MagicMock(return_value= - {"errors": - [{"description": - [{"error": { - "errorDetails": "Couldn't find this code in EDR.", - "code": u"notFound"}, - "meta": { - "sourceDate": "2017-04-25T11:56:36+00:00", - 'id': document_id, "version": version, - 'author': author - }}]}] - }))) - ]) + {"errors": + [{"description": + [{"error": { + "errorDetails": "Couldn't find this code in EDR.", + "code": u"notFound"}, + "meta": { + "detailsSourceDate": self.source_date, + 'id': self.document_id, "version": version, + 'author': author}}]}]})))]) for result in expected_result: self.assertEquals(check_queue.get(), result) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') @requests_mock.Mocker() @patch('gevent.sleep') def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): """Recieve 404 and not valid data (worker dies). Check that worker get up""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'data': [{}]}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_id[1]}}]) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', edr_req_id[0], edr_req_id[1]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_details_req_id = self.gen_req_id[0] + edr_req_id = self.gen_req_id[0:2] + mrequest.get(self.url, [{'json': {'data': [{}]}, + 'headers': {'X-Request-ID': edr_req_id[0]}}, # data contains dict, instead of list + self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url_id(321), [self.stat_200([{}], self.source_date, edr_details_req_id)]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) self.assertEqual(mrequest.call_count, 2) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id=123') + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -561,38 +429,21 @@ def test_job_get_edr_data_dead(self, mrequest, gevent_sleep): def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): """Accept dict instead of list in first response to /verify endpoint. Check that worker get up""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}]}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[2]}}]) + edr_req_id = self.gen_req_id[0:3] + mrequest.get(self.url, [self.stat_c(403, 0, '', edr_req_id[0]), + {'json': {'data': [{}]}, 'status_code': 200, + 'headers': {'X-Request-ID': edr_req_id[1]}}, + self.stat_200([{}], self.source_date, edr_req_id[2])]) edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], - edr_req_id[1], - edr_req_id[2]]}})) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, self.upload_to_doc_service_queue, MagicMock()) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1], edr_req_id[2]]))) self.assertEqual(mrequest.call_count, 3) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[2].url, u'127.0.0.1:80/api/1.0/verify?id=123') + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[2].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertIsNotNone(mrequest.request_history[2].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -600,41 +451,27 @@ def test_job_retry_get_edr_data_dead(self, mrequest, gevent_sleep): def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): """Accept 6 times errors in response while requesting /verify""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id(), generate_request_id(), - generate_request_id(), generate_request_id(), generate_request_id(), - generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[2]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[3]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[4]}}, - {'json': {'errors': [{'description': ''}]}, 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[5]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_id[6]}}]) + edr_req_id = self.gen_req_id[0:8] + mrequest.get(self.url, + [self.stat_c(403, 0, '', edr_req_id[0]), + self.stat_c(403, 0, '', edr_req_id[1]), + self.stat_c(403, 0, '', edr_req_id[2]), + self.stat_c(403, 0, '', edr_req_id[3]), + self.stat_c(403, 0, '', edr_req_id[4]), + self.stat_c(403, 0, '', edr_req_id[5]), + self.stat_200([{}], self.source_date, edr_req_id[6])]) edrpou_codes_queue = Queue(10) upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], edr_req_id[6]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[6]]))) self.assertEqual(mrequest.call_count, 7) # processing 7 requests - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') # check first url - self.assertEqual(mrequest.request_history[6].url, u'127.0.0.1:80/api/1.0/verify?id=123') # check 7th url + self.assertEqual(mrequest.request_history[0].url, + self.urls('verify?id={}'.format(self.edr_ids))) # check first url + self.assertEqual(mrequest.request_history[6].url, + self.urls('verify?id={}'.format(self.edr_ids))) # check 7th url self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) @requests_mock.Mocker() @@ -642,89 +479,69 @@ def test_retry_5_times_get_edr_data(self, mrequest, gevent_sleep): def test_timeout(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'errors': [{'description': [{u'message': u'Gateway Timeout Error'}]}]}, - 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}]) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], edr_req_id[1]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_details_req_id = [self.gen_req_id[0], self.document_id] + edr_req_id = self.gen_req_id[0:7] + mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], edr_req_id[0]), + self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], + edr_details_req_id[0]), + self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, + [edr_req_id[0], edr_req_id[1]]))) self.assertEqual(mrequest.call_count, 2) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @requests_mock.Mocker() @patch('gevent.sleep') def test_identifier_id_type(self, mrequest, gevent_sleep): """Create filter_tenders and edr_handler workers. Test when identifier.id is type int (not str).""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex bid_id = uuid.uuid4().hex - edr_req_id = generate_request_id() + edr_req_id = self.gen_req_id[0] # create queues filtered_tender_ids_queue = Queue(10) edrpou_codes_queue = Queue(10) upload_to_doc_service_queue = Queue(10) - filtered_tender_ids_queue.put(tender_id) + filtered_tender_ids_queue.put(self.tender_id) # create workers and responses client = MagicMock() client.request.return_value = ResponseMock({'X-Request-ID': 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220'}, - munchify({'prev_page': {'offset': '123'}, - 'next_page': {'offset': '1234'}, - 'data': {'status': "active.pre-qualification", - 'id': tender_id, - 'procurementMethodType': 'aboveThresholdEU', - 'awards': [{'id': award_id, - 'status': 'pending', - 'bid_id': bid_id, - 'suppliers': [{'identifier': { - 'scheme': 'UA-EDR', - 'id': 14360570} # int instead of str type - }]}, ]}})) - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id}}]) + munchify({'prev_page': {'offset': '123'}, + 'next_page': {'offset': '1234'}, + 'data': {'status': "active.pre-qualification", + 'id': self.tender_id, + 'procurementMethodType': 'aboveThresholdEU', + 'awards': [{'id': self.award_id, + 'status': 'pending', + 'bid_id': bid_id, + 'suppliers': [{'identifier': { + 'scheme': 'UA-EDR', + 'id': 14360570} + # int instead of str type + }]}, ]}})) + mrequest.get(self.url, [self.stat_200([{}], self.source_date, edr_req_id)]) filter_tenders_worker = FilterTenders.spawn(client, filtered_tender_ids_queue, edrpou_codes_queue, {}, {}) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) obj = upload_to_doc_service_queue.get() - self.assertEqual(obj.tender_id, tender_id) - self.assertEqual(obj.item_id, award_id) + self.assertEqual(obj.tender_id, self.tender_id) + self.assertEqual(obj.item_id, self.award_id) self.assertEqual(obj.code, '14360570') self.assertEqual(obj.item_name, 'awards') self.assertEqual(obj.file_content['data'], {}) - self.assertEqual(obj.file_content['meta']['sourceDate'], "2017-04-25T11:56:36+00:00") + self.assertEqual(obj.file_content['meta']['sourceDate'], self.source_date[0]) self.assertIsNotNone(obj.file_content['meta']['id']) self.assertEqual(obj.file_content['meta']['version'], version) self.assertEqual(obj.file_content['meta']['author'], author) - self.assertEqual(obj.file_content['meta']['sourceRequests'], ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', edr_req_id]) - - worker.shutdown() + self.assertEqual(obj.file_content['meta']['sourceRequests'], + ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', edr_req_id]) filter_tenders_worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) self.assertEqual(filtered_tender_ids_queue.qsize(), 0) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) @@ -733,71 +550,45 @@ def test_identifier_id_type(self, mrequest, gevent_sleep): def test_processing_items(self, mrequest, gevent_sleep): """Return list of objects from EDR. Check number of edr_ids in processing_items.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex qualification_id = uuid.uuid4().hex document_ids = [generate_doc_id(), generate_doc_id()] - edr_req_id = [generate_request_id(), generate_request_id()] - processing_items = {} - award_key = '{}_{}'.format(tender_id, award_id) - qualification_key = '{}_{}'.format(tender_id, qualification_id) - data_1 = Data(tender_id, award_id, '123', "awards", - {'data': {'id': 321}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": "{}.{}.{}".format(document_ids[0], 2, 1), - "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0]]}}) - data_2 = Data(tender_id, award_id, '123', "awards", - {'data': {'id': 322}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": "{}.{}.{}".format(document_ids[0], 2, 2), - "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0]]}}) - data_3 = Data(tender_id, qualification_id, '124', 'qualifications', - {'data': {'id': 321}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": "{}.{}.{}".format(document_ids[1], 3, 1), - "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[1]]}}) - data_4 = Data(tender_id, qualification_id, '124', 'qualifications', - {'data': {'id': 322}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": "{}.{}.{}".format(document_ids[1], 3, 2), - "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[1]]}}) - data_5 = Data(tender_id, qualification_id, '124', 'qualifications', - {'data': {'id': 323}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": "{}.{}.{}".format(document_ids[1], 3, 3), - "version": version, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[1]]}}) - - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'data': [{'id': 321}, {'id': 322}], - "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{'id': 321}, {'id': 322}, {'id': 323}], - "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}]) + edr_req_id = self.gen_req_id[0:2] + award_key = '{}_{}'.format(self.tender_id, self.award_id) + qualification_key = '{}_{}'.format(self.tender_id, qualification_id) + data_1 = Data(self.tender_id, self.award_id, '123', "awards", + self.file_con({}, document_ids[0], 2, 1, [edr_req_id[0]])) + data_2 = Data(self.tender_id, self.award_id, '123', "awards", + self.file_con({}, document_ids[0], 2, 2, [edr_req_id[0]])) + data_3 = Data(self.tender_id, qualification_id, '124', 'qualifications', + self.file_con({}, document_ids[1], 3, 1, [edr_req_id[1]])) + data_4 = Data(self.tender_id, qualification_id, '124', 'qualifications', + self.file_con({}, document_ids[1], 3, 2, [edr_req_id[1]])) + data_5 = Data(self.tender_id, qualification_id, '124', 'qualifications', + self.file_con({}, document_ids[1], 3, 3, [edr_req_id[1]])) + mrequest.get(self.url, [ + self.stat_200([{}, {}], ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00"], edr_req_id[0]), + self.stat_200([{}, {}, {}], + ["2017-04-25T11:56:36+00:00", "2017-04-25T11:56:36+00:00", + "2017-04-25T11:56:36+00:00"], edr_req_id[1])]) # create queues edrpou_codes_queue = Queue(10) upload_to_doc_service = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_ids[0], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - edrpou_codes_queue.put(Data(tender_id, qualification_id, '124', 'qualifications', - {'meta': {'id': document_ids[1], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, '123', "awards", + {'meta': {'id': document_ids[0], 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) + edrpou_codes_queue.put(Data(self.tender_id, qualification_id, '124', 'qualifications', + {'meta': {'id': document_ids[1], 'author': author, + 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service, processing_items) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, upload_to_doc_service, + self.processing_items) for data in [data_1, data_2, data_3, data_4, data_5]: self.assertEquals(upload_to_doc_service.get(), data) - - worker.shutdown() self.assertEqual(edrpou_codes_queue.qsize(), 0) - self.assertEqual(processing_items[award_key], 2) - self.assertEqual(processing_items[qualification_key], 3) + self.assertEqual(self.processing_items[award_key], 2) + self.assertEqual(self.processing_items[qualification_key], 3) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) @@ -805,134 +596,83 @@ def test_processing_items(self, mrequest, gevent_sleep): @patch('gevent.sleep') def test_wrong_ip(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'errors': [{'description': [{u'message': u'Content-Type of EDR API response is not application/json'}]}]}, - 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}]) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, {}) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], edr_req_id[1]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_req_id = self.gen_req_id[0:2] + edr_details_req_id = self.gen_req_id[0:2] + mrequest.get(self.url, [self.stat_c(403, 0, [{u'message': u'Forbidden'}], edr_req_id[0]), + self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url_id(321), [self.stat_c(403, 0, [{u'message': u'Forbidden'}], edr_details_req_id[0]), + self.stat_200([{}], self.source_date, edr_details_req_id[1])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) self.assertEqual(mrequest.call_count, 2) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) @requests_mock.Mocker() @patch('gevent.sleep') def test_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for edrpou_codes_queue """ - document_ids = [generate_doc_id(), generate_doc_id()] + document_ids = [self.document_id, self.document_id] gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_id = [generate_request_id(), generate_request_id()] - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, 'status_code': 200, - 'headers': {'X-Request-ID': edr_req_id[1]}}]) - + edr_req_id = self.gen_req_id[0:2] + edr_details_req_id = self.gen_req_id[0:2] + mrequest.get(self.uri, + [self.stat_200([{}], self.source_date, edr_req_id[0]), + self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url_id(self.local_edr_ids[0]), + [self.stat_200([{}], self.source_date, edr_details_req_id[0])]) + mrequest.get(self.url_id(self.local_edr_ids[1]), + [self.stat_200([{}], self.source_date, edr_details_req_id[1])]) edrpou_codes_queue = MagicMock() - check_queue = Queue(10) - expected_result = [] edrpou_codes_queue_list = [LoopExit()] for i in range(2): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue_list.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_ids[i], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", "id": document_ids[i], - "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[i]]}})) # result - - edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, - default=LoopExit()) - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + {'meta': {'id': document_ids[i], 'author': author, 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data + expected_result.append( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, document_ids[i], 1, 1, [edr_req_id[i]]))) # result + edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, default=LoopExit()) + self.worker.retry_edrpou_codes_queue = MagicMock() + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, self.upload_to_doc_service_queue, + MagicMock()) for result in expected_result: - self.assertEquals(check_queue.get(), result) - - self.assertEqual(mrequest.request_history[0].url,u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[1].code)) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(expected_result[0].code)) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) - - worker.shutdown() self.assertEqual(mrequest.call_count, 2) @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for retry_edrpou_codes_queue """ - document_ids = [generate_doc_id(), generate_doc_id()] + document_ids = [self.document_id, self.document_id] gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_id = [generate_request_id(), generate_request_id()] - mrequest.get("{uri}".format(uri=proxy_client.verify_url), - [{'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[0]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}}]) - - edrpou_codes_queue = Queue(1) - check_queue = Queue(10) + edr_req_id = self.gen_req_id[0:2] + edr_details_req_id = self.gen_req_id[0:2] + mrequest.get(self.uri, [self.stat_200([{}], self.source_date, edr_req_id[0]), + self.stat_200([{}], self.source_date, edr_req_id[1])]) + mrequest.get(self.url_id(self.local_edr_ids[0]), [self.stat_200([{}], self.source_date, edr_details_req_id[0])]) + mrequest.get(self.url_id(self.local_edr_ids[1]), [self.stat_200([{}], self.source_date, edr_details_req_id[1])]) edrpou_codes_queue_list = [LoopExit()] - expected_result = [] for i in range(2): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(2)] - edrpou_codes_queue_list.append(Data(tender_id, award_id, edr_ids[i], "awards", - {"meta": {"id": document_ids[i], 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'data': {}, - "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", "id": document_ids[i], - "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[i]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - worker.retry_edrpou_codes_queue = MagicMock() - worker.retry_edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, - default=LoopExit()) - + edrpou_codes_queue_list.append(Data(self.tender_id, self.award_id, self.edr_ids, "awards", + {"meta": {"id": document_ids[i], 'author': author, 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data + expected_result.append( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", + self.file_con({}, document_ids[i], 1, 1, [edr_req_id[i]]))) # result + self.worker.retry_edrpou_codes_queue = MagicMock() + self.worker.retry_edrpou_codes_queue.peek.side_effect = generate_answers(answers=edrpou_codes_queue_list, + default=LoopExit()) for result in expected_result: - self.assertEquals(check_queue.get(), result) - - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) - self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[1].code)) + self.assertEquals(self.upload_to_doc_service_queue.get(), result) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}').format(expected_result[0].code)) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) - - worker.shutdown() self.assertEqual(mrequest.call_count, 2) @requests_mock.Mocker() @@ -940,50 +680,33 @@ def test_retry_edrpou_codes_queue_loop_exit(self, mrequest, gevent_sleep): def test_retry_exception(self, mrequest, gevent_sleep): """ Raise RetryException in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_ids = [generate_request_id(), generate_request_id()] + edr_req_ids = self.gen_req_id[0:2] retry_response = MagicMock() retry_response.status_code = 500 - mrequest.get("{url}".format(url=proxy_client.verify_url), + mrequest.get(self.url, [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'exc': RetryException('Retry Exception', retry_response)}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, + {'json': {'data': [{}], "meta": {"detailsSourceDate": self.source_date}}, 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) - edrpou_codes_queue = Queue(10) check_queue = Queue(10) - expected_result = [] for i in range(1): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(1)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {"data": {}, 'meta': {'sourceDate': '2017-04-25T11:56:36+00:00', - 'id': document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", + self.file_con({}, self.document_id, 1, 1, + [edr_req_ids[0], edr_req_ids[1]]))) # result + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) for result in expected_result: self.assertEquals(check_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[6].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) + self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(mrequest.call_count, 7) @requests_mock.Mocker() @@ -991,15 +714,14 @@ def test_retry_exception(self, mrequest, gevent_sleep): def test_retry_exception_404(self, mrequest, gevent_sleep): """ Raise RetryException in retry_get_edr_data with status_code 404""" gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_ids = generate_request_id() + edr_req_ids = self.gen_req_id[0] retry_response = MagicMock() retry_response.status_code = 404 retry_response.json = MagicMock() retry_response.json.return_value = {'errors': [ {'description': [{'error': {"errorDetails": "Couldn't find this code in EDR.", 'code': "notFound"}, - 'meta': {"sourceDate": "2017-04-25T11:56:36+00:00"}}]}]} - mrequest.get("{url}".format(url=proxy_client.verify_url), + 'meta': {"detailsSourceDate": self.source_date}}]}]} + mrequest.get(self.url, [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids}}, @@ -1012,30 +734,22 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): expected_result = [] for i in range(1): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(1)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {'error': {'errorDetails': "Couldn't find this code in EDR.", 'code': 'notFound'}, - 'meta': {'sourceDate': '2017-04-25T11:56:36+00:00', - 'id': document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids]}})) # result + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", + {'error': {'errorDetails': "Couldn't find this code in EDR.", + 'code': 'notFound'}, + 'meta': {'detailsSourceDate': self.source_date, + 'id': self.document_id, "version": version, 'author': author, + 'sourceRequests': [ + 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', + edr_req_ids]}})) # result - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) for result in expected_result: self.assertEquals(check_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(mrequest.call_count, 6) @requests_mock.Mocker() @@ -1043,48 +757,31 @@ def test_retry_exception_404(self, mrequest, gevent_sleep): def test_exception(self, mrequest, gevent_sleep): """ Raise Exception in retry_get_edr_data""" gevent_sleep.side_effect = custom_sleep - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edr_req_ids = [generate_request_id(), generate_request_id()] - mrequest.get("{url}".format(url=proxy_client.verify_url), + edr_req_ids = self.gen_req_id[0:2] + mrequest.get(self.url, [{'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, {'status_code': 500, 'headers': {'X-Request-ID': edr_req_ids[0]}}, - {'exc': Exception()}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_ids[1]}}]) + {'exc': Exception()}, self.stat_200([{}], self.source_date, edr_req_ids[1])]) edrpou_codes_queue = Queue(10) check_queue = Queue(10) expected_result = [] for i in range(1): - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_ids = [str(random.randrange(10000000, 99999999)) for _ in range(1)] - edrpou_codes_queue.put(Data(tender_id, award_id, edr_ids[i], "awards", - {'meta': {'id': document_id, 'author': author, - 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) # data - expected_result.append(Data(tender_id, award_id, edr_ids[i], "awards", - {"data": {}, 'meta': {'sourceDate': '2017-04-25T11:56:36+00:00', - 'id': document_id, "version": version, 'author': author, - 'sourceRequests': [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_ids[0], edr_req_ids[1]]}})) # result - - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, check_queue, MagicMock()) - + edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", self.meta())) # data + expected_result.append(Data(self.tender_id, self.award_id, self.edr_ids[i], "awards", + self.file_con({}, self.document_id, 1, 1, + [edr_req_ids[0], edr_req_ids[1]]))) # result + self.worker = EdrHandler.spawn(self.proxy_client, edrpou_codes_queue, check_queue, MagicMock()) for result in expected_result: self.assertEquals(check_queue.get(), result) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) - self.assertEqual(mrequest.request_history[6].url, u'127.0.0.1:80/api/1.0/verify?id={edr_code}'.format(edr_code=expected_result[0].code)) + self.assertEqual(mrequest.request_history[6].url, self.urls('verify?id={}'.format(expected_result[0].code))) self.assertIsNotNone(mrequest.request_history[6].headers['X-Client-Request-ID']) - - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0, 'Queue must be empty') self.assertEqual(mrequest.call_count, 7) @requests_mock.Mocker() @@ -1092,40 +789,23 @@ def test_exception(self, mrequest, gevent_sleep): def test_value_error(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - mrequest.get("{url}".format(url=proxy_client.verify_url), - [{'json': {'errors': [{'description': [{u'message': u'Gateway Timeout Error'}]}]}, - 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[0]}}, + edr_req_id = self.gen_req_id[0:2] + mrequest.get(self.url, + [self.stat_c(403, 0, [{u'message': u'Gateway Timeout Error'}], edr_req_id[0]), {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, {"text": "resp", 'status_code': 403, 'headers': {'X-Request-ID': edr_req_id[1]}}, - {'json': {'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}}, - 'status_code': 200, 'headers': {'X-Request-ID': edr_req_id[1]}} - ]) - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], edr_req_id[1]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + self.stat_200([{}], self.source_date, edr_req_id[1])]) + self.edrpou_codes_queue.put(Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker.retry_edr_ids_queue = MagicMock() + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) self.assertEqual(mrequest.call_count, 7) - self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/api/1.0/verify?id=123') - self.assertEqual(mrequest.request_history[1].url, u'127.0.0.1:80/api/1.0/verify?id=123') + self.assertEqual(mrequest.request_history[0].url, self.urls('verify?id={}'.format(self.edr_ids))) + self.assertEqual(mrequest.request_history[1].url, self.urls('verify?id={}'.format(self.edr_ids))) self.assertIsNotNone(mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone(mrequest.request_history[1].headers['X-Client-Request-ID']) @@ -1134,57 +814,35 @@ def test_value_error(self, mrequest, gevent_sleep): def test_value_error_mock(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - bid_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - worker.retry_edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker.get_edr_data_request = MagicMock(side_effect=[RetryException("Exception", Response()), - MagicMock(headers={'X-Request-ID': edr_req_id[0]}, status_code=200, - json=MagicMock(return_value=munchify({'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}})))]) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_req_id = self.gen_req_id[0:2] + self.worker.retry_edr_ids_queue = MagicMock() + self.worker.retry_edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker.get_edr_data_request = MagicMock(side_effect=[RetryException("Exception", Response()), + MagicMock(headers={'X-Request-ID': edr_req_id[0]}, + status_code=200, + json=MagicMock(return_value=munchify( + {'data': [{}], "meta": { + "detailsSourceDate": + self.source_date}})))]) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0]]))) @requests_mock.Mocker() @patch('gevent.sleep') def test_429_mock(self, mrequest, gevent_sleep): """Accept 'Gateway Timeout Error' while requesting /verify, then accept 200 status code.""" gevent_sleep.side_effect = custom_sleep - tender_id = uuid.uuid4().hex - award_id = uuid.uuid4().hex - bid_id = uuid.uuid4().hex - document_id = generate_doc_id() - edr_req_id = [generate_request_id(), generate_request_id()] - proxy_client = ProxyClient(host='127.0.0.1', port='80', user='', password='') - edrpou_codes_queue = Queue(10) - upload_to_doc_service_queue = Queue(10) - worker = EdrHandler.spawn(proxy_client, edrpou_codes_queue, upload_to_doc_service_queue, MagicMock()) - worker.retry_edrpou_codes_queue.put(Data(tender_id, award_id, '123', "awards", - {'meta': {'id': document_id, 'author': author, 'sourceRequests': ['req-db3ed1c6-9843-415f-92c9-7d4b08d39220']}})) - worker.get_edr_data_request = MagicMock(side_effect=[MagicMock(headers={'X-Request-ID': edr_req_id[0], 'Retry-After': '1'}, status_code=429), - MagicMock(headers={'X-Request-ID': edr_req_id[1]}, status_code=200, - json=MagicMock(return_value=munchify({'data': [{}], "meta": {"detailsSourceDate": ["2017-04-25T11:56:36+00:00"]}})))]) - self.assertEquals(upload_to_doc_service_queue.get(), - Data(tender_id=tender_id, item_id=award_id, - code='123', item_name='awards', - file_content={'data': {}, "meta": {"sourceDate": "2017-04-25T11:56:36+00:00", - "id": document_id, "version": version, 'author': author, - "sourceRequests": [ - 'req-db3ed1c6-9843-415f-92c9-7d4b08d39220', - edr_req_id[0], edr_req_id[1]]}})) - worker.shutdown() - self.assertEqual(edrpou_codes_queue.qsize(), 0) + edr_req_id = self.gen_req_id[0:2] + self.worker.retry_edr_ids_queue = MagicMock() + self.worker.retry_edrpou_codes_queue.put( + Data(self.tender_id, self.award_id, self.edr_ids, "awards", self.meta())) + self.worker.get_edr_data_request = MagicMock( + side_effect=[MagicMock(headers={'X-Request-ID': edr_req_id[0], 'Retry-After': '1'}, status_code=429), + MagicMock(headers={'X-Request-ID': edr_req_id[1]}, status_code=200, + json=MagicMock(return_value=munchify( + {'data': [{}], "meta": {"detailsSourceDate": self.source_date}})))]) + self.assertEquals(self.upload_to_doc_service_queue.get(), + Data(self.tender_id, self.award_id, self.edr_ids, 'awards', + self.file_con({}, self.document_id, 1, 1, [edr_req_id[0], edr_req_id[1]]))) From cac173c3732259d5ac45c335230a212f2a4cb515 Mon Sep 17 00:00:00 2001 From: Shevchenko Vladislav Date: Fri, 21 Jul 2017 09:28:19 +0300 Subject: [PATCH 4/7] Solution of issue --- .../identification/databridge/upload_file.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index c3ea6ca..a789191 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -8,8 +8,7 @@ from datetime import datetime from gevent import Greenlet, spawn from gevent.hub import LoopExit -from restkit import ResourceError -import re as regex +from restkit import ResourceError, Unauthorized from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, data_string from openprocurement.bot.identification.databridge.journal_msg_ids import ( @@ -28,7 +27,6 @@ class UploadFile(Greenlet): pre_qualification_procurementMethodType = ('aboveThresholdEU', 'competitiveDialogueUA', 'competitiveDialogueEU') qualification_procurementMethodType = ('aboveThresholdUA', 'aboveThresholdUA.defense', 'aboveThresholdEU', 'competitiveDialogueUA.stage2', 'competitiveDialogueEU.stage2') sleep_change_value = 0 - start_of_retry_uploading_error_message = "Can't add document" def __init__(self, client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, processed_items, doc_service_client, increment_step=1, decrement_step=1, delay=15): super(UploadFile, self).__init__() @@ -239,7 +237,7 @@ def retry_upload_to_tender(self): item_name_id = tender_data.item_name[:-1].upper() + "_ID" try: self.client_upload_to_tender(tender_data) - except ResourceError as re: + except (ResourceError, Unauthorized(http_code=401)) as re: if re.status_int == 422: # WARNING and don't retry logger.warning("Accept 422, skip {} doc_id: {}. Message {}".format(data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, @@ -254,7 +252,7 @@ def retry_upload_to_tender(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step - elif re.status_int == 403 or self.start_of_retry_uploading_error_message in re.message["errors"][0]["description"]: + elif re.status_int == 403: logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format( data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, @@ -264,6 +262,17 @@ def retry_upload_to_tender(self): self.retry_upload_to_tender_queue.get() UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 continue + elif re.status_int == 401: + logger.warning("Accept 401 while uploading to {} doc_id: {}. Message {}".format( + data_string(tender_data), document_id, re.msg), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id}) + ) + self.update_processing_items(tender_data.tender_id, tender_data.item_id) + self.retry_upload_to_tender_queue.get() + UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 + continue else: logger.info('ResourceError while retry uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format( data_string(tender_data), document_id, re.status_int, re.message), From f4121f372ce1f6e81ce5165f9c2be165c46baa45 Mon Sep 17 00:00:00 2001 From: Shevchenko Vladislav Date: Fri, 21 Jul 2017 17:55:23 +0300 Subject: [PATCH 5/7] Fixed issue with Unauthorized error --- .../identification/databridge/upload_file.py | 24 +++++++-------- .../bot/identification/tests/upload_file.py | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index a789191..d1ab45f 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -166,6 +166,17 @@ def upload_to_tender(self): {'data': document_data}, '{}/{}/documents'.format(tender_data.item_name, tender_data.item_id)) + except Unauthorized as u: + if u.status_int == None: + logger.warning("Unauthorized while uploading to {} doc_id: {}. Message {}".format( + data_string(tender_data), document_id, u.msg), + extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, + {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id})) + self.update_processing_items(tender_data.tender_id, tender_data.item_id) + UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 + self.upload_to_tender_queue.get() + continue except ResourceError as re: if re.status_int == 422: # WARNING and don't retry logger.warning("Accept 422, skip {} doc_id: {}. Message: {}".format(data_string(tender_data), document_id, re.msg), @@ -237,7 +248,7 @@ def retry_upload_to_tender(self): item_name_id = tender_data.item_name[:-1].upper() + "_ID" try: self.client_upload_to_tender(tender_data) - except (ResourceError, Unauthorized(http_code=401)) as re: + except ResourceError as re: if re.status_int == 422: # WARNING and don't retry logger.warning("Accept 422, skip {} doc_id: {}. Message {}".format(data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_422_UPLOAD_TO_TENDER}, @@ -262,17 +273,6 @@ def retry_upload_to_tender(self): self.retry_upload_to_tender_queue.get() UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 continue - elif re.status_int == 401: - logger.warning("Accept 401 while uploading to {} doc_id: {}. Message {}".format( - data_string(tender_data), document_id, re.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id}) - ) - self.update_processing_items(tender_data.tender_id, tender_data.item_id) - self.retry_upload_to_tender_queue.get() - UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 - continue else: logger.info('ResourceError while retry uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format( data_string(tender_data), document_id, re.status_int, re.message), diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index e77dc49..1c81be3 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -225,6 +225,35 @@ def test_upload_to_tender_exception(self, mrequest, gevent_sleep): self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(worker.sleep_change_value, 0) + @requests_mock.Mocker() + @patch('gevent.sleep') + def test_upload_to_tender_exception_unauthorized(self, mrequest, gevent_sleep): + gevent_sleep.side_effect = custom_sleep + doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') + tender_id = uuid.uuid4().hex + award_id = uuid.uuid4().hex + document_id = generate_doc_id() + key = '{}_{}'.format(tender_id, award_id) + processing_items = {key: 1} + processed_items = {} + upload_to_doc_service_queue = Queue(10) + upload_to_tender_queue = Queue(10) + upload_to_tender_queue.put(Data(tender_id, award_id, '123', 'awards', + {'meta': {'id': document_id}, 'test_data': 'test_data'})) + client = MagicMock() + client._create_tender_resource_item = MagicMock(side_effect=[Unauthorized()]) + worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, + processed_items, + doc_service_client) + worker.client = client + worker.client_upload_to_tender = MagicMock(side_effect=Unauthorized()) + while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or + worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): + sleep(1) # sleep while at least one queue is not empty + worker.shutdown() + self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty') + self.assertEqual(worker.sleep_change_value, 0) + @requests_mock.Mocker() @patch('gevent.sleep') def test_retry_upload_to_tender(self, mrequest, gevent_sleep): From 681b537adc137fdff29300ca49c20c650baffad2 Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Sun, 23 Jul 2017 15:47:57 +0300 Subject: [PATCH 6/7] Rewrited error handling --- .../identification/databridge/upload_file.py | 20 ++++++++----------- .../bot/identification/tests/upload_file.py | 5 ++--- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index d1ab45f..fde1365 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -166,17 +166,6 @@ def upload_to_tender(self): {'data': document_data}, '{}/{}/documents'.format(tender_data.item_name, tender_data.item_id)) - except Unauthorized as u: - if u.status_int == None: - logger.warning("Unauthorized while uploading to {} doc_id: {}. Message {}".format( - data_string(tender_data), document_id, u.msg), - extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, - "DOCUMENT_ID": document_id})) - self.update_processing_items(tender_data.tender_id, tender_data.item_id) - UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 - self.upload_to_tender_queue.get() - continue except ResourceError as re: if re.status_int == 422: # WARNING and don't retry logger.warning("Accept 422, skip {} doc_id: {}. Message: {}".format(data_string(tender_data), document_id, re.msg), @@ -204,6 +193,13 @@ def upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step + elif re.status_int is None: + logger.info(re.__dict__) + logger.info(re.__dict__['msg']) + self.update_processing_items(tender_data.tender_id, tender_data.item_id) + UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 + self.upload_to_tender_queue.get() + continue else: logger.warning('ResourceError while uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, re.message), @@ -296,7 +292,7 @@ def retry_upload_to_tender(self): UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 gevent.sleep(UploadFile.sleep_change_value) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) def client_upload_to_tender(self, tender_data): """Process upload to tender request for retry queue objects.""" document_data = tender_data.file_content.get('data', {}) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index 1c81be3..a8594ca 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -227,7 +227,7 @@ def test_upload_to_tender_exception(self, mrequest, gevent_sleep): @requests_mock.Mocker() @patch('gevent.sleep') - def test_upload_to_tender_exception_unauthorized(self, mrequest, gevent_sleep): + def test_upload_to_tender_exception_status_int_none(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') tender_id = uuid.uuid4().hex @@ -243,8 +243,7 @@ def test_upload_to_tender_exception_unauthorized(self, mrequest, gevent_sleep): client = MagicMock() client._create_tender_resource_item = MagicMock(side_effect=[Unauthorized()]) worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, - processed_items, - doc_service_client) + processed_items, doc_service_client) worker.client = client worker.client_upload_to_tender = MagicMock(side_effect=Unauthorized()) while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or From 06e4163c964bb2824a9c202bf1c8f21497dc6a2c Mon Sep 17 00:00:00 2001 From: Shevchenko Vlad Date: Sun, 23 Jul 2017 16:14:42 +0300 Subject: [PATCH 7/7] Removal of unnecessary --- .../identification/databridge/upload_file.py | 26 +++++++------------ .../bot/identification/tests/upload_file.py | 1 - 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/openprocurement/bot/identification/databridge/upload_file.py b/openprocurement/bot/identification/databridge/upload_file.py index fde1365..628e0fa 100644 --- a/openprocurement/bot/identification/databridge/upload_file.py +++ b/openprocurement/bot/identification/databridge/upload_file.py @@ -8,7 +8,7 @@ from datetime import datetime from gevent import Greenlet, spawn from gevent.hub import LoopExit -from restkit import ResourceError, Unauthorized +from restkit import ResourceError from openprocurement.bot.identification.databridge.utils import journal_context, Data, create_file, data_string from openprocurement.bot.identification.databridge.journal_msg_ids import ( @@ -176,9 +176,9 @@ def upload_to_tender(self): self.upload_to_tender_queue.get() UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 continue - elif re.status_int == 403: - logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format( - data_string(tender_data), document_id, re.msg), + elif re.status_int == 403 or re.status_int is None: + logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( + re.status_int, data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id}) @@ -193,13 +193,6 @@ def upload_to_tender(self): {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step - elif re.status_int is None: - logger.info(re.__dict__) - logger.info(re.__dict__['msg']) - self.update_processing_items(tender_data.tender_id, tender_data.item_id) - UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 - self.upload_to_tender_queue.get() - continue else: logger.warning('ResourceError while uploading file to {} doc_id: {}. Message: {}'.format( data_string(tender_data), document_id, re.message), @@ -259,11 +252,12 @@ def retry_upload_to_tender(self): extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})) UploadFile.sleep_change_value += self.increment_step - elif re.status_int == 403: - logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format( - data_string(tender_data), document_id, re.msg), + elif re.status_int == 403 or re.status_int is None: + logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format( + re.status_int, data_string(tender_data), document_id, re.msg), extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING}, - {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id}) + {"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, + "DOCUMENT_ID": document_id}) ) self.update_processing_items(tender_data.tender_id, tender_data.item_id) self.retry_upload_to_tender_queue.get() @@ -292,7 +286,7 @@ def retry_upload_to_tender(self): UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0 gevent.sleep(UploadFile.sleep_change_value) - @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1) + @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def client_upload_to_tender(self, tender_data): """Process upload to tender request for retry queue objects.""" document_data = tender_data.file_content.get('data', {}) diff --git a/openprocurement/bot/identification/tests/upload_file.py b/openprocurement/bot/identification/tests/upload_file.py index a8594ca..11d3ce2 100644 --- a/openprocurement/bot/identification/tests/upload_file.py +++ b/openprocurement/bot/identification/tests/upload_file.py @@ -245,7 +245,6 @@ def test_upload_to_tender_exception_status_int_none(self, mrequest, gevent_sleep worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items, processed_items, doc_service_client) worker.client = client - worker.client_upload_to_tender = MagicMock(side_effect=Unauthorized()) while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()): sleep(1) # sleep while at least one queue is not empty