Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Error correction #94

Merged
merged 9 commits into from
Jul 25, 2017
Merged

Error correction #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
11978e3
Handled ResourceError in upload_file.py
Jul 18, 2017
b198b9e
Handled ResourceError status_int as 403
Jul 19, 2017
e5ef5ef
Merging
Jul 19, 2017
cac173c
Solution of issue
Jul 21, 2017
f4121f3
Fixed issue with Unauthorized error
Jul 21, 2017
abb745e
Merge branch 'master' into dev_error_correction
Jul 21, 2017
681b537
Rewrited error handling
Jul 23, 2017
06e4163
Removal of unnecessary
Jul 23, 2017
21159cc
Merge pull request #59 from ITVaan/dev_error_correction
Jul 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions openprocurement/bot/identification/databridge/upload_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ def upload_to_tender(self):
self.upload_to_tender_queue.get()
UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0
continue
elif re.status_int == 403:
logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format(
data_string(tender_data), document_id, re.msg),
elif re.status_int == 403 or re.status_int is None:
logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format(
re.status_int, data_string(tender_data), document_id, re.msg),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING},
{"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id,
"DOCUMENT_ID": document_id})
Expand All @@ -194,7 +194,7 @@ def upload_to_tender(self):
"DOCUMENT_ID": document_id}))
UploadFile.sleep_change_value += self.increment_step
else:
logger.warning('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format(
logger.warning('ResourceError while uploading file to {} doc_id: {}. Message: {}'.format(
data_string(tender_data), document_id, re.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER},
params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id,
Expand All @@ -203,7 +203,6 @@ def upload_to_tender(self):
self.upload_to_tender_queue.get()
UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0
except Exception as e:

logger.info('Exception while uploading file to {} doc_id: {}. Message: {}'.format(
data_string(tender_data), document_id, e.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER},
Expand Down Expand Up @@ -253,19 +252,20 @@ def retry_upload_to_tender(self):
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING},
{"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id}))
UploadFile.sleep_change_value += self.increment_step
elif re.status_int == 403:
logger.warning("Accept 403 while uploading to {} doc_id: {}. Message {}".format(
data_string(tender_data), document_id, re.msg),
elif re.status_int == 403 or re.status_int is None:
logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format(
re.status_int, data_string(tender_data), document_id, re.msg),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING},
{"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id})
{"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id,
"DOCUMENT_ID": document_id})
)
self.update_processing_items(tender_data.tender_id, tender_data.item_id)
self.retry_upload_to_tender_queue.get()
UploadFile.sleep_change_value = UploadFile.sleep_change_value - self.decrement_step if self.decrement_step < UploadFile.sleep_change_value else 0
continue
else:
logger.info('Exception while retry uploading file to {} doc_id: {}. Message: {}'.format(
data_string(tender_data), document_id, re.message),
logger.info('ResourceError while retry uploading file to {} doc_id: {}. ResourceErrorStatus_int: {}. Message: {}'.format(
data_string(tender_data), document_id, re.status_int, re.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_RETRY_UPLOAD_TO_TENDER},
params={"TENDER_ID": tender_data.tender_id, item_name_id: tender_data.item_id, "DOCUMENT_ID": document_id}))
except Exception as e:
Expand Down
27 changes: 27 additions & 0 deletions openprocurement/bot/identification/tests/upload_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,33 @@ def test_upload_to_tender_exception(self, mrequest, gevent_sleep):
self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty')
self.assertEqual(worker.sleep_change_value, 0)

@requests_mock.Mocker()
@patch('gevent.sleep')
def test_upload_to_tender_exception_status_int_none(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='')
tender_id = uuid.uuid4().hex
award_id = uuid.uuid4().hex
document_id = generate_doc_id()
key = '{}_{}'.format(tender_id, award_id)
processing_items = {key: 1}
processed_items = {}
upload_to_doc_service_queue = Queue(10)
upload_to_tender_queue = Queue(10)
upload_to_tender_queue.put(Data(tender_id, award_id, '123', 'awards',
{'meta': {'id': document_id}, 'test_data': 'test_data'}))
client = MagicMock()
client._create_tender_resource_item = MagicMock(side_effect=[Unauthorized()])
worker = UploadFile.spawn(client, upload_to_doc_service_queue, upload_to_tender_queue, processing_items,
processed_items, doc_service_client)
worker.client = client
while (upload_to_doc_service_queue.qsize() or upload_to_tender_queue.qsize() or
worker.retry_upload_to_doc_service_queue.qsize() or worker.retry_upload_to_tender_queue.qsize()):
sleep(1) # sleep while at least one queue is not empty
worker.shutdown()
self.assertEqual(upload_to_tender_queue.qsize(), 0, 'Queue should be empty')
self.assertEqual(worker.sleep_change_value, 0)

@requests_mock.Mocker()
@patch('gevent.sleep')
def test_retry_upload_to_tender(self, mrequest, gevent_sleep):
Expand Down