Commit ffaef951 authored by Thomas Matthews's avatar Thomas Matthews

Merge branch 'timeout_fix' into 'development'

added retry and backoff to timeout(http408) errors

See merge request !61
parents e7b68031 8251c82d
Pipeline #7973 passed with stage
in 8 minutes and 8 seconds
......@@ -11,6 +11,7 @@ import logging
import threading
from rauth import OAuth2Service
from requests.adapters import HTTPAdapter
from requests.exceptions import HTTPError as request_HTTPError
from Model.Project import Project
......@@ -23,6 +24,9 @@ from Validation.offlineValidation import validate_URL_form
from API.pubsub import send_message
HTTP_MAX_RETRIES = 5
HTTP_BACKOFF_FACTOR = 1
class ApiCalls(object):
_instance = None
......@@ -110,7 +114,9 @@ class ApiCalls(object):
logging.debug("Token is probably expired, going to get a new session.")
oauth_service = self.get_oauth_service()
access_token = self.get_access_token(oauth_service)
self._session = oauth_service.get_session(access_token)
new_session = oauth_service.get_session(access_token)
self._session = self.add_timeout_backoff(new_session)
finally:
self._session_lock.release()
......@@ -128,19 +134,47 @@ class ApiCalls(object):
returns session (OAuth2Session object)
"""
self.cached_projects = None
# set http backoff option
self.http_max_retries = HTTP_MAX_RETRIES
self.http_backoff_factor = HTTP_BACKOFF_FACTOR
if self.base_URL[-1:] != "/":
self.base_URL = self.base_URL + "/"
if validate_URL_form(self.base_URL):
oauth_service = self.get_oauth_service()
access_token = self.get_access_token(oauth_service)
self._session = oauth_service.get_session(access_token)
new_session = oauth_service.get_session(access_token)
self._session = self.add_timeout_backoff(new_session)
if self.validate_URL_existence(self.base_URL, use_session=True) is False:
raise Exception("Cannot create session. Verify your credentials are correct.")
else:
raise URLError(self.base_URL + " is not a valid URL")
def add_timeout_backoff(self, new_session):
# method stolen from https://www.programcreek.com/python/example/102997/requests.adapters example 3
# Adds a retry counter and backoff to requests that timeout
try:
# Some older versions of requests to not have the urllib3
# vendorized package
from requests.packages.urllib3.util.retry import Retry
except ImportError:
retries = self.http_max_retries
else:
# use a requests session to reuse connections between requests
retries = Retry(
total=self.http_max_retries,
read=self.http_max_retries,
backoff_factor=self.http_backoff_factor,
status_forcelist=[408, 504, 522, 524]
)
new_session.mount('https://', HTTPAdapter(max_retries=retries))
new_session.mount('http://', HTTPAdapter(max_retries=retries))
return new_session
def get_oauth_service(self):
"""
get oauth service to be used to get access token
......@@ -713,7 +747,7 @@ class ApiCalls(object):
e = SequenceFileError("Error {status_code}: {err_msg}\n".format(
status_code=str(response.status_code),
err_msg=response.reason))
logging.info("Got an error when uploading [{}]: [{}]".format(sample.get_id(), err_msg))
logging.info("Got an error when uploading [{}]: [{}]".format(sample.get_id(), e))
logging.info(response.text)
send_message(sample.upload_failed_topic, exception=e)
raise e
......
......@@ -102,6 +102,7 @@ class TestApiCalls(unittest.TestCase):
self.assertEqual(is_valid, valid)
API.apiCalls.urlopen.assert_called_with(url, timeout=api.max_wait_time)
@patch("API.apiCalls.ApiCalls.add_timeout_backoff")
@patch("API.apiCalls.ApiCalls.validate_URL_existence")
@patch("API.apiCalls.ApiCalls.get_access_token")
@patch("API.apiCalls.ApiCalls.get_oauth_service")
......@@ -109,7 +110,7 @@ class TestApiCalls(unittest.TestCase):
def test_create_session_valid_base_url_no_slash(
self, mock_validate_url_form,
mock_get_oauth_service, mock_get_access_token,
mock_validate_url_existence):
mock_validate_url_existence, mock_add_timeout_backoff):
oauth_service = Foo()
access_token = Foo()
......@@ -132,6 +133,7 @@ class TestApiCalls(unittest.TestCase):
mock_validate_url_existence.assert_called_with(
base_URL1 + "/", use_session=True)
@patch("API.apiCalls.ApiCalls.add_timeout_backoff")
@patch("API.apiCalls.ApiCalls.validate_URL_existence")
@patch("API.apiCalls.ApiCalls.get_access_token")
@patch("API.apiCalls.ApiCalls.get_oauth_service")
......@@ -139,7 +141,7 @@ class TestApiCalls(unittest.TestCase):
def test_create_session_valid_base_url_slash(
self, mock_validate_url_form,
mock_get_oauth_service, mock_get_access_token,
mock_validate_url_existence):
mock_validate_url_existence, mock_add_timeout_backoff):
oauth_service = Foo()
access_token = Foo()
......@@ -163,6 +165,7 @@ class TestApiCalls(unittest.TestCase):
base_URL2, use_session=True)
# This test validates that the api is a singleton, and does not make extra requests when re-init with same params
@patch("API.apiCalls.ApiCalls.add_timeout_backoff")
@patch("API.apiCalls.ApiCalls.validate_URL_existence")
@patch("API.apiCalls.ApiCalls.get_access_token")
@patch("API.apiCalls.ApiCalls.get_oauth_service")
......@@ -170,7 +173,8 @@ class TestApiCalls(unittest.TestCase):
def test_create_session_back_to_back(
self, mock_validate_url_form,
mock_get_oauth_service, mock_get_access_token,
mock_validate_url_existence):
mock_validate_url_existence,
mock_add_timeout_backoff):
oauth_service = Foo()
access_token = Foo()
setattr(oauth_service, "get_session", lambda x: "newSession3")
......@@ -225,6 +229,7 @@ class TestApiCalls(unittest.TestCase):
self.assertTrue("not a valid URL" in str(err.exception))
mock_validate_url_form.assert_called_with(base_URL)
@patch("API.apiCalls.ApiCalls.add_timeout_backoff")
@patch("API.apiCalls.ApiCalls.validate_URL_existence")
@patch("API.apiCalls.ApiCalls.get_access_token")
@patch("API.apiCalls.ApiCalls.get_oauth_service")
......@@ -232,7 +237,8 @@ class TestApiCalls(unittest.TestCase):
def test_create_session_invalid_session(self, mock_validate_url_form,
mock_get_oauth_service,
mock_get_access_token,
mock_validate_url_existence):
mock_validate_url_existence,
mock_add_timeout_backoff):
oauth_service = Foo()
access_token = Foo()
......
[Application]
name=IRIDA Uploader
version=2.1.2-SNAPSHOT
version=2.1.4
entry_point=run_IRIDA_Uploader:main
icon=GUI/images/iu.ico
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment