Commit 26934e11 authored by Franklin Bristow's avatar Franklin Bristow

Merge with development.

parents 2a787b24 defcfec3
Pipeline #2578 passed with stage
in 5 minutes and 49 seconds
before_script:
- rm -rf /tmp/irida/*
build-against-irida-master:
script: make test IRIDA_VERSION=master
build-against-irida-development:
script: make test IRIDA_VERSION=development
job2:
pep8:
script: "scripts/verifyPEP8.sh"
#build:
# script: make windows
# artifacts:
# paths:
# - build/nsis/IRIDA_Uploader_*.exe
import logging
import threading
from os import path
from API.pubsub import send_message
from API import ApiCalls
from API.config import read_config_option
from requests.exceptions import ConnectionError
from urllib2 import URLError
class APIConnectorTopics(object):
connection_error_topic = "APIConnector.connection_error_topic"
connection_error_url_topic = connection_error_topic + ".url"
connection_error_credentials_topic = connection_error_topic + ".credentials"
connection_error_user_credentials_topic = connection_error_credentials_topic + ".user"
connection_error_client_id_topic = connection_error_credentials_topic + ".client_id"
connection_error_client_secret_topic = connection_error_credentials_topic + ".client_secret"
connection_success_topic = "APIConnector.connection_success_topic"
connection_success_valid_url = "APIConnector.connection_success_valid_url"
connection_success_valid_client_id = "APIConnector.connection_success_valid_client_id"
connection_success_valid_client_secret = "APIConnector.connection_success_valid_client_secret"
lock = threading.Lock()
def connect_to_irida():
"""Connect to IRIDA for online validation.
Returns:
A configured instance of API.apiCalls.
"""
client_id = read_config_option("client_id")
client_secret = read_config_option("client_secret")
baseURL = read_config_option("baseURL")
username = read_config_option("username")
password = read_config_option("password")
try:
# Several threads might be attempting to connect at the same time, so lock
# the connection step, but **do not** block (acquire(False) means do not block)
# and just return if someone else is already trying to connect.
if lock.acquire(False):
logging.info("About to try connecting to IRIDA.")
api = ApiCalls(client_id, client_secret, baseURL, username, password)
send_message(APIConnectorTopics.connection_success_topic, api=api)
return api
else:
logging.info("Someone else is already trying to connect to IRIDA.")
except ConnectionError as e:
logging.info("Got a connection error when trying to connect to IRIDA.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {}. The server might be down. Make "
"sure that the connection address is correct (you can change the "
"address by clicking on the 'Open Settings' button below) and try"
" again, try again later, or contact an administrator."
).format(baseURL))
raise
except (SyntaxError, ValueError) as e:
logging.info("Connected, but the response was garbled.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {}. The server is up, but I "
"didn't understand the response. Make sure that the connection "
"address is correct (you can change the address by clicking on "
"the 'Open Settings' button below) and try again, try again"
" later, or contact an administrator."
).format(baseURL))
raise
except KeyError as e:
logging.info("Connected, but the OAuth credentials are wrong.", exc_info=True)
# this is credentials related, but let's try to figure out why the server
# is telling us that we can't log in.
message = str(e.message)
if "Bad credentials" in message:
topic = APIConnectorTopics.connection_error_user_credentials_topic
# if we're getting bad credentials, then that means the API is allowing
# us to try authenticate with a username and password, so our client id
# and secret are both correct:
send_message(APIConnectorTopics.connection_success_valid_client_secret)
elif "clientId does not exist" in message:
topic = APIConnectorTopics.connection_error_client_id_topic
elif "Bad client credentials" in message:
topic = APIConnectorTopics.connection_error_client_secret_topic
# if we're getting a bad client secret message, that means that the
# client ID is valid.
send_message(APIConnectorTopics.connection_success_valid_client_id)
else:
topic = APIConnectorTopics.connection_error_credentials_topic
send_message(topic, error_message=(
"We couldn't connect to IRIDA at {}. The server is up, but it's "
"reporting that your credentials are wrong. Click on the 'Open Settings'"
" button below and check your credentials, then try again. If the "
"connection still doesn't work, contact an administrator."
).format(baseURL))
# in spite of it all, this means that we're probably actually trying to connect
# to a real IRIDA server, so let the settings dialog know that it can render
# a success icon beside the URL
send_message(APIConnectorTopics.connection_success_valid_url)
raise
except URLError as e:
logging.info("Couldn't connect to IRIDA because the URL is invalid.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {} because it isn't a valid URL. "
"Click on the 'Open Settings' button below to enter a new URL and "
"try again."
).format(baseURL))
raise
except:
logging.info("Some other kind of error happened.", exc_info=True)
send_message(APIConnectorTopics.connection_error_topic, error_message=(
"We couldn't connect to IRIDA at {} for an unknown reason. Click "
"on the 'Open Settings' button below to check the URL and your "
"credentials, then try again. If the connection still doesn't "
"work, contact an administrator."
).format(baseURL))
raise
finally:
try:
lock.release()
except:
pass
from apiCalls import ApiCalls
from config import read_config_option, write_config_option
This diff is collapsed.
import os
import logging
from ConfigParser import RawConfigParser, NoOptionError
from appdirs import user_config_dir
from collections import namedtuple
user_config_file = os.path.join(user_config_dir("iridaUploader"), "config.conf")
conf_parser = RawConfigParser()
SettingsDefault = namedtuple('SettingsDefault', ['setting', 'default_value'])
default_settings = [SettingsDefault._make(["client_id", ""]),
SettingsDefault._make(["client_secret", ""]),
SettingsDefault._make(["username", ""]),
SettingsDefault._make(["password", ""]),
SettingsDefault._make(["baseurl", ""]),
SettingsDefault._make(["completion_cmd", ""]),
SettingsDefault._make(["default_dir", os.path.expanduser("~")]),
SettingsDefault._make(["monitor_default_dir", "False"])]
if os.path.exists(user_config_file):
logging.info("Loading configuration settings from {}".format(user_config_file))
conf_parser.read(user_config_file)
for config in default_settings:
if not conf_parser.has_option("Settings", config.setting):
conf_parser.set("Settings", config.setting, config.default_value)
else:
logging.info("No default config file exists, loading defaults.")
conf_parser.add_section("Settings")
for config in default_settings:
conf_parser.set("Settings", config.setting, config.default_value)
def read_config_option(key, expected_type=None, default_value=None):
"""Read the specified value from the configuration file.
Args:
key: the name of the key to read from the config file.
expected_type: read the config option as the specified type (if specified)
default_value: if the key doesn't exist, just return the default value.
If the default value is not specified, the function will throw whatever
error was raised by the configuration parser
"""
logging.info("Reading config option {} with expected type {}".format(key, expected_type))
try:
if not expected_type:
value = conf_parser.get("Settings", key)
logging.info("Got configuration for key {}: {}".format(key, value))
return conf_parser.get("Settings", key)
elif expected_type is bool:
return conf_parser.getboolean("Settings", key)
except (ValueError, NoOptionError) as e:
if default_value:
return default_value
else:
raise
def write_config_option(field_name, field_value):
"""Write the configuration file out with the new key.
Args:
field_name: the name of the field to write
field_value: the value to write to the file
"""
conf_parser.set("Settings", field_name, field_value)
if not os.path.exists(os.path.dirname(user_config_file)):
os.makedirs(os.path.dirname(user_config_file))
with open(user_config_file, 'wb') as config_file:
conf_parser.write(config_file)
import os
import logging
from wx.lib.pubsub import pub
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileCreatedEvent
from API.pubsub import send_message
from API.directoryscanner import find_runs_in_directory
from GUI.SettingsDialog import SettingsDialog
class DirectoryMonitorTopics(object):
"""Topics for monitoring directories for new runs."""
new_run_observed = "new_run_observed"
finished_discovering_run = "finished_discovering_run"
shut_down_directory_monitor = "shut_down_directory_monitor"
class CompletedJobInfoEventHandler(FileSystemEventHandler):
"""A subclass of watchdog.events.FileSystemEventHandler that will run
a directory scan on the monitored directory. This will filter explicitly on
a file creation event for a file with the name `CompletedJobInfo.xml`."""
def on_created(self, event):
"""Overrides `on_created` in `FileSystemEventHandler` to filter on
file creation events for `CompletedJobInfo.xml`."""
if isinstance(event, FileCreatedEvent) and event.src_path.endswith('CompletedJobInfo.xml'):
logging.info("Observed new run in {}, telling the UI to start uploading it.".format(event.src_path))
directory = os.path.dirname(event.src_path)
# tell the UI to clean itself up before observing new runs
send_message(DirectoryMonitorTopics.new_run_observed)
# this will send a bunch of events that the UI is listening for, but
# unlike the UI (which runs this in a separate thread), we're going to do this
# in our own thread and block on it so we can tell the UI to start
# uploading once we've finished discovering the run
find_runs_in_directory(directory)
# now tell the UI to start
send_message(DirectoryMonitorTopics.finished_discovering_run)
else:
logging.debug("Ignoring file event [{}] with path [{}]".format(str(event), event.src_path))
def monitor_directory(directory):
"""Starts monitoring the specified directory in a background thread. File events
will be passed to the `CompletedJobInfoEventHandler`.
Arguments:
directory: the directory to monitor.
"""
observer = Observer()
logging.info("Getting ready to monitor directory {}".format(directory))
event_handler = CompletedJobInfoEventHandler()
observer.schedule(event_handler, directory, recursive=True)
def stop_monitoring(*args, **kwargs):
"""Tells watchdog to stop watching the directory when the newly processed run
was discovered."""
logging.info("Halting monitoring on directory because run was discovered.")
observer.stop()
observer.join()
pub.subscribe(stop_monitoring, SettingsDialog.settings_closed_topic)
pub.subscribe(stop_monitoring, DirectoryMonitorTopics.new_run_observed)
pub.subscribe(stop_monitoring, DirectoryMonitorTopics.shut_down_directory_monitor)
observer.start()
......@@ -8,8 +8,14 @@ from Validation.offlineValidation import validate_sample_sheet, validate_sample_
from Parsers.miseqParser import parse_metadata, complete_parse_samples
from Model.SequencingRun import SequencingRun
from API.fileutils import find_file_by_name
from API.pubsub import send_message
logging.basicConfig(level = logging.INFO)
class DirectoryScannerTopics(object):
"""Topics issued by `find_runs_in_directory`"""
finished_run_scan = "finished_run_scan"
run_discovered = "run_discovered"
garbled_sample_sheet = "garbled_sample_sheet"
missing_files = "missing_files"
def find_runs_in_directory(directory):
"""Find and validate all runs the specified directory.
......@@ -35,6 +41,8 @@ def find_runs_in_directory(directory):
logging.info("filtered sample sheets: {}".format(", ".join(sheets_to_upload)))
sequencing_runs = [process_sample_sheet(sheet) for sheet in sheets_to_upload]
send_message(DirectoryScannerTopics.finished_run_scan)
return sequencing_runs
def run_is_uploaded(run_directory):
......@@ -67,19 +75,33 @@ def process_sample_sheet(sample_sheet):
Returns: an individual SequencingRun object for the sample sheet,
ready to be uploaded.
"""
logging.info("going to parse metadata")
run_metadata = parse_metadata(sample_sheet)
try:
logging.info("going to parse metadata")
run_metadata = parse_metadata(sample_sheet)
logging.info("going to parse samples")
samples = complete_parse_samples(sample_sheet)
logging.info("going to build sequencing run")
sequencing_run = SequencingRun(run_metadata, samples, sample_sheet)
logging.info("going to parse samples")
samples = complete_parse_samples(sample_sheet)
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
logging.info("going to build sequencing run")
sequencing_run = SequencingRun(run_metadata, samples, sample_sheet)
send_message(DirectoryScannerTopics.run_discovered, run=sequencing_run)
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
return sequencing_run
except SampleSheetError, e:
logging.exception("Failed to parse sample sheet.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sample_sheet, error=e)
except SampleError, e:
logging.exception("Failed to parse sample.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sample_sheet, error=e)
except SequenceFileError as e:
logging.exception("Failed to find files for sample sheet.")
send_message(DirectoryScannerTopics.missing_files, sample_sheet=sample_sheet, error=e)
return sequencing_run
return None
def validate_run(sequencing_run):
"""Do the validation on a run, its samples, and files.
......@@ -95,8 +117,9 @@ def validate_run(sequencing_run):
validation = validate_sample_sheet(sequencing_run.sample_sheet)
if not validation.is_valid():
raise SampleSheetError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()))
send_message(sequencing_run.offline_validation_topic, run=sequencing_run, errors=validation.get_errors())
raise SampleSheetError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()), validation.error_list())
validation = validate_sample_list(sequencing_run.sample_list)
if not validation.is_valid():
raise SampleError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()))
raise SampleError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()), validation.error_list())
......@@ -5,8 +5,6 @@ from wx.lib.pubsub import pub
def send_message(message_id, *args, **kwargs):
app = wx.GetApp()
if app:
logging.info("Sending message to GUI")
wx.CallAfter(pub.sendMessage, message_id, *args, **kwargs)
else:
logging.info("Sending message to command line")
pub.sendMessage(message_id, *args, **kwargs)
import gzip
import sys
import wx
from wx.lib.pubsub import pub
def parse_fastq(filename):
fastq_file = gzip.open(filename)
for i, line in enumerate(fastq_file):
if i % 4 == 1:
yield line.rstrip('\n')
def fastq_stats(filename, event_name=None):
total_bases = 0
total_reads = 0
for sequence in parse_fastq(filename):
total_reads += 1
total_bases += len(sequence)
if event_name:
wx.CallAfter(pub.sendMessage, event_name, fastq_stats=(total_reads, total_bases))
return (total_reads, total_bases)
if __name__ == "__main__":
filename = sys.argv[1]
(reads, bases) = fastq_stats(filename)
print "Total reads: [{}], total bases: [{}].".format(reads, bases)
from Validation.onlineValidation import project_exists, sample_exists
from wx.lib.pubsub import pub
import logging
from Exceptions.ProjectError import ProjectError
from API.pubsub import send_message
from os import path
from wx.lib.pubsub import pub
import os
import json
import logging
import threading
class RunUploaderTopics(object):
start_online_validation = "start_online_validation"
online_validation_failure = "online_validation_failure"
start_checking_samples = "start_checking_samples"
start_uploading_samples = "start_uploading_samples"
finished_uploading_samples = "finished_uploading_samples"
started_post_processing = "started_post_processing"
finished_post_processing = "finished_post_processing"
failed_post_processing = "failed_post_processing"
class RunUploader(threading.Thread):
"""A convenience thread wrapper for uploading runs to the server."""
def __init__(self, api, runs, post_processing_task=None, name='RunUploaderThread'):
"""Initialize a `RunUploader`.
Args:
api: an initialized connection to IRIDA.
runs: a list of runs to upload to the server.
post_processing_task: the system command to execute after the runs are uploaded
name: the name of the thread.
"""
self._stop_event = threading.Event()
self._api = api
self._runs = runs
self._post_processing_task = post_processing_task
threading.Thread.__init__(self, name=name)
def run(self):
"""Initiate upload. The upload happens serially, one run at a time."""
for run in self._runs:
upload_run_to_server(api=self._api, sequencing_run=run)
# once the run uploads are complete, we can launch the post-processing
# command
if self._post_processing_task:
send_message(RunUploaderTopics.started_post_processing)
logging.info("About to launch post-processing command: {}".format(self._post_processing_task))
# blocks until the command is complete
try:
exit_code = os.system(self._post_processing_task)
except:
exit_code = 1
if not exit_code:
logging.info("Post-processing command is complete.")
send_message(RunUploaderTopics.finished_post_processing)
else:
logging.error("The post-processing command is reporting failure")
send_message(RunUploaderTopics.failed_post_processing)
def upload_run_to_server(api, sequencing_run, progress_callback):
def join(self, timeout=None):
"""Kill the thread.
This will politely ask the API to terminate connections.
Args:
timeout: the length of time to wait before bailing out.
"""
logging.info("Going to try killing connections on exit.")
self._api._kill_connections()
threading.Thread.join(self, timeout)
def upload_run_to_server(api, sequencing_run):
"""Upload a single run to the server.
Arguments:
api -- the API object to use for interacting with the server
sequencing_run -- the run to upload to the server
progress_callback -- the function to call for indicating upload progress
Publishes messages:
start_online_validation -- when running an online validation (checking project ids) starts
......@@ -25,12 +90,12 @@ def upload_run_to_server(api, sequencing_run, progress_callback):
filename = path.join(sequencing_run.sample_sheet_dir,
".miseqUploaderInfo")
def _handle_upload_sample_complete(sample):
def _handle_upload_sample_complete(sample=None):
"""Handle the event that happens when a sample has finished uploading.
"""
if sample is None:
raise Exception("sample is required!")
with open(filename, "rb") as reader:
uploader_info = json.load(reader)
logging.info(uploader_info)
......@@ -41,19 +106,7 @@ def upload_run_to_server(api, sequencing_run, progress_callback):
with open(filename, 'wb') as writer:
json.dump(uploader_info, writer)
logging.info("Finished updating info file.")
def _sample_already_uploaded(sample):
"""Check whether or not a sample was already uploaded
"""
with open(filename, "rb") as reader:
uploader_info = json.load(reader)
logging.info(uploader_info)
try:
logging.info("Checking if {} was already uploaded in {}.".format(sample.get_id(), uploader_info['uploaded_samples']))
return sample.get_id() in uploader_info['uploaded_samples']
except KeyError:
logging.info("sample {} was not uploaded.".format(sample.get_id()))
return False
pub.unsubscribe(_handle_upload_sample_complete, sample.upload_completed_topic)
# do online validation first.
_online_validation(api, sequencing_run)
......@@ -70,29 +123,33 @@ def upload_run_to_server(api, sequencing_run, progress_callback):
uploader_info = json.load(reader)
run_id = uploader_info['Upload ID']
send_message("start_checking_samples")
send_message(RunUploaderTopics.start_checking_samples)
logging.info("Starting to check samples. [{}]".format(len(sequencing_run.sample_list)))
# only send samples that aren't already on the server
samples_to_create = filter(lambda sample: not sample_exists(api, sample), sequencing_run.sample_list)
logging.info("Sending samples to server: [{}].".format(", ".join([str(x) for x in samples_to_create])))
api.send_samples(samples_to_create)
pub.subscribe(_handle_upload_sample_complete, 'completed_uploading_sample')
samples_to_upload = filter(lambda sample: not _sample_already_uploaded(sample), sequencing_run.sample_list)
skipped_samples = filter(lambda sample: _sample_already_uploaded(sample), sequencing_run.sample_list)
for sample in sequencing_run.samples_to_upload:
pub.subscribe(_handle_upload_sample_complete, sample.upload_completed_topic)
send_message("start_uploading_samples", sheet_dir = sequencing_run.sample_sheet_dir,
skipped_sample_ids = [sample.get_id() for sample in skipped_samples],
skipped_sample_ids = [sample.get_id() for sample in sequencing_run.uploaded_samples],
run_id = run_id)
send_message(sequencing_run.upload_started_topic)
logging.info("About to start uploading samples.")
api.send_sequence_files(samples_list = samples_to_upload,
callback = progress_callback, upload_id = run_id)
send_message("finished_uploading_samples", sheet_dir = sequencing_run.sample_sheet_dir)
api.set_seq_run_complete(run_id)
_create_miseq_uploader_info_file(sequencing_run.sample_sheet_dir, run_id, "Complete")
try:
api.send_sequence_files(samples_list = sequencing_run.samples_to_upload,
upload_id = run_id)
send_message("finished_uploading_samples", sheet_dir = sequencing_run.sample_sheet_dir)
send_message(sequencing_run.upload_completed_topic)
api.set_seq_run_complete(run_id)
_create_miseq_uploader_info_file(sequencing_run.sample_sheet_dir, run_id, "Complete")
except Exception as e:
logging.exception("Encountered error while uploading files to server, updating status of run to error state.")
api.set_seq_run_error(run_id)
raise
def _online_validation(api, sequencing_run):
"""Do online validation for the specified sequencing run.
......
1.7.0 to 2.0.0
==============
* The UI is completely rewritten to better show what's happened with samples as they're uploaded, and to facilitate quality control implementation later.
* Add an experimental auto-upload feature to automatically upload new runs as the sequencer finishes a run (monitoring run directories for `CompletedJobInfo.xml`).
* Use the sample name and ID columns correctly in `SampleSheet.csv`: the sample name column is used for naming `.fastq.gz` files, the sample ID column is used for sending data to IRIDA.
* Discard the cache of projects whenever sending data to the server.
1.5.0 to 1.6.0
==============
* Add an about dialog to show the version number in the UI.
......
class SampleError(Exception):
pass
"""An exception to be raised when issues with samples arise.
Examples include when IRIDA responds with an error during sample creation,
or when the parsing component can't parse the sample section of the sample
sheet.
"""
def __init__(self, message, errors):
"""Initialize a SampleError.
Args:
message: the summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SampleSheetError(Exception):
pass
"""An exception raised when errors are encountered with a sample sheet.
Examples include when a sample sheet can't be parsed because it's garbled, or
if IRIDA rejects the creation of a run because fields are missing or invalid
from the sample sheet.
"""
def __init__(self, message, errors):
"""Initalize a SampleSheetError.
Args:
message: a summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SequenceFileError(Exception):
pass
"""An exception that's raised when errors are encountered with a sequence file.
Examples include when files cannot be found for samples that are in the sample
sheet, or when the server rejects a file during upload.
"""
def __init__(self, message, errors):
"""Initialize a SequenceFileError.
Args:
message: a summary message of the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError
from Exceptions.SampleError import SampleError
import wx
from wx.lib.wordwrap import wordwrap
import webbrowser
import logging
from GUI.MainPanel import MainPanel
from os import path
path_to_module = path.dirname(__file__)