...
 
Commits (152)
......@@ -12,3 +12,6 @@ pynsist_pkgs
wxPython*
.virtualenv
.cache
.idea/
.python-version
.pytest_cache/
This diff is collapsed.
......@@ -48,7 +48,10 @@ def read_config_option(key, expected_type=None, default_value=None):
try:
if not expected_type:
value = conf_parser.get("Settings", key)
logging.info("Got configuration for key {}: {}".format(key, value))
if key is "password":
logging.info("Got configuration for key {}: ****".format(key))
else:
logging.info("Got configuration for key {}: {}".format(key, value))
return conf_parser.get("Settings", key)
elif expected_type is bool:
return conf_parser.getboolean("Settings", key)
......
import os
import logging
import time
import threading
from wx.lib.pubsub import pub
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileCreatedEvent
from API.pubsub import send_message
from API.directoryscanner import find_runs_in_directory
from GUI.SettingsDialog import SettingsDialog
toMonitor = True
TIMEBETWEENMONITOR = 120
class DirectoryMonitorTopics(object):
"""Topics for monitoring directories for new runs."""
new_run_observed = "new_run_observed"
finished_discovering_run = "finished_discovering_run"
shut_down_directory_monitor = "shut_down_directory_monitor"
start_up_directory_monitor = "start_up_directory_monitor"
finished_uploading_run = "finished_uploading_run"
class RunMonitor(threading.Thread):
"""A convenience thread wrapper for monitoring a directory for runs"""
def __init__(self, directory, cond, name="RunMonitorThread"):
"""Initialize a `RunMonitor`"""
self._directory = directory
self._condition = cond
super(RunMonitor, self).__init__(name=name)
def run(self):
"""Initiate directory monitor. The monitor checks the default
directory every 2 minutes
"""
monitor_directory(self._directory, self._condition)
class CompletedJobInfoEventHandler(FileSystemEventHandler):
"""A subclass of watchdog.events.FileSystemEventHandler that will run
a directory scan on the monitored directory. This will filter explicitly on
a file creation event for a file with the name `CompletedJobInfo.xml`."""
def on_created(self, event):
"""Overrides `on_created` in `FileSystemEventHandler` to filter on
file creation events for `CompletedJobInfo.xml`."""
if isinstance(event, FileCreatedEvent) and event.src_path.endswith('CompletedJobInfo.xml'):
logging.info("Observed new run in {}, telling the UI to start uploading it.".format(event.src_path))
directory = os.path.dirname(event.src_path)
# tell the UI to clean itself up before observing new runs
send_message(DirectoryMonitorTopics.new_run_observed)
# this will send a bunch of events that the UI is listening for, but
# unlike the UI (which runs this in a separate thread), we're going to do this
# in our own thread and block on it so we can tell the UI to start
# uploading once we've finished discovering the run
find_runs_in_directory(directory)
# now tell the UI to start
send_message(DirectoryMonitorTopics.finished_discovering_run)
else:
logging.debug("Ignoring file event [{}] with path [{}]".format(str(event), event.src_path))
def monitor_directory(directory):
"""Starts monitoring the specified directory in a background thread. File events
will be passed to the `CompletedJobInfoEventHandler`.
Arguments:
directory: the directory to monitor.
def join(self, timeout=None):
"""Kill the thread"""
global toMonitor
logging.info("going to kill monitoring")
toMonitor = False
threading.Thread.join(self, timeout)
def on_created(directory, cond):
"""When a CompletedJobInfo.xml file is found without a .miseqUploaderInfo file,
an automatic upload is triggered
"""
observer = Observer()
logging.info("Observed new run in {}, telling the UI to start uploading it.".format(directory))
directory = os.path.dirname(directory)
# tell the UI to clean itself up before observing new runs
send_message(DirectoryMonitorTopics.new_run_observed)
if toMonitor:
find_runs_in_directory(directory)
# check if monitoring is still "on" after returning from find_runs_in_directory()
if toMonitor:
send_message(DirectoryMonitorTopics.finished_discovering_run)
# using locks to prevent the monitor from running while an upload is happening.
cond.acquire()
cond.wait()
cond.release()
send_message(DirectoryMonitorTopics.finished_uploading_run)
def monitor_directory(directory, cond):
"""Calls the function searches the default directory every 2 minutes unless
monitoring is no longer required
"""
global toMonitor
logging.info("Getting ready to monitor directory {}".format(directory))
event_handler = CompletedJobInfoEventHandler()
observer.schedule(event_handler, directory, recursive=True)
def stop_monitoring(*args, **kwargs):
"""Tells watchdog to stop watching the directory when the newly processed run
was discovered."""
logging.info("Halting monitoring on directory because run was discovered.")
observer.stop()
observer.join()
pub.subscribe(stop_monitoring, SettingsDialog.settings_closed_topic)
pub.subscribe(stop_monitoring, DirectoryMonitorTopics.new_run_observed)
pub.subscribe(stop_monitoring, DirectoryMonitorTopics.shut_down_directory_monitor)
pub.subscribe(start_monitoring, DirectoryMonitorTopics.start_up_directory_monitor)
time.sleep(10)
while toMonitor:
search_for_upload(directory, cond)
i = 0
while toMonitor and i < TIMEBETWEENMONITOR:
time.sleep(10)
i = i+10
def search_for_upload(directory, cond):
"""loop through subdirectories of the default directory looking for CompletedJobInfo.xml without
.miseqUploaderInfo files.
"""
global toMonitor
if not os.access(directory, os.W_OK):
logging.warning("Could not access directory while monitoring for samples, directory is not writeable {}".format(directory))
return
root = next(os.walk(directory))[0]
dirs = next(os.walk(directory))[1]
for name in dirs:
check_for_comp_job = os.path.join(root, name, "CompletedJobInfo.xml")
check_for_miseq = os.path.join(root, name, ".miseqUploaderInfo")
if os.path.isfile(check_for_comp_job):
if not os.path.isfile(check_for_miseq):
path_to_upload = check_for_comp_job
if toMonitor:
on_created(path_to_upload, cond)
# After upload, start back at the start of directories
return
# Check each step of loop if monitoring is still required
if not toMonitor:
return
return
def stop_monitoring():
"""Stop directory monitoring by setting toMonitor to False"""
global toMonitor
if toMonitor:
logging.info("Halting monitoring on directory.")
toMonitor = False
observer.start()
def start_monitoring():
"""Restart directory monitoring by setting toMonitor to True"""
global toMonitor
if not toMonitor:
logging.info("Restarting monitor on directory")
toMonitor = True
import json
import logging
from os import path
import os
from Exceptions.SampleSheetError import SampleSheetError
from Exceptions.SequenceFileError import SequenceFileError
from Exceptions.SampleError import SampleError
from Validation.offlineValidation import validate_sample_sheet, validate_sample_list
from Parsers.miseqParser import parse_metadata, complete_parse_samples
from Model.SequencingRun import SequencingRun
from API.fileutils import find_file_by_name
from API.pubsub import send_message
class DirectoryScannerTopics(object):
"""Topics issued by `find_runs_in_directory`"""
finished_run_scan = "finished_run_scan"
......@@ -17,6 +17,7 @@ class DirectoryScannerTopics(object):
garbled_sample_sheet = "garbled_sample_sheet"
missing_files = "missing_files"
def find_runs_in_directory(directory):
"""Find and validate all runs the specified directory.
......@@ -27,43 +28,100 @@ def find_runs_in_directory(directory):
Arguments:
directory -- the directory to find sequencing runs
Usage:
Can be used on the directory containing the SampleSheet.csv file (a single run)
Can be used on the directory containing directories with SampleSheet.csv files in them (a group of runs)
Returns: a list of populated sequencing run objects found
in the directory, ready to be uploaded.
"""
logging.info("looking for sample sheet in {}".format(directory))
sample_sheets = find_file_by_name(directory = directory,
name_pattern = '^SampleSheet.csv$',
depth = 2)
logging.info("found sample sheets: {}".format(", ".join(sample_sheets)))
# filter directories that have been completely uploaded
sheets_to_upload = filter(lambda dir: not run_is_uploaded(path.dirname(dir)), sample_sheets)
logging.info("filtered sample sheets: {}".format(", ".join(sheets_to_upload)))
sequencing_runs = [process_sample_sheet(sheet) for sheet in sheets_to_upload]
def find_run_directory_list(run_dir):
"""Find and return all directories (including this one) in the specified directory.
send_message(DirectoryScannerTopics.finished_run_scan)
Arguments:
directory -- the directory to find directories in
return sequencing_runs
Returns: a list of directories including current directory
"""
def run_is_uploaded(run_directory):
"""Check if a run has already been uploaded.
# Checks if we can access to the given directory, return empty and log a warning if we cannot.
if not os.access(run_dir, os.W_OK):
logging.warning("The following directory is not writeable, "
"can not upload any samples from this directory {}".format(run_dir))
return []
This function checks for the existence of a file `.miseqUploaderInfo`, then
evaluates the status of the run by looking at the "Upload Status" field.
dir_list = next(os.walk(run_dir))[1] # Gets the list of directories in the directory
dir_list.append(run_dir) # Add the current directory to the list too
return dir_list
Arguments:
run_directory -- the sequencing run directory
def dir_has_samples_not_uploaded(sample_dir):
"""Find and validate runs in the specified directory.
Returns: true if the run has already been uploaded, false if it has not.
"""
uploader_info_file = find_file_by_name(run_directory, '.miseqUploaderInfo', depth = 1)
Validates if run already has been uploaded, partially uploaded, or not uploaded
Arguments:
directory -- the directory to find sequencing runs
Returns: Boolean,
True: Directory has samples not uploaded,
Directory has partially uploaded samples
False: Directory has no samples
Directory samples are already uploaded
Directory can not be read, permissions issue
"""
if uploader_info_file:
with open(uploader_info_file[0], "rb") as reader:
info_file = json.load(reader)
return info_file["Upload Status"] == "Complete"
# Checks if we can write to the directory, return false and log a warning if we cannot.
if not os.access(sample_dir, os.W_OK):
logging.warning("The following directory is not writeable, "
"can not upload any samples from this directory {}".format(sample_dir))
return False
file_list = next(os.walk(sample_dir))[2] # Gets the list of files in the directory
if 'SampleSheet.csv' in file_list:
if '.miseqUploaderInfo' in file_list: # Must check status of upload to determine if upload is completed
uploader_info_file = os.path.join(sample_dir, '.miseqUploaderInfo')
with open(uploader_info_file, "rb") as reader:
info_file = json.load(reader)
return info_file["Upload Status"] != "Complete" # if True, has samples, not completed uploading
else: # SampleSheet.csv with no .miseqUploaderInfo file, has samples not uploaded yet
return True
return False # No SampleSheet.csv, does not have samples
logging.info("looking for sample sheet in {}".format(directory))
sample_sheets = []
directory_list = find_run_directory_list(directory)
for d in directory_list:
current_directory = os.path.join(directory, d)
if dir_has_samples_not_uploaded(current_directory):
sample_sheets.append(os.path.join(current_directory, 'SampleSheet.csv'))
logging.info("found sample sheets (filtered): {}".format(", ".join(sample_sheets)))
# Only appending sheets to the list that do not have errors
# The errors are collected to create a list to show the user
sequencing_runs = []
for sheet in sample_sheets:
try:
sequencing_runs.append(process_sample_sheet(sheet))
except SampleSheetError, e:
logging.exception("Failed to parse sample sheet.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sheet, error=e)
except SampleError, e:
logging.exception("Failed to parse sample.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sheet, error=e)
except SequenceFileError as e:
logging.exception("Failed to find files for sample sheet.")
send_message(DirectoryScannerTopics.missing_files, sample_sheet=sheet, error=e)
send_message(DirectoryScannerTopics.finished_run_scan)
return sequencing_runs
return False
def process_sample_sheet(sample_sheet):
"""Create a SequencingRun object for the specified sample sheet.
......@@ -75,33 +133,23 @@ def process_sample_sheet(sample_sheet):
Returns: an individual SequencingRun object for the sample sheet,
ready to be uploaded.
"""
try:
logging.info("going to parse metadata")
run_metadata = parse_metadata(sample_sheet)
logging.info("going to parse samples")
samples = complete_parse_samples(sample_sheet)
logging.info("going to parse metadata")
run_metadata = parse_metadata(sample_sheet)
logging.info("going to parse samples")
samples = complete_parse_samples(sample_sheet)
logging.info("going to build sequencing run")
sequencing_run = SequencingRun(run_metadata, samples, sample_sheet)
logging.info("going to build sequencing run")
sequencing_run = SequencingRun(run_metadata, samples, sample_sheet)
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
send_message(DirectoryScannerTopics.run_discovered, run=sequencing_run)
send_message(DirectoryScannerTopics.run_discovered, run=sequencing_run)
return sequencing_run
except SampleSheetError, e:
logging.exception("Failed to parse sample sheet.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sample_sheet, error=e)
except SampleError, e:
logging.exception("Failed to parse sample.")
send_message(DirectoryScannerTopics.garbled_sample_sheet, sample_sheet=sample_sheet, error=e)
except SequenceFileError as e:
logging.exception("Failed to find files for sample sheet.")
send_message(DirectoryScannerTopics.missing_files, sample_sheet=sample_sheet, error=e)
return sequencing_run
return None
def validate_run(sequencing_run):
"""Do the validation on a run, its samples, and files.
......@@ -118,8 +166,10 @@ def validate_run(sequencing_run):
validation = validate_sample_sheet(sequencing_run.sample_sheet)
if not validation.is_valid():
send_message(sequencing_run.offline_validation_topic, run=sequencing_run, errors=validation.get_errors())
raise SampleSheetError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()), validation.error_list())
raise SampleSheetError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()),
validation.error_list())
validation = validate_sample_list(sequencing_run.sample_list)
if not validation.is_valid():
raise SampleError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()), validation.error_list())
raise SampleError('Sample sheet {} is invalid. Reason:\n {}'.format(sample_sheet, validation.get_errors()),
validation.error_list())
import os
import re
def find_file_by_name(directory, name_pattern, depth=-1):
"""Find a file by a name pattern in a directory
Traverse through a directory and a level below it searching for
a file that matches the given SampleSheet pattern.
Arguments:
directory -- top level directory to start searching from
name_pattern -- SampleSheet pattern to try and match
using fnfilter/ fnmatch.filter
depth -- optional, the max depth to descend into the directory. a depth of
-1 implies no limit.
Returns: list containing files that match pattern
"""
if depth == -1:
walk = lambda directory, depth: os.walk(directory)
else:
walk = lambda directory, depth: walklevel(directory, depth)
result_list = []
if os.path.isdir(directory):
for root, dirs, files in walk(directory, depth):
for filename in filter(lambda file: re.search(name_pattern, file), files):
result_list.append(os.path.join(root, filename))
return result_list
def walklevel(directory, depth):
"""Descend into a directory, but only to the specified depth.
This method is gracelessly borrowed from:
http://stackoverflow.com/questions/229186/os-walk-without-digging-into-directories-below
Arguments:
directory -- the directory in which to start the walk
depth -- the depth to descend into the directory.
Returns: a generator for directories in under the top level directory.
"""
directory = directory.rstrip(os.path.sep)
assert os.path.isdir(directory)
num_sep = directory.count(os.path.sep)
for root, dirs, files in os.walk(directory):
yield root, dirs, files
num_sep_this = root.count(os.path.sep)
if num_sep + depth <= num_sep_this:
del dirs[:]
......@@ -23,7 +23,7 @@ class RunUploaderTopics(object):
class RunUploader(threading.Thread):
"""A convenience thread wrapper for uploading runs to the server."""
def __init__(self, api, runs, post_processing_task=None, name='RunUploaderThread'):
def __init__(self, api, runs, cond, post_processing_task=None, name='RunUploaderThread'):
"""Initialize a `RunUploader`.
Args:
......@@ -36,12 +36,14 @@ class RunUploader(threading.Thread):
self._api = api
self._runs = runs
self._post_processing_task = post_processing_task
self._condition = cond
threading.Thread.__init__(self, name=name)
def run(self):
"""Initiate upload. The upload happens serially, one run at a time."""
for run in self._runs:
upload_run_to_server(api=self._api, sequencing_run=run)
upload_run_to_server(api=self._api, sequencing_run=run, condition=self._condition)
send_message(RunUploaderTopics.finished_uploading_samples)
# once the run uploads are complete, we can launch the post-processing
# command
if self._post_processing_task:
......@@ -72,7 +74,7 @@ class RunUploader(threading.Thread):
self._api._kill_connections()
threading.Thread.join(self, timeout)
def upload_run_to_server(api, sequencing_run):
def upload_run_to_server(api, sequencing_run, condition):
"""Upload a single run to the server.
Arguments:
......@@ -128,7 +130,12 @@ def upload_run_to_server(api, sequencing_run):
# only send samples that aren't already on the server
samples_to_create = filter(lambda sample: not sample_exists(api, sample), sequencing_run.sample_list)
logging.info("Sending samples to server: [{}].".format(", ".join([str(x) for x in samples_to_create])))
api.send_samples(samples_to_create)
try:
api.send_samples(samples_to_create)
except Exception as e:
logging.exception("Encountered error while uploading files to server, updating status of run to error state.")
api.set_seq_run_error(run_id)
raise
for sample in sequencing_run.samples_to_upload:
pub.subscribe(_handle_upload_sample_complete, sample.upload_completed_topic)
......@@ -144,6 +151,10 @@ def upload_run_to_server(api, sequencing_run):
upload_id = run_id)
send_message("finished_uploading_samples", sheet_dir = sequencing_run.sample_sheet_dir)
send_message(sequencing_run.upload_completed_topic)
# acquring lock so it can be released so that directory monitoring can resume if it was running
condition.acquire()
condition.notify()
condition.release()
api.set_seq_run_complete(run_id)
_create_miseq_uploader_info_file(sequencing_run.sample_sheet_dir, run_id, "Complete")
except Exception as e:
......
2.0.0 to 2.1.4
==============
* Internal Changes to how threads are started and killed
* Added a timeout + retry to calls to IRIDA, so dropped conections will be retried.
* Updated the `requests` library for security patch
* Changed the api layer to use a singleton so multiple instances of the api are not spun off.
* Small bug fixes for GUI
* Fixed UI hanging when recursive directories are searched for runs.
1.7.0 to 2.0.0
==============
* The UI is completely rewritten to better show what's happened with samples as they're uploaded, and to facilitate quality control implementation later.
......
class ProjectError(Exception):
pass
class ProjectError(Exception):
pass
class SampleError(Exception):
"""An exception to be raised when issues with samples arise.
Examples include when IRIDA responds with an error during sample creation,
or when the parsing component can't parse the sample section of the sample
sheet.
"""
def __init__(self, message, errors):
"""Initialize a SampleError.
Args:
message: the summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SampleError(Exception):
"""An exception to be raised when issues with samples arise.
Examples include when IRIDA responds with an error during sample creation,
or when the parsing component can't parse the sample section of the sample
sheet.
"""
def __init__(self, message, errors):
"""Initialize a SampleError.
Args:
message: the summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SampleSheetError(Exception):
"""An exception raised when errors are encountered with a sample sheet.
Examples include when a sample sheet can't be parsed because it's garbled, or
if IRIDA rejects the creation of a run because fields are missing or invalid
from the sample sheet.
"""
def __init__(self, message, errors):
"""Initalize a SampleSheetError.
Args:
message: a summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SampleSheetError(Exception):
"""An exception raised when errors are encountered with a sample sheet.
Examples include when a sample sheet can't be parsed because it's garbled, or
if IRIDA rejects the creation of a run because fields are missing or invalid
from the sample sheet.
"""
def __init__(self, message, errors):
"""Initalize a SampleSheetError.
Args:
message: a summary message that's causing the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SequenceFileError(Exception):
"""An exception that's raised when errors are encountered with a sequence file.
Examples include when files cannot be found for samples that are in the sample
sheet, or when the server rejects a file during upload.
"""
def __init__(self, message, errors):
"""Initialize a SequenceFileError.
Args:
message: a summary message of the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
class SequenceFileError(Exception):
"""An exception that's raised when errors are encountered with a sequence file.
Examples include when files cannot be found for samples that are in the sample
sheet, or when the server rejects a file during upload.
"""
def __init__(self, message, errors):
"""Initialize a SequenceFileError.
Args:
message: a summary message of the error.
errors: a more detailed list of errors.
"""
self._message = message
self._errors = errors
@property
def message(self):
return self._message
@property
def errors(self):
return self._errors
def __str__(self):
return self.message
......@@ -9,6 +9,7 @@ from wx.lib.wordwrap import wordwrap
from Exceptions import SampleError, SampleSheetError, SequenceFileError
from API.directoryscanner import DirectoryScannerTopics
from API.directorymonitor import DirectoryMonitorTopics
from API.pubsub import send_message
from GUI.SettingsDialog import SettingsDialog
......@@ -71,6 +72,7 @@ class InvalidSampleSheetsPanel(wx.Panel):
sample_sheet: the sample sheet that failed to be parsed.
error: the error that was raised during validation.
"""
send_message(DirectoryMonitorTopics.shut_down_directory_monitor)
sheet_name = basename(dirname(sample_sheet)) + separator + "SampleSheet.csv"
logging.info("Handling sample sheet error for {}".format(sheet_name))
self.Freeze()
......@@ -79,9 +81,9 @@ class InvalidSampleSheetsPanel(wx.Panel):
sheet_errors_type = None
if isinstance(error, SampleError):
sheet_errors_type = self._errors_tree.AppendItem(sheet_errors_root, "Missing Project ID")
sheet_errors_type = self._errors_tree.AppendItem(sheet_errors_root, "Error with Sample Data")
elif isinstance(error, SequenceFileError):
sheet_errors_type = self._errors_tree.AppendItem(sheet_errors_root, "Missing FASTQ files")
sheet_errors_type = self._errors_tree.AppendItem(sheet_errors_root, "Missing or Duplicate FASTQ files")
elif isinstance(error, SampleSheetError):
sheet_errors_type = self._errors_tree.AppendItem(sheet_errors_root, "Missing Important Data")
......
......@@ -41,6 +41,10 @@ class RunPanel(ScrolledPanel):
self._last_progress = 0
self._last_timer_progress = 0
self._sample_panels = {}
self._progress = wx.Gauge(self, id=wx.ID_ANY, range=100, size=(250, 20))
self._progress.Hide()
self._progress_text = wx.StaticText(self, label=" 0%")
self._progress_text.Hide()
# the current overall progress for the run is calculated as a percentage
# of the total file size of all samples in the run.
self._progress_max = sum(sample.get_files_size() for sample in run.samples_to_upload)
......@@ -100,11 +104,10 @@ class RunPanel(ScrolledPanel):
logging.info("Upload started for {} with max size {}".format(self._run.upload_started_topic, self._progress_max))
pub.unsubscribe(self._upload_started, self._run.upload_started_topic)
self.Freeze()
self._progress = wx.Gauge(self, id=wx.ID_ANY, range=100, size=(250, 20))
self._progress_text = wx.StaticText(self, label=" 0%")
progress_sizer = wx.BoxSizer(wx.HORIZONTAL)
self._progress.Show()
progress_sizer.Add(self._progress, proportion=1)
self._progress_text.Show()
progress_sizer.Add(self._progress_text, proportion=0)
self._sizer.Insert(0, progress_sizer, flag=wx.EXPAND | wx.LEFT | wx.RIGHT, border=5)
......@@ -124,11 +127,12 @@ class RunPanel(ScrolledPanel):
pub.unsubscribe(self._upload_failed, self._run.upload_failed_topic)
pub.unsubscribe(self._handle_progress, self._run.upload_progress_topic)
pub.unsubscribe(self._upload_complete, self._run.upload_completed_topic)
self.Freeze()
self._timer.Stop()
self._progress_text.Destroy()
self._progress.Destroy()
if self._timer.IsRunning():
self._timer.Stop()
self._progress_text.Hide()
self._progress.Hide()
error_label = wx.StaticText(self, label=u"✘ Yikes!")
error_label.SetForegroundColour(wx.Colour(255, 0, 0))
error_label.SetFont(wx.Font(14, wx.DEFAULT, wx.NORMAL, wx.BOLD))
......
This diff is collapsed.
class Project:
# project_id is optional because it's not necessary when creating a Project
# object to send.
def __init__(self, proj_name, proj_description=None, proj_id=None):
# proj_id is the identifier key when getting projects from the API.
self.project_name = proj_name
self.project_description = str(proj_description)
self.project_id = str(proj_id)
def get_id(self):
return self.project_id
def get_name(self):
return self.project_name
def get_description(self):
return self.project_description
def get_dict(self): # for sending
return {"name": self.project_name,
"projectDescription": self.project_description}
def __str__(self):
return "ID:" + self.project_id + " Name:" + self.project_name + \
" Description: " + self.project_description
class Project:
# project_id is optional because it's not necessary when creating a Project
# object to send.
def __init__(self, proj_name, proj_description=None, proj_id=None):
# proj_id is the identifier key when getting projects from the API.
self.project_name = proj_name
self.project_description = str(proj_description)
self.project_id = str(proj_id)
def get_id(self):
return self.project_id
def get_name(self):
return self.project_name
def get_description(self):
return self.project_description
def get_dict(self): # for sending
return {"name": self.project_name,
"projectDescription": self.project_description}
def __str__(self):
return "ID:" + self.project_id + " Name:" + self.project_name + \
" Description: " + self.project_description
import json
import logging
"""
A Sample will store (key: value) pairs using a dictionary.
e.g {"sequencerSampleId": "01-1111"}
Keys: 'sampleName','description','sequencerSampleId','sampleProject'
"""
class Sample(object):
def __init__(self, new_samp_dict, run=None, sample_number=None):
self.sample_dict = dict(new_samp_dict)
self.seq_file = None
self._run = run
self._sample_number = sample_number
self._already_uploaded = False
def get_id(self):
# When pulling sample records from the server, the sample name *is* the
# identifier for the sample, so if it's not specified in the dictionary
# that we're using to build this sample, set it as the sample name that
# we got from the server.
try:
return self.sample_dict["sequencerSampleId"]
except KeyError:
return self.sample_dict["sampleName"]
@property
def already_uploaded(self):
return self._already_uploaded
@already_uploaded.setter
def already_uploaded(self, already_uploaded=False):
self._already_uploaded = already_uploaded
@property
def sample_name(self):
return self.get("sampleName")
@property
def sample_number(self):
return self._sample_number
def get_project_id(self):
return self.get("sampleProject")
def get_dict(self):
return self.sample_dict
def __getitem__(self, key):
ret_val = None
if key in self.sample_dict:
ret_val = self.sample_dict[key]
return ret_val
def get(self, key):
return self.__getitem__(key)
def get_sample_metadata(self):
return self.seq_file.get_properties()
def get_files(self):
return self.seq_file.get_files()
def get_files_size(self):
return self.seq_file.get_files_size()
def set_seq_file(self, seq_file):
self.seq_file = seq_file
def is_paired_end(self):
return len(self.seq_file.get_files()) == 2
def __str__(self):
return str(self.sample_dict) + str(self.seq_file)
@property
def upload_progress_topic(self):
return self._run.upload_progress_topic + "." + self.get_id()
@property
def upload_started_topic(self):
return self._run.upload_started_topic + "." + self.get_id()
@property
def upload_completed_topic(self):
return self._run.upload_completed_topic + "." + self.get_id()
@property
def upload_failed_topic(self):
return self._run.upload_failed_topic + "." + self.get_id()
@property
def online_validation_topic(self):
return self._run.online_validation_topic + "." + self.get_id()
@property
def run(self):
return self._run
@run.setter
def run(self, run):
logging.info("Setting run.")
self._run = run
class JsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Sample):
sample_dict = dict(obj.get_dict())
# get sample dict and make a copy of it
sample_dict.pop("sampleProject")
if "sequencerSampleId" in sample_dict:
# if the sample ID field is populated, then we've just Finished
# reading the run from disk and we're preparing to send data
# to the server. The server is using the sample ID field as the
# name of the sample, so overwrite whatever we *were* using to
# find files with the sample ID field.
sample_dict["sampleName"] = sample_dict["sequencerSampleId"]
return sample_dict
else:
return json.JSONEncoder.default(self, obj)
import json
import logging
"""
A Sample will store (key: value) pairs using a dictionary.
e.g {"sequencerSampleId": "01-1111"}
Keys: 'sampleName','description','sequencerSampleId','sampleProject'
"""
class Sample(object):
def __init__(self, new_samp_dict, run=None, sample_number=None):
self.sample_dict = dict(new_samp_dict)
self.seq_file = None
self._run = run
self._sample_number = sample_number
self._already_uploaded = False
def get_id(self):
# When pulling sample records from the server, the sample name *is* the
# identifier for the sample, so if it's not specified in the dictionary
# that we're using to build this sample, set it as the sample name that
# we got from the server.
try:
return self.sample_dict["sequencerSampleId"]
except KeyError:
return self.sample_dict["sampleName"]
@property
def already_uploaded(self):
return self._already_uploaded
@already_uploaded.setter
def already_uploaded(self, already_uploaded=False):
self._already_uploaded = already_uploaded
@property
def sample_name(self):
return self.get("sampleName")
@property
def sample_number(self):
return self._sample_number
def get_project_id(self):
return self.get("sampleProject")
def get_dict(self):
return self.sample_dict
def __getitem__(self, key):
ret_val = None
if key in self.sample_dict:
ret_val = self.sample_dict[key]
return ret_val
def get(self, key):
return self.__getitem__(key)
def get_sample_metadata(self):
return self.seq_file.get_properties()
def get_files(self):
return self.seq_file.get_files()
def get_files_size(self):
return self.seq_file.get_files_size()
def set_seq_file(self, seq_file):
self.seq_file = seq_file
def is_paired_end(self):
return len(self.seq_file.get_files()) == 2
def __str__(self):
return str(self.sample_dict) + str(self.seq_file)
@property
def upload_progress_topic(self):
return self._run.upload_progress_topic + "." + self.get_id()
@property
def upload_started_topic(self):
return self._run.upload_started_topic + "." + self.get_id()
@property
def upload_completed_topic(self):
return self._run.upload_completed_topic + "." + self.get_id()
@property
def upload_failed_topic(self):
return self._run.upload_failed_topic + "." + self.get_id()
@property
def online_validation_topic(self):
return self._run.online_validation_topic + "." + self.get_id()
@property
def run(self):
return self._run
@run.setter
def run(self, run):
logging.info("Setting run.")
self._run = run
class JsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Sample):
sample_dict = dict(obj.get_dict())
# get sample dict and make a copy of it
sample_dict.pop("sampleProject")
if "sequencerSampleId" in sample_dict:
# if the sample ID field is populated, then we've just Finished
# reading the run from disk and we're preparing to send data
# to the server. The server is using the sample ID field as the
# name of the sample, so overwrite whatever we *were* using to
# find files with the sample ID field.
sample_dict["sampleName"] = sample_dict["sequencerSampleId"]
return sample_dict
else:
return json.JSONEncoder.default(self, obj)
from os import path
"""
Holds files and Sample metadata:
samplePlate
sampleWell
i7IndexID
index
i5IndexID
index2
etc.
"""
class SequenceFile:
def __init__(self, properties_dict, file_list):
self.properties_dict = properties_dict # Sample metadata
self.file_list = file_list
self.file_list.sort()
def get_properties(self):
return self.properties_dict
def get(self, key):
retVal = None
if self.properties_dict in key:
retVal = self.properties_dict[key]
return retVal
def get_files_size(self):
return sum([path.getsize(file) for file in self.file_list])
def get_files(self):
return self.file_list
def __str__(self):
return str(self.properties_dict) + str(self.file_list)
from os import path
"""
Holds files and Sample metadata:
samplePlate
sampleWell
i7IndexID
index
i5IndexID
index2
etc.
"""
class SequenceFile:
def __init__(self, properties_dict, file_list):
self.properties_dict = properties_dict # Sample metadata
self.file_list = file_list
self.file_list.sort()
def get_properties(self):
return self.properties_dict
def get(self, key):
retVal = None
if self.properties_dict in key:
retVal = self.properties_dict[key]
return retVal
def get_files_size(self):
return sum([path.getsize(file) for file in self.file_list])
def get_files(self):
return self.file_list
def __str__(self):
return str(self.properties_dict) + str(self.file_list)
import os
import logging
class SequencingRun(object):
def __init__(self, metadata = None, sample_list = None, sample_sheet = None):
self._sample_list = sample_list
self._metadata = metadata
if sample_sheet is None:
raise ValueError("Sample sheet cannot be None!")
self._sample_sheet = sample_sheet
self._sample_sheet_dir = os.path.dirname(sample_sheet)
self._sample_sheet_name = os.path.basename(self._sample_sheet_dir)
for sample in self._sample_list:
logging.info("Setting run.")
sample.run = self
@property
def metadata(self):
return self._metadata
@metadata.setter
def metadata(self, metadata_dict):
self._metadata = metadata_dict
def get_workflow(self):
return self._metadata["workflow"]
@property
def sample_list(self):
return self._sample_list
@sample_list.setter
def sample_list(self, sample_list):
self._sample_list = sample_list
@property
def uploaded_samples(self):
return filter(lambda sample: sample.already_uploaded, self.sample_list)
@property
def samples_to_upload(self):
return filter(lambda sample: not sample.already_uploaded, self.sample_list)
def get_sample(self, sample_id):
for sample in self._sample_list:
if sample.get_id() == sample_id:
return sample
else:
raise ValueError("No sample with id {} found!.".format(sample_id))
def set_files(self, sample_id, file_list):
for sample in self._sample_list:
if sample.get_id() == sample_id:
sample.set_files(file_list)
break
def get_files(self, sample_id):
sample = self._get_sample(sample_id)
return sample.get_files()
@property
def sample_sheet(self):
return self._sample_sheet
@sample_sheet.setter
def sample_sheet(self, sample_sheet):
self._sample_sheet = sample_sheet
@property
def sample_sheet_dir(self):
return self._sample_sheet_dir
@property
def sample_sheet_name(self):
return self._sample_sheet_name
@property
def upload_started_topic(self):
return self._sample_sheet_name + ".upload_started"
@property
def upload_progress_topic(self):
return self._sample_sheet_name + ".upload_progress"
@property
def upload_completed_topic(self):
return self._sample_sheet_name + ".upload_completed"
@property
def upload_failed_topic(self):
return self._sample_sheet_name + ".upload_failed"
@property
def offline_validation_topic(self):
return self._sample_sheet_name + ".offline_validation"
@property
def online_validation_topic(self):
return self._sample_sheet_name + ".online_validation"
import os
import logging
class SequencingRun(object):
def __init__(self, metadata = None, sample_list = None, sample_sheet = None):
self._sample_list = sample_list
self._metadata = metadata
if sample_sheet is None:
raise ValueError("Sample sheet cannot be None!")
self._sample_sheet = sample_sheet
self._sample_sheet_dir = os.path.dirname(sample_sheet)
self._sample_sheet_name = os.path.basename(self._sample_sheet_dir)
for sample in self._sample_list:
logging.info("Setting run.")
sample.run = self
@property
def metadata(self):
return self._metadata
@metadata.setter
def metadata(self, metadata_dict):
self._metadata = metadata_dict
def get_workflow(self):
return self._metadata["workflow"]
@property
def sample_list(self):
return self._sample_list
@sample_list.setter
def sample_list(self, sample_list):
self._sample_list = sample_list
@property
def uploaded_samples(self):
return filter(lambda sample: sample.already_uploaded, self.sample_list)
@property
def samples_to_upload(self):
return filter(lambda sample: not sample.already_uploaded, self.sample_list)
def get_sample(self, sample_id):
for sample in self._sample_list:
if sample.get_id() == sample_id:
return sample
else:
raise ValueError("No sample with id {} found!.".format(sample_id))
def set_files(self, sample_id, file_list):
for sample in self._sample_list:
if sample.get_id() == sample_id:
sample.set_files(file_list)
break
def get_files(self, sample_id):
sample = self._get_sample(sample_id)
return sample.get_files()
@property
def sample_sheet(self):
return self._sample_sheet
@sample_sheet.setter
def sample_sheet(self, sample_sheet):
self._sample_sheet = sample_sheet
@property
def sample_sheet_dir(self):
return self._sample_sheet_dir
@property
def sample_sheet_name(self):
return self._sample_sheet_name
@property
def upload_started_topic(self):
return self._sample_sheet_name + ".upload_started"
@property
def upload_progress_topic(self):
return self._sample_sheet_name + ".upload_progress"
@property
def upload_completed_topic(self):
return self._sample_sheet_name + ".upload_completed"
@property
def upload_failed_topic(self):
return self._sample_sheet_name + ".upload_failed"
@property
def offline_validation_topic(self):
return self._sample_sheet_name + ".offline_validation"
@property
def online_validation_topic(self):
return self._sample_sheet_name + ".online_validation"
class ValidationResult:
def __init__(self):
self.valid = None
self.error_msgs = []
def add_error_msg(self, msg):
self.error_msgs.append(msg)
def set_valid(self, boolean):
self.valid = boolean
def is_valid(self):
return self.valid
def error_count(self):
return len(self.error_msgs)
def error_list(self):
return self.error_msgs
def get_errors(self):
ret_val = ""
if len(self.error_msgs) > 0:
ret_val = "\n".join(self.error_msgs)
else:
ret_val = "No error messages"
return ret_val
class ValidationResult:
def __init__(self):
self.valid = None
self.error_msgs = []
def add_error_msg(self, msg):
self.error_msgs.append(msg)
def set_valid(self, boolean):
self.valid = boolean
def is_valid(self):
return self.valid
def error_count(self):
return len(self.error_msgs)
def error_list(self):
return self.error_msgs
def get_errors(self):
ret_val = ""
if len(self.error_msgs) > 0:
ret_val = "\n".join(self.error_msgs)
else:
ret_val = "No error messages"
return ret_val
This diff is collapsed.
......@@ -16,9 +16,15 @@ Install pip and wxpython:
### virtualenv usage
Install virtualenv
Install virtualenv and setuptools
$ pip install virtualenv
$ pip install setuptools
If you already have these packages installed, ensure they are up to date
$ pip install virtualenv -U
$ pip install setuptools -U