Commit 0260520e authored by Thomas Matthews's avatar Thomas Matthews

Merge branch 're-build' into 're-master'

Build for Linux

See merge request !65
parents 09d031b9 bbf2801d
......@@ -2,17 +2,13 @@
*.swp
*.coverage
*.bak
Tests/integrationTests/repos/*
*.log
*.miseqUploaderInfo
*.miseqUploaderComplete
docs/_build/*
build/
pynsist_pkgs
wxPython*
.virtualenv
.cache
.idea/
.python-version
.pytest_cache/
cli_*
\ No newline at end of file
before_script:
- rm -rf /tmp/irida/*
build-against-irida-master:
script: make test IRIDA_VERSION=master
build-against-irida-development:
script: make test IRIDA_VERSION=development
pep8:
script: "scripts/verifyPEP8.sh"
#build:
# script: make windows
# artifacts:
# paths:
# - build/nsis/IRIDA_Uploader_*.exe
SHELL=/bin/bash
IRIDA_VERSION?=master
all: clean requirements windows
all: clean requirements
clean:
rm -rf .cache
rm -rf pynsist_pkgs
rm -rf .virtualenv
rm -rf build
find -name "*pyc" -delete
rm -rf Tests/integrationTests/repos/
requirements:
virtualenv -p python3.7 .virtualenv
python3 -m venv .virtualenv
source .virtualenv/bin/activate
pip install -r requirements.txt
deactivate
windows: requirements
wget --no-clobber http://downloads.sourceforge.net/project/wxpython/wxPython/3.0.2.0/wxPython3.0-win32-3.0.2.0-py27.exe
rm -rf pynsist_pkgs
innoextract -d pynsist_pkgs -s wxPython3.0-win32-3.0.2.0-py27.exe
mv pynsist_pkgs/app/wx-3.0-msw/wx pynsist_pkgs/
rm -rf pynsist_pkgs/{app,code*}
source .virtualenv/bin/activate
pynsist irida-uploader.cfg 2>&1 > /dev/null
test: clean requirements
source .virtualenv/bin/activate
xvfb-run --auto-servernum --server-num=1 py.test --integration --irida-version=$(IRIDA_VERSION)
......
......@@ -8,6 +8,7 @@ from http import HTTPStatus
from os import path
from rauth import OAuth2Service
from requests import ConnectionError
from requests.adapters import HTTPAdapter
from urllib.parse import urljoin, urlparse
from urllib.error import URLError
......@@ -22,7 +23,7 @@ from . import exceptions
class ApiCalls(object):
def __init__(self, client_id, client_secret,
base_url, username, password, max_wait_time=20):
base_url, username, password, max_wait_time=20, http_max_retries=5):
"""
Create OAuth2Session and store it
......@@ -42,6 +43,7 @@ class ApiCalls(object):
self.username = username
self.password = password
self.max_wait_time = max_wait_time
self.http_max_retries = http_max_retries
self._stop_upload = False
......@@ -71,7 +73,11 @@ class ApiCalls(object):
def _reinitialize_session(self):
oauth_service = self._get_oauth_service()
access_token = self._get_access_token(oauth_service)
self._session_instance = oauth_service.get_session(access_token)
_sess = oauth_service.get_session(access_token)
# We add a HTTPAdapter with max retries so we don't fail out if one request gets lost
_sess.mount('https://', HTTPAdapter(max_retries=self.http_max_retries))
_sess.mount('http://', HTTPAdapter(max_retries=self.http_max_retries))
self._session_instance = _sess
def _create_session(self):
"""
......@@ -703,9 +709,13 @@ class ApiCalls(object):
metadata_dict["uploadStatus"] = "UPLOADING"
keys_to_remove = []
for key in metadata_dict.keys():
if key not in acceptable_properties:
del metadata_dict[key]
keys_to_remove.append(key)
for key in keys_to_remove:
del metadata_dict[key]
json_obj = json.dumps(metadata_dict)
......
import os
import logging
import global_settings
from configparser import RawConfigParser, NoOptionError
from appdirs import user_config_dir
from collections import namedtuple
# Initialize and setup components for the configuration file
user_config_file = os.path.join(user_config_dir("iridaUploader"), "config.conf")
# If a config file was passed as a parameter, use it, else use the default config directory
if global_settings.config_file:
user_config_file = global_settings.config_file
if not os.path.exists(user_config_file):
logging.error("Could not find config file: {}".format(user_config_file))
exit(1)
else:
user_config_file = os.path.join(user_config_dir("irida-uploader"), "config.conf")
conf_parser = RawConfigParser()
logging.debug("User config file: " + user_config_file)
......@@ -21,20 +31,39 @@ default_settings = [SettingsDefault._make(["client_id", ""]),
SettingsDefault._make(["base_url", ""]),
SettingsDefault._make(["parser", "directory"])]
load_from_file = os.path.exists(user_config_file)
# Loading config from file
if os.path.exists(user_config_file):
logging.debug("Loading configuration settings from {}".format(user_config_file))
if load_from_file:
try:
logging.debug("Loading configuration settings from {}".format(user_config_file))
conf_parser.read(user_config_file)
conf_parser.read(user_config_file)
for config in default_settings:
if not conf_parser.has_option("Settings", config.setting):
conf_parser.set("Settings", config.setting, config.default_value)
else:
logging.debug("No default config file exists, loading defaults.")
for config in default_settings:
if not conf_parser.has_option("Settings", config.setting):
conf_parser.set("Settings", config.setting, config.default_value)
except:
logging.warning("Error occurred when trying to load config file")
logging.error("Config file {} is not valid.".format(user_config_file))
exit(1)
# Create a new user config file and fill with defaults
if not load_from_file:
logging.info("Creating a new config file.")
conf_parser.add_section("Settings")
if not os.path.exists(os.path.dirname(user_config_file)):
os.makedirs(os.path.dirname(user_config_file))
for config in default_settings:
conf_parser.set("Settings", config.setting, config.default_value)
conf_parser.set("Settings", config.setting, config.default_value)
with open(user_config_file, 'w') as config_file:
conf_parser.write(config_file)
logging.info("Config File Created: " + str(user_config_file))
logging.info("Please edit your config file to connect to IRIDA")
def read_config_option(key, expected_type=None, default_value=None):
......@@ -74,8 +103,5 @@ def write_config_option(field_name, field_value):
conf_parser.set("Settings", field_name, field_value)
if not os.path.exists(os.path.dirname(user_config_file)):
os.makedirs(os.path.dirname(user_config_file))
with open(user_config_file, 'wb') as config_file:
conf_parser.write(config_file)
with open(user_config_file, 'w') as c_file:
conf_parser.write(c_file)
from . import logger
from . import cli_entry
from appdirs import user_log_dir
import os
import logging
log_directory_name = "irida_uploader"
log_file_name = 'irida-uploader.log'
if not os.path.exists(user_log_dir(log_directory_name)):
os.makedirs(user_log_dir(log_directory_name))
log_format = '%(asctime)s %(levelname)-8s %(message)s'
root_logger = logging.getLogger()
root_logger.handlers = []
# Log to file
logging.basicConfig(
format=log_format,
level=logging.DEBUG,
datefmt='%Y-%m-%d %H:%M:%S',
filename=os.path.join(user_log_dir(log_directory_name), log_file_name),
filemode='w')
# Log to the user
console = logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(logging.Formatter(log_format))
root_logger.addHandler(console)
"""
This file manages globals across files.
It is used to handle passing a config file as a parameter
"""
config_file = None
[Application]
name=IRIDA Uploader
version=2.1.1-SNAPSHOT
entry_point=run_IRIDA_Uploader:main
icon=GUI/images/iu.ico
[Python]
version=2.7.11
bitness=32
[Include]
packages = rauth
requests
appdirs
API
Exceptions
Model
Parsers
Validation
github3
uritemplate
GUI
files = irida-uploader.cfg
[Build]
nsi_template=irida-uploader.nsi
[% extends "pyapp_w_pylauncher.nsi" %]
; from nsis documentation: http://nsis.sourceforge.net/Auto-uninstall_old_before_installing_new
[% block sections %]
[[ super() ]]
Function .onInit
ReadRegStr $R0 HKLM \
"Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" \
"UninstallString"
StrCmp $R0 "" done
MessageBox MB_OKCANCEL|MB_ICONINFORMATION \
"${PRODUCT_NAME} is already installed. $\n$\nThe old version will be uninstalled \
before installing ${PRODUCT_VERSION}. Click OK to remove the \
old version or Cancel to cancel the upgrade." \
IDOK uninst
Abort
;Run the uninstaller
uninst:
ClearErrors
ExecWait '$R0 _?=$INSTDIR' ;Do not copy the uninstaller to a temp file
IfErrors no_remove_uninstaller done
;You can either use Delete /REBOOTOK in the uninstaller or add some code
;here to remove the uninstaller. Use a registry key to check
;whether the user has chosen to uninstall. If you are using an uninstaller
;components page, make sure all sections are uninstalled.
no_remove_uninstaller:
done:
FunctionEnd
[% endblock sections %]
#!/usr/bin/env bash
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${ROOT_DIR}/.virtualenv/bin/activate
if [[ $? -eq 0 ]]; then
true
else
echo "Failed to start Uploader Environment. Please run 'make' before running this script"
exit 1
fi
python upload_run.py "$@"
......@@ -81,8 +81,9 @@ class Parser:
sample_sheet_file_name = 'SampleList.csv'
file_list = next(os.walk(directory))[2] # Gets the list of files in the directory
if sample_sheet_file_name not in file_list:
logging.error("The directory {} has no sample sheet file".format(directory))
raise exceptions.DirectoryError("The directory {} has no sample sheet file with the name {}"
logging.error("No sample sheet file in the Directory Upload format found")
raise exceptions.DirectoryError("The directory {} has no sample sheet file in the Directory Upload format "
"with the name {}"
"".format(directory, sample_sheet_file_name), directory)
else:
logging.debug("Sample sheet found")
......
......@@ -81,8 +81,10 @@ class Parser:
sample_sheet_file_name = 'SampleSheet.csv'
file_list = next(os.walk(directory))[2] # Gets the list of files in the directory
if sample_sheet_file_name not in file_list:
logging.error("The directory {} has no sample sheet file".format(directory))
raise exceptions.DirectoryError("The directory {} has no sample sheet file".format(directory), directory)
logging.error("No sample sheet file in the MiSeq format found")
raise exceptions.DirectoryError("The directory {} has no sample sheet file in the MiSeq format"
" with the name {}"
.format(directory, sample_sheet_file_name), directory)
else:
logging.debug("Sample sheet found")
return os.path.join(directory, sample_sheet_file_name)
......
......@@ -150,7 +150,7 @@ def _parse_sample_list(sample_sheet_file):
logging.info("Looking for files with pattern {}".format(file_pattern))
regex = re.compile(file_pattern)
pf_list = filter(regex.search, data_dir_file_list)
pf_list = list(filter(regex.search, data_dir_file_list))
if not pf_list:
# we **still** didn't find anything. It's pretty likely, then that
......
#!/usr/bin/env python
import wx
import logging
import webbrowser
import ConfigParser
import argparse
import wx.lib.delayedresult as dr
import wx.lib.agw.hyperlink as hl
from os import path, makedirs
from distutils.version import LooseVersion
from github3 import GitHub
from GUI import UploaderAppFrame, SettingsDialog
from appdirs import user_config_dir, user_log_dir
from wx.lib.pubsub import pub
path_to_module = path.dirname(__file__)
app_config = path.join(path_to_module, 'irida-uploader.cfg')
if not path.isfile(app_config):
app_config = path.join(path_to_module, '..', 'irida-uploader.cfg')
if not path.exists(user_log_dir("iridaUploader")):
makedirs(user_log_dir("iridaUploader"))
log_format = '%(asctime)s %(levelname)s\t%(filename)s:%(funcName)s:%(lineno)d - %(message)s'
# if any logging gets called before `basicConfig`, our attempts to configure the
# logging here will be clobbered. This removes any existing handlers that might
# have been set up when some other log message was printed, so that we can
# actually configure the logging the way we want.
logging.getLogger().handlers = []
logging.basicConfig(level=logging.DEBUG,
filename=path.join(user_log_dir("iridaUploader"), 'irida-uploader.log'),
format=log_format,
filemode='w')
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
console.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(console)
class Uploader(wx.App):
def __init__(self, show_new_ui=False, redirect=False, filename=None):
wx.App.__init__(self, redirect, filename)
self.get_app_info()
self.check_for_update()
user_config_file = path.join(user_config_dir("iridaUploader"), "config.conf")
if not path.exists(user_config_file):
dialog = SettingsDialog(first_run=True)
dialog.ShowModal()
self._show_main_app()
def _show_main_app(self):
frame = UploaderAppFrame(app_name=self.__app_name__, app_version=self.__app_version__, app_url=self.url)
frame.Show()
def get_app_info(self):
config_parser = ConfigParser.ConfigParser()
config_parser.read(app_config)
self.__app_version__ = config_parser.get('Application', 'version', None)
self.__app_name__ = config_parser.get('Application', 'name', None)
@property
def url(self):
return "https://github.com/phac-nml/irida-miseq-uploader"
def check_for_update(self):
def find_update():
logging.debug("Checking remote for new updates.")
try:
gh = GitHub()
repo = gh.repository("phac-nml", "irida-miseq-uploader")
# get the latest tag from github
return next(repo.iter_tags(number=1))
except:
logging.warn("Couldn't reach github to check for new version.")
raise
def handle_update(result):
latest_tag = result.get()
logging.debug("Found latest version: [{}]".format(latest_tag))
release_url = self.url + "/releases/latest"
if LooseVersion(self.__app_version__) < LooseVersion(latest_tag.name):
logging.info("Newer version found.")
dialog = NewVersionMessageDialog(
parent=None,
id=wx.ID_ANY,
message=("A new version of the IRIDA MiSeq "
"Uploader tool is available. You can"
" download the latest version from "),
title="IRIDA MiSeq Uploader update available",
download_url=release_url,
style=wx.CAPTION|wx.CLOSE_BOX|wx.STAY_ON_TOP)
dialog.ShowModal()
dialog.Destroy()
else:
logging.debug("No new versions found.")
dr.startWorker(handle_update, find_update)
class NewVersionMessageDialog(wx.Dialog):
def __init__(self, parent, id, title, message, download_url, size=wx.DefaultSize, pos=wx.DefaultPosition, style=wx.DEFAULT_DIALOG_STYLE, name='dialog'):
wx.Dialog.__init__(self, parent, id, title, pos, size, style, name)
label = wx.StaticText(self, label=message)
button = wx.Button(self, id=wx.ID_OK, label="Close")
button.SetDefault()
line = wx.StaticLine(self, wx.ID_ANY, size=(20, -1), style=wx.LI_HORIZONTAL)
download_ctrl = hl.HyperLinkCtrl(self, wx.ID_ANY, download_url, URL=download_url)
sizer = wx.BoxSizer(wx.VERTICAL)
button_sizer = wx.StdDialogButtonSizer()
button_sizer.AddButton(button)
button_sizer.Realize()
sizer.Add(label, 0, wx.ALIGN_CENTER|wx.ALL, 5)
sizer.Add(download_ctrl, 0, wx.ALL, 10)
sizer.Add(line, 0, wx.GROW|wx.ALIGN_CENTER_VERTICAL|wx.RIGHT|wx.TOP, 5)
sizer.Add(button_sizer, 0, wx.ALIGN_CENTER_VERTICAL|wx.ALL, 5)
self.SetSizer(sizer)
sizer.Fit(self)
def main():
app = Uploader()
app.MainLoop()
if __name__ == "__main__":
main()
......@@ -4,8 +4,8 @@ path_of_this_file=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
iu_path=$(dirname $path_of_this_file)
make requirements
source .virtualenv/bin/activate
pep8 --exclude="Tests/integrationTests/repos/*",".git","bin","include","lib",\
"local" --statistics "$iu_path/API" "$iu_path/Exceptions" "$iu_path/GUI" \
"$iu_path/Model" "$iu_path/Parsers" "$iu_path/scripts" "$iu_path/Tests" \
"$iu_path/Validation"
pycodestyle --show-source --ignore=E501 --exclude="Tests/integrationTests/repos/*",".git","bin","include","lib",\
"local" --statistics "$iu_path/api" "$iu_path/config" "$iu_path/core" \
"$iu_path/messaging" "$iu_path/model" "$iu_path/parser" "$iu_path/progress" \
"$iu_path/scripts"
deactivate
#!/bin/bash
VIRTUAL_ENV=".virtualenv"
pyVersion=$(ls $VIRTUAL_ENV/lib/ | grep python2.*)
python2 -c "import sys;import os;import wx;sys.stdout.write(os.path.dirname(wx.__file__)[:-3])" > $VIRTUAL_ENV/lib/$pyVersion/site-packages/wx.pth
import argparse
import global_settings
class ConfigAction(argparse.Action):
"""
This class is called when a config option is passed to via command line.
It sets the global_settings config_file option to whatever was passed in the argument
"""
def __init__(self, option_strings, dest, nargs=None, **kwargs):
"""
Boilerplate for an argparse Action
:param option_strings: A list of command-line option strings which
should be associated with this action.
:param dest: The name of the attribute to hold the created object(s)
:param nargs: The number of command-line arguments that should be
consumed. See argparse docs for details
:param kwargs: All other options. See argparse docs for details
"""
if nargs is not None:
raise ValueError("nargs not allowed")
super(ConfigAction, self).__init__(option_strings, dest, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
"""
Gets executed when the argument is included from the command line
:param parser: Not used.
:param namespace: Not used.
:param values: The value passed via the command line. The global config_file variable will be set to this.
:param option_string: not used
:return: None
"""
global_settings.config_file = values
# Set up an argument parser. We are using defaults to stay consistent with other software.
# description gets added to the usage statements
argument_parser = argparse.ArgumentParser(description='This program parses sequencing runs and uploads them to IRIDA.')
# Our main argument. It is required or else an error will be thrown when the program is run
argument_parser.add_argument('directory',
help='Location of sequencing run to upload')
# Optional argument, for using an alternative config file.
argument_parser.add_argument('-c', '--config',
action=ConfigAction,
help='Path to an alternative configuration file.'
'This overrides the default config file in the config directory')
def main():
# Parse the arguments passed from the command line and start the upload
args = argument_parser.parse_args()
upload(args.directory)
def upload(run_directory):
# We import here instead of at the top so argparse can set the config files before the config module is imported
from core import cli_entry
cli_entry.validate_and_upload_single_entry(run_directory)
# This is called when the program is run for the first time
if __name__ == "__main__":
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment