Commit c63bcc3d authored by Joel Thiessen's avatar Joel Thiessen

import from JSON parameter

parent b5f46768
......@@ -6,3 +6,4 @@ skip
.ropeproject
.coverage
htmlcov
irida_import/sample.dat
......@@ -95,8 +95,6 @@ Then to run the tests, navigate to `$GALAXY_ROOT/tools/irida_import/` and invok
py.test
```
In addition, one or more of the paths in `$GALAXY_ROOT/tools/irida_import/prelim_json.json` should be modified to point to local files to test uploading with.
#### Generating Code Coverage Reports:
......
......@@ -6,6 +6,7 @@ import json
from sample import Sample
from sample_file import SampleFile
import optparse
import os.path
class IridaImport:
......@@ -21,58 +22,28 @@ class IridaImport:
ILLUMINA_PATH = '/illumina_reads'
REFERENCE_PATH = '/references'
def get_samples(self, param_dict):
def get_samples(self, json_params_dict):
"""
Create sample objects from a dictionary.
:type param_dict: dict
:param param_dict: a dictionary to parse. See one of the test
:type json_params_dict: dict
:param json_params__dict: a dictionary to parse. See one of the test
json files for formating information (the format will likely
change soon)
:return: list of output samples
"""
samples_remain = True
sample_num = 1
samples = []
logging.debug(param_dict)
while samples_remain:
sample_name_key = 'sample' + str(sample_num) + '_name'
if sample_name_key not in param_dict:
logging.debug('no sample found for key: ' + sample_name_key)
samples_remain = False
else:
logging.debug('found sample for key: ' + sample_name_key +
' with value: ' + param_dict[sample_name_key])
sample_path_key = 'sample' + str(sample_num) + '_path'
sample_name = param_dict[sample_name_key]
sample_path = param_dict[sample_path_key]
sample = Sample(sample_name, sample_path)
sample_files_remain = True
sample_file_num = 1
while sample_files_remain:
sample_file_key = 'sample' + str(sample_num) + \
'_file' + str(sample_file_num) + '_path'
if sample_file_key not in param_dict:
sample_files_remain = False
logging.debug('no sample_file for key: ' +
sample_file_key)
else:
logging.debug('found sample_file for key: ' +
sample_file_key + 'with value ' +
param_dict[sample_file_key])
sample_file_path = param_dict[sample_file_key]
sample_file = SampleFile(sample_file_path)
logging.debug("sample file made:" + str(sample_file))
sample.sample_files.append(sample_file)
sample_file_num += 1
samples.append(sample)
sample_num += 1
for sample_input in json_params_dict['_embedded']['samples']:
sample_name = sample_input['name']
sample_path = sample_input['_links']['self']['href']
sample = Sample(sample_name, sample_path)
for sample_file_input in sample_input['_embedded']['sample_files']:
sample_file_path = sample_file_input['_links']['self']['href']
sample.sample_files.append(SampleFile(sample_file_path))
samples.append(sample)
return samples
def get_first_or_make_lib(self, desired_lib_name):
......@@ -204,22 +175,25 @@ class IridaImport:
(prefix, sample_file.path))
if prefix == 'file':
# Get e.g. '/folder/folder56/myfile.fastq' from
# 'file://folder/folder56/myfile.fastq'
file_path = sample_file.path.split(':/')[1]
# 'file:///folder/folder56/myfile.fastq'
file_path = sample_file.path.split('://')[1]
logging.debug("File path is"+file_path)
folder_id = self.reg_gi.libraries.get_folders(
self.library.id,
name=sample_folder_path)[0]['id']
uploaded = self.reg_gi.libraries.upload_from_galaxy_filesystem(
self.library.id,
file_path,
folder_id=folder_id,
link_data_only='link_to_files')
logging.debug('wrote file!')
if os.path.isfile(file_path):
uploaded = self.reg_gi.libraries.upload_from_galaxy_filesystem(
self.library.id,
file_path,
folder_id=folder_id,
link_data_only='link_to_files')
logging.debug('wrote file!')
else:
raise IOError('file not found: '+file_path)
return uploaded
# TODO: finish this method
def assign_ownership_if_nec(self, sample):
"""
Assign ownership to the files in a sample, if neccessary
......@@ -227,6 +201,7 @@ class IridaImport:
:type sample: Sample
:param sample: the sample who's sample files will be assigned ownership
"""
# TODO: finish this method
return True # neccessary here
def import_to_galaxy(self, json_parameter_file, irida_info):
......@@ -246,8 +221,9 @@ class IridaImport:
json.dumps(full_param_dict, indent=2))
param_dict = full_param_dict['param_dict']
json_params_dict = json.loads(param_dict['json_params'])
desired_lib_name = param_dict['library_name']
desired_lib_name = json_params_dict['_embedded']['library']['name']
self.gi = GalaxyInstance(self.GALAXY_URL, self.ADMIN_KEY)
......@@ -258,7 +234,7 @@ class IridaImport:
key=self.ADMIN_KEY)
# Each sample contains a list of sample files
samples = self.get_samples(param_dict)
samples = self.get_samples(json_params_dict)
# samples = self.get_samples(irida_info_dict)
# Set up the library
self.library = self.get_first_or_make_lib(desired_lib_name)
......@@ -300,7 +276,7 @@ if __name__ == '__main__':
# this test JSON file does not have to be configured to run the tests
logging.debug("Opening a test json file")
test_json_file = \
'/home/jthiessen/galaxy-dist/tools/irida_import/sample.json'
'/home/jthiessen/galaxy-dist/tools/irida_import_tool_for_galaxy/irida_import/sample.dat'
importer = IridaImport()
......
{"param_dict": {"__new_file_path__": "/home/jthiessen/galaxy-dist/database/tmp", "runtool_btn": "Execute", "__user_id__": "1", "output": "/home/jthiessen/galaxy-dist/database/files/000/dataset_836.dat", "GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "chromInfo": "/home/jthiessen/galaxy-dist/tool-data/shared/ucsc/chrom/?.len", "__tool_data_path__": "/home/jthiessen/galaxy-dist/tool-data", "userId": "1", "userEmail": "joel.thiessen@phac-aspc.gc.ca", "dbkey": "?", "__user_name__": "jthiessen", "__datatypes_config__": "/home/jthiessen/galaxy-dist/database/tmp/tmpe5DJvI", "__app__": "galaxy.app:UniverseApplication", "__user__": "galaxy.model:User", "GALAXY_DATA_INDEX_DIR": "/home/jthiessen/galaxy-dist/tool-data", "__root_dir__": "/home/jthiessen/galaxy-dist", "__user_email__": "joel.thiessen@phac-aspc.gc.ca", "__admin_users__": "joel.thiessen@phac-aspc.gc.ca", "json_params": "{\"_embedded\":{\"library\":{\"name\":\"project4-fbristow\"},\"samples\":[{\"name\":\"01-1111\",\"_links\":{\"self\":{\"href\":\"http://sample/path/will/go/here\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/3/2/01-1111_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/4/2/01-1111_S1_L001_R2_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/9/2/01-1111_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/10/2/01-1111_S1_L001_R2_001.fastq\"}}}]}},{\"name\":\"02-2222\",\"_links\":{\"self\":{\"href\":\"http://sample/path/will/go/here\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/1/2/02-2222_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/2/2/02-2222_S1_L001_R2_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/7/2/02-2222_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/8/2/02-2222_S1_L001_R2_001.fastq\"}}}]}},{\"name\":\"03-3333\",\"_links\":{\"self\":{\"href\":\"http://sample/path/will/go/here\"}},\"_embedded\":{\"sample_files\":[{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/5/2/03-3333_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/6/2/03-3333_S1_L001_R2_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/11/2/03-3333_S1_L001_R1_001.fastq\"}}},{\"_links\":{\"self\":{\"href\":\"file:///tmp/sequence-files/12/2/03-3333_S1_L001_R2_001.fastq\"}}}]}}]}}", "__get_data_table_entry__": "<function get_data_table_entry at 0x7fd16823d7d0>", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpe5DJvI"}, "output_data": [{"extra_files_path": "/home/jthiessen/galaxy-dist/database/job_working_directory/000/836/dataset_836_files", "ext": "auto", "out_data_name": "output", "hda_id": 685, "file_name": "/home/jthiessen/galaxy-dist/database/files/000/dataset_836.dat", "dataset_id": 836}], "job_config": {"GALAXY_ROOT_DIR": "/home/jthiessen/galaxy-dist", "TOOL_PROVIDED_JOB_METADATA_FILE": "galaxy.json", "GALAXY_DATATYPES_CONF_FILE": "/home/jthiessen/galaxy-dist/database/tmp/tmpe5DJvI"}}
\ No newline at end of file
......@@ -14,7 +14,6 @@ from mock import Mock
import mock
class TestIridaImport:
""" TestIridaImport performs unit tests on IridaImport."""
......@@ -27,24 +26,14 @@ class TestIridaImport:
@pytest.fixture(scope="class")
def setup_json(self):
"""Create a JSON string representative of the Galaxy dataset file"""
logging.debug("Making a test json file")
test_json_dict = {
"param_dict": {
"library_name": "boblib",
"userId": "1",
"dbkey": "?",
"sample1_name": "thisissample1'sname",
"__user__": "galaxy.model:User",
"__app__": "galaxy.app:UniverseApplication",
"sample1_path": "http://localhost/some_IRIDA_API_path/Projects/1/Samples/1",
"__user_name__": "jthiessen",
"runtool_btn": "Execute",
"__user_id__": "1",
"sample1_file2_path": "file://localhost/home/jthiessen/lib_imp_dir/test/test.fastq",
"__root_dir__": "/home/jthiessen/galaxy-dist",
"sample1_file1_path": "http://www.dgfdg.com/sample1file1"}}
return json.dumps(test_json_dict)
"""Create a json string from a text file"""
logging.debug("Opening a test json string")
test_path = 'tests/unit/data/test.dat'
# py.test messes up the paths to modules and packages
# I haven't found a way to get around it without hardcoding paths
test_json_file = open(test_path)
test_json = test_json_file.read()
return test_json
@pytest.fixture(scope="function")
def imp(self):
......@@ -63,11 +52,13 @@ class TestIridaImport:
The tested method is expected to change soon.
"""
param_dict = json.loads(setup_json)['param_dict']
samples = imp.get_samples(param_dict)
assert isinstance(samples, list), 'A list must be returned'
json_params = json.loads(param_dict['json_params'])
samples = imp.get_samples(json_params)
assert isinstance(
samples, list), 'A list must be returned'
for sample in samples:
assert isinstance(sample, Sample), 'The list must contain samples'
assert len(samples) == 1, 'Number of samples is incorrect'
assert len(samples) == 3, 'Number of samples is incorrect'
def test_get_first_or_make_lib_empty(self, imp):
"""Test library creation if there are no preexisting libraries"""
......@@ -111,12 +102,15 @@ class TestIridaImport:
lib_to_make.deleted = is_deleted
return lib_to_make
def test_create_folder_if_nec_base(self, imp):
def test_create_folder_if_nec_base(
self,
imp):
"""Create a folder, as if its base folder exists"""
base_folder = mock.create_autospec(Folder)
base_folder = mock.create_autospec(
Folder)
folder_name = 'sample1'
folder_path = '/illumina_reads/'+folder_name
folder_path = '/illumina_reads/' + folder_name
folder = mock.create_autospec(Folder)
folder.name = folder_path
......@@ -128,10 +122,13 @@ class TestIridaImport:
imp.library.get_folder = Mock(return_value=base_folder)
imp.library.create_folder = Mock(return_value=folder)
# IridaImport.exists_in_lib(...) is tested elsewhere
imp.exists_in_lib = Mock(return_value=False)
# IridaImport.exists_in_lib(...) is tested
# elsewhere
imp.exists_in_lib = Mock(
return_value=False)
made_folder = imp.create_folder_if_nec(folder_path)
made_folder = imp.create_folder_if_nec(
folder_path)
assert (imp.reg_gi.libraries.get_folders.call_count is 1,
'Base folders should only be looked for once')
......@@ -140,7 +137,8 @@ class TestIridaImport:
# Can't add assertion message here--either inside the method, or
# by wrapping with an assert
imp.library.get_folder.assert_called_with(picked_f_id)
imp.library.get_folder.assert_called_with(
picked_f_id)
assert (imp.library.create_folder.call_count is 1,
'Only 1 folder must be created')
......@@ -152,12 +150,11 @@ class TestIridaImport:
"""Raise an exception if the folder's base folder doesn't exist"""
folder_name = 'sample1'
folder_path = '/illumina_reads/'+folder_name
folder_path = '/illumina_reads/' + folder_name
folder = mock.create_autospec(Folder)
folder.name = folder_path
imp.reg_gi.libraries.get_folders = Mock(
return_value=[])
imp.reg_gi.libraries.get_folders = Mock(return_value=[])
imp.library = self.make_lib('wolib', False)
imp.library.id = 123
......@@ -168,9 +165,12 @@ class TestIridaImport:
""" Test if a folder can be found in a library among chaff items """
imp.library = self.make_lib('wolib', False)
items = []
items.append(self.make_content_info('file', 'name', 'sally.fastq'))
items.append(self.make_content_info('folder', 'name', 'bob.fasta'))
items.append(self.make_content_info('file', 'name', 'bob.fasta'))
items.append(
self.make_content_info('file', 'name', 'sally.fastq'))
items.append(
self.make_content_info('folder', 'name', 'bob.fasta'))
items.append(
self.make_content_info('file', 'name', 'bob.fasta'))
imp.library.content_infos = items
exists = imp.exists_in_lib('file', 'name', 'bob.fasta')
assert exists, 'file must exist in library'
......@@ -199,9 +199,13 @@ class TestIridaImport:
imp.exists_in_lib = Mock(return_value=False)
imp.upload_or_link = Mock(return_value=file_list)
sampleFile1 = SampleFile("/imaginary/path/file1.fasta")
sampleFile2 = SampleFile("/imaginary/path/file2.fasta")
sample = Sample("bobname", "/imaginary/path/Samples/1")
sampleFile1 = SampleFile(
"/imaginary/path/file1.fasta")
sampleFile2 = SampleFile(
"/imaginary/path/file2.fasta")
sample = Sample(
"bobname",
"/imaginary/path/Samples/1")
sample.sample_files.append(sampleFile1)
sample.sample_files.append(sampleFile2)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment