Skip to content

Commit e3ad668

Browse files
pintohutchpintohutch
committed
Merge pull request #1 from FCP-INDI/master
Pull in multiproc
2 parents 0503c23 + a8f8006 commit e3ad668

File tree

16 files changed

+755
-72
lines changed

16 files changed

+755
-72
lines changed

‎CHANGES

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
Next Release
22
============
3+
* API: Default model level for the bedpostx workflow has been set to "2" following FSL 5.0.9 lead
34
* ENH: New interfaces for interacting with AWS S3: S3DataSink and S3DataGrabber (https://github.com/nipy/nipype/pull/1201)
45

56
Release 0.11.0 (September 15, 2015)

‎circle.yml

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,52 +5,77 @@ dependencies:
55
- "~/examples/feeds"
66
- "~/mcr"
77
- "~/spm12"
8+
- "~/fsl"
89
- "~/examples/fsl_course_data"
910
override:
1011
- pip install --upgrade pip
1112
- pip install -e .
1213
- pip install matplotlib sphinx ipython boto
1314
- gem install fakes3
15+
- if [[ ! -d ~/fsl ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.9-centos6_64.tar.gz"; tar zxvf fsl-5.0.9-centos6_64.tar.gz; mv fsl ~/fsl; fi
1416
- if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2"; tar jxvf nipype-tutorial.tar.bz2; mkdir ~/examples; mv nipype-tutorial/* ~/examples/; fi
1517
# we download this manually because CircleCI does not cache apt
1618
- if [[ ! -d ~/examples/feeds ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.8-feeds.tar.gz"; tar zxvf fsl-5.0.8-feeds.tar.gz; mv feeds ~/examples/; fi
1719
- if [[ ! -d ~/examples/fsl_course_data ]]; then wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt1.tar.gz" ; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/fdt2.tar.gz"; wget -c "http://fsl.fmrib.ox.ac.uk/fslcourse/tbss.tar.gz"; mkdir ~/examples/fsl_course_data; tar zxvf fdt1.tar.gz -C ~/examples/fsl_course_data; tar zxvf fdt2.tar.gz -C ~/examples/fsl_course_data; tar zxvf tbss.tar.gz -C ~/examples/fsl_course_data; fi
18-
- wget -O- http://neuro.debian.net/lists/trusty.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
19-
- sudo apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu:80 0xA5D32F012649A5A9
20-
- sudo apt-get update -y; sudo apt-get install -y fsl-core fsl-atlases
2120
- bash ~/nipype/tools/install_spm_mcr.sh
2221
- mkdir -p ~/.nipype && echo "[logging]" > ~/.nipype/nipype.cfg && echo "workflow_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "interface_level = DEBUG" >> ~/.nipype/nipype.cfg && echo "filemanip_level = DEBUG" >> ~/.nipype/nipype.cfg
2322
test:
2423
override:
25-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && nosetests --with-doctest --logging-level=DEBUG --verbosity=3:
24+
- nosetests --with-doctest --logging-level=DEBUG --verbosity=3:
2625
environment:
2726
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
2827
FORCE_SPMMCR: 1
2928
FSL_COURSE_DATA: "$HOME/examples/fsl_course_data"
29+
FSLDIR: "$HOME/fsl/"
30+
PATH: "$HOME/fsl/bin:$PATH"
31+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
32+
FSLOUTPUTTYPE: "NIFTI_GZ"
3033
timeout: 2600
3134
- set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt
3235
- cat ~/log.txt && if grep -q "ERROR" ~/log.txt; then false; else true; fi
33-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
36+
- python ~/nipype/tools/run_examples.py test_spm Linear workflow3d workflow4d:
3437
pwd: ../examples
3538
environment:
3639
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
3740
FORCE_SPMMCR: 1
41+
FSLDIR: "$HOME/fsl/"
42+
PATH: "$HOME/fsl/bin:$PATH"
43+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
44+
FSLOUTPUTTYPE: "NIFTI_GZ"
3845
timeout: 1600
39-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
46+
- python ~/nipype/tools/run_examples.py fmri_fsl_feeds Linear l1pipeline:
4047
pwd: ../examples
41-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
48+
environment:
49+
FSLDIR: "$HOME/fsl/"
50+
PATH: "$HOME/fsl/bin:$PATH"
51+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
52+
FSLOUTPUTTYPE: "NIFTI_GZ"
53+
- python ~/nipype/tools/run_examples.py fmri_spm_dartel Linear level1 l2pipeline:
4254
pwd: ../examples
4355
environment:
4456
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
4557
FORCE_SPMMCR: 1
58+
FSLDIR: "$HOME/fsl/"
59+
PATH: "$HOME/fsl/bin:$PATH"
60+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
61+
FSLOUTPUTTYPE: "NIFTI_GZ"
4662
timeout: 1600
47-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
63+
- python ~/nipype/tools/run_examples.py fmri_fsl_reuse Linear level1_workflow:
4864
pwd: ../examples
49-
- . /usr/share/fsl/5.0/etc/fslconf/fsl.sh && python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
65+
environment:
66+
FSLDIR: "$HOME/fsl/"
67+
PATH: "$HOME/fsl/bin:$PATH"
68+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
69+
FSLOUTPUTTYPE: "NIFTI_GZ"
70+
- python ~/nipype/tools/run_examples.py fmri_spm_nested Linear level1 l2pipeline:
5071
pwd: ../examples
5172
environment:
5273
SPMMCRCMD: "$HOME/spm12/run_spm12.sh $HOME/mcr/v85/ script"
5374
FORCE_SPMMCR: 1
75+
FSLDIR: "$HOME/fsl/"
76+
PATH: "$HOME/fsl/bin:$PATH"
77+
LD_LIBRARY_PATH: "$HOME/fsl/lib"
78+
FSLOUTPUTTYPE: "NIFTI_GZ"
5479

5580
general:
5681
artifacts:

‎nipype/interfaces/base.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,8 @@ def __init__(self, **inputs):
750750
raise Exception('No input_spec in class: %s' %
751751
self.__class__.__name__)
752752
self.inputs = self.input_spec(**inputs)
753+
self.memory = 1
754+
self.num_threads = 1
753755

754756
@classmethod
755757
def help(cls, returnhelp=False):

‎nipype/interfaces/fsl/dti.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ class FSLXCommandInputSpec(FSLCommandInputSpec):
114114
n_fibres = traits.Range(
115115
usedefault=True, low=1, default=2, argstr='--nfibres=%d',
116116
desc=('Maximum number of fibres to fit in each voxel'), mandatory=True)
117-
model = traits.Enum(1, 2, argstr='--model=%d',
117+
model = traits.Enum(1, 2, 3, argstr='--model=%d',
118118
desc=('use monoexponential (1, default, required for '
119119
'single-shell) or multiexponential (2, multi-'
120120
'shell) model'))
@@ -246,7 +246,7 @@ class BEDPOSTX5InputSpec(FSLXCommandInputSpec):
246246
n_fibres = traits.Range(
247247
usedefault=True, low=1, default=2, argstr='-n %d',
248248
desc=('Maximum number of fibres to fit in each voxel'), mandatory=True)
249-
model = traits.Enum(1, 2, argstr='-model %d',
249+
model = traits.Enum(1, 2, 3, argstr='-model %d',
250250
desc=('use monoexponential (1, default, required for '
251251
'single-shell) or multiexponential (2, multi-'
252252
'shell) model'))

‎nipype/interfaces/io.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ def _add_output_traits(self, base):
134134
# Class to track percentage of S3 file upload
135135
class ProgressPercentage(object):
136136
'''
137-
Call-able class instsance (via __call__ method) that displays
137+
Callable class instsance (via __call__ method) that displays
138138
upload percentage of a file to S3
139139
'''
140140

@@ -367,7 +367,7 @@ def _substitute(self, pathstr):
367367
def _check_s3_base_dir(self):
368368
'''
369369
Method to see if the datasink's base directory specifies an
370-
S3 bucket path; it it does, it parses the path for the bucket
370+
S3 bucket path; if it does, it parses the path for the bucket
371371
name in the form 's3://bucket_name/...' and adds a bucket
372372
attribute to the data sink instance, i.e. self.bucket
373373
@@ -451,7 +451,7 @@ def _return_aws_keys(self, creds_path):
451451
# Fetch bucket object
452452
def _fetch_bucket(self, bucket_name):
453453
'''
454-
Method to a return a bucket object which can be used to interact
454+
Method to return a bucket object which can be used to interact
455455
with an AWS S3 bucket using credentials found in a local file.
456456
457457
Parameters

‎nipype/pipeline/plugins/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,13 @@
99
from .condor import CondorPlugin
1010
from .dagman import CondorDAGManPlugin
1111
from .multiproc import MultiProcPlugin
12+
from .multiproc import ResourceMultiProcPlugin
1213
from .ipython import IPythonPlugin
1314
from .somaflow import SomaFlowPlugin
1415
from .pbsgraph import PBSGraphPlugin
1516
from .sgegraph import SGEGraphPlugin
1617
from .lsf import LSFPlugin
1718
from .slurm import SLURMPlugin
1819
from .slurmgraph import SLURMGraphPlugin
20+
21+
from .callback_log import log_nodes_cb

‎nipype/pipeline/plugins/base.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import numpy as np
1818
import scipy.sparse as ssp
1919

20-
2120
from ..utils import (nx, dfs_preorder, topological_sort)
2221
from ..engine import (MapNode, str2bool)
2322

@@ -261,10 +260,15 @@ def run(self, graph, config, updatehash=False):
261260
graph=graph)
262261
else:
263262
logger.debug('Not submitting')
264-
sleep(float(self._config['execution']['poll_sleep_duration']))
263+
self._wait()
265264
self._remove_node_dirs()
266265
report_nodes_not_run(notrun)
267266

267+
268+
269+
def _wait(self):
270+
sleep(float(self._config['execution']['poll_sleep_duration']))
271+
268272
def _get_result(self, taskid):
269273
raise NotImplementedError
270274

‎nipype/pipeline/plugins/callback_log.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import datetime
2+
import logging
3+
4+
def log_nodes_cb(node, status):
5+
print 'status', status
6+
logger = logging.getLogger('callback')
7+
if status == 'start':
8+
message = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' +\
9+
node._id + '"' + ',"start":' + '"' +str(datetime.datetime.now()) +\
10+
'"' + ',"memory":' + str(node._interface.memory) + ',"num_threads":' \
11+
+ str(node._interface.num_threads) + '}'
12+
13+
logger.debug(message)
14+
15+
elif status == 'end':
16+
message = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' + \
17+
node._id + '"' + ',"finish":' + '"' + str(datetime.datetime.now()) +\
18+
'"' + ',"memory":' + str(node._interface.memory) + ',"num_threads":' \
19+
+ str(node._interface.num_threads) + '}'
20+
21+
logger.debug(message)
22+
23+
else:
24+
message = '{"name":' + '"' + node.name + '"' + ',"id":' + '"' + \
25+
node._id + '"' + ',"finish":' + '"' + str(datetime.datetime.now()) +\
26+
'"' + ',"memory":' + str(node._interface.memory) + ',"num_threads":' \
27+
+ str(node._interface.num_threads) + ',"error":"True"}'
28+
29+
logger.debug(message)

0 commit comments

Comments
 (0)