Skip to content

Commit 0f82aac

Browse files
stymystymy
committed
Merge pull request #1 from nipy/master
pull from nipype
2 parents 7fb9eca + 504858e commit 0f82aac

File tree

9 files changed

+398
-94
lines changed

9 files changed

+398
-94
lines changed

‎.travis.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ python:
44
- 2.7
55
# Setup anaconda
66
before_install:
7-
- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then wget http://repo.continuum.io/miniconda/Miniconda-2.2.2-Linux-x86_64.sh -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-2.2.2-Linux-x86_64.sh -O miniconda.sh; fi
7+
- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then wget http://repo.continuum.io/miniconda/Miniconda-3.0.0-Linux-x86_64.sh -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-3.0.0-Linux-x86_64.sh -O miniconda.sh; fi
88
- chmod +x miniconda.sh
99
- ./miniconda.sh -b
10-
- export PATH=/home/travis/anaconda/bin:$PATH
10+
- export PATH=/home/travis/miniconda/bin:$PATH
1111
# The next couple lines fix a crash with multiprocessing on Travis
1212
- sudo rm -rf /dev/shm
1313
- sudo ln -s /run/shm /dev/shm
@@ -18,8 +18,10 @@ before_install:
1818

1919
# Install packages
2020
install:
21+
- conda update --yes conda
2122
- conda create -n testenv --yes pip python=$TRAVIS_PYTHON_VERSION
2223
- source activate testenv
24+
- if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then pip install ordereddict; fi
2325
- conda install --yes numpy scipy nose traits networkx dateutil
2426
- pip install nibabel --use-mirrors
2527
- pip install python-coveralls --use-mirrors

‎Vagrantfile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,13 @@ $script = <<SCRIPT
2020
# qconf -aattr queue slots "2, [neuro=3]" main.q
2121
2222
# install anaconda
23-
wget http://repo.continuum.io/miniconda/Miniconda-2.2.2-Linux-x86_64.sh -O miniconda.sh
23+
wget http://repo.continuum.io/miniconda/Miniconda-3.0.0-Linux-x86_64.sh -O miniconda.sh
2424
chmod +x miniconda.sh
2525
./miniconda.sh -b
26-
echo "export PATH=$HOME/anaconda/bin:\\$PATH" >> .bashrc
26+
echo "export PATH=$HOME/miniconda/bin:\\$PATH" >> .bashrc
2727
2828
# install nipype dependencies
29+
$HOME/anaconda/bin/conda update --yes conda
2930
$HOME/anaconda/bin/conda install --yes pip numpy scipy nose traits networkx
3031
$HOME/anaconda/bin/conda install --yes dateutil ipython-notebook matplotlib
3132
$HOME/anaconda/bin/pip install nibabel --use-mirrors

‎nipype/interfaces/base.py

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,12 +1109,26 @@ def run_command(runtime, output=None, timeout=0.01):
11091109
The returned runtime contains a merged stdout+stderr log with timestamps
11101110
"""
11111111
PIPE = subprocess.PIPE
1112-
proc = subprocess.Popen(runtime.cmdline,
1113-
stdout=PIPE,
1114-
stderr=PIPE,
1115-
shell=True,
1116-
cwd=runtime.cwd,
1117-
env=runtime.environ)
1112+
1113+
if output == 'file':
1114+
errfile = os.path.join(runtime.cwd, 'stderr.nipype')
1115+
outfile = os.path.join(runtime.cwd, 'stdout.nipype')
1116+
stderr = open(errfile, 'wt')
1117+
stdout = open(outfile, 'wt')
1118+
1119+
proc = subprocess.Popen(runtime.cmdline,
1120+
stdout=stdout,
1121+
stderr=stderr,
1122+
shell=True,
1123+
cwd=runtime.cwd,
1124+
env=runtime.environ)
1125+
else:
1126+
proc = subprocess.Popen(runtime.cmdline,
1127+
stdout=PIPE,
1128+
stderr=PIPE,
1129+
shell=True,
1130+
cwd=runtime.cwd,
1131+
env=runtime.environ)
11181132
result = {}
11191133
errfile = os.path.join(runtime.cwd, 'stderr.nipype')
11201134
outfile = os.path.join(runtime.cwd, 'stdout.nipype')
@@ -1154,14 +1168,6 @@ def _process(drain=0):
11541168
result['stderr'] = stderr.split('\n')
11551169
result['merged'] = ''
11561170
if output == 'file':
1157-
stderr = open(errfile, 'wt')
1158-
stdout = open(outfile, 'wt')
1159-
proc = subprocess.Popen(runtime.cmdline,
1160-
stdout=stdout,
1161-
stderr=stderr,
1162-
shell=True,
1163-
cwd=runtime.cwd,
1164-
env=runtime.environ)
11651171
ret_code = proc.wait()
11661172
stderr.flush()
11671173
stdout.flush()

‎nipype/pipeline/engine.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,15 @@ def __repr__(self):
205205

206206
def save(self, filename=None):
207207
if filename is None:
208-
filename = 'temp.npz'
209-
np.savez(filename, object=self)
208+
filename = 'temp.pklz'
209+
savepkl(filename, self)
210210

211211
def load(self, filename):
212-
return np.load(filename)
212+
if '.npz' in filename:
213+
DeprecationWarning(('npz files will be deprecated in the next '
214+
'release. you can use numpy to open them.'))
215+
return np.load(filename)
216+
return loadpkl(filename)
213217

214218

215219
class Workflow(WorkflowBase):

‎nipype/pipeline/plugins/base.py

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from copy import deepcopy
77
from glob import glob
88
import os
9+
import pickle
910
import pwd
1011
import shutil
1112
from socket import gethostname
@@ -55,7 +56,7 @@ def report_crash(node, traceback=None, hostname=None):
5556
exc_traceback)
5657
timeofcrash = strftime('%Y%m%d-%H%M%S')
5758
login_name = pwd.getpwuid(os.geteuid())[0]
58-
crashfile = 'crash-%s-%s-%s.npz' % (timeofcrash,
59+
crashfile = 'crash-%s-%s-%s.pklz' % (timeofcrash,
5960
login_name,
6061
name)
6162
crashdir = node.config['execution']['crashdump_dir']
@@ -66,7 +67,8 @@ def report_crash(node, traceback=None, hostname=None):
6667
crashfile = os.path.join(crashdir, crashfile)
6768
logger.info('Saving crash info to %s' % crashfile)
6869
logger.info(''.join(traceback))
69-
np.savez(crashfile, node=node, traceback=traceback)
70+
savepkl(crashfile, dict(node=node, traceback=traceback))
71+
#np.savez(crashfile, node=node, traceback=traceback)
7072
return crashfile
7173

7274

@@ -303,8 +305,8 @@ def _submit_mapnode(self, jobid):
303305
self.mapnodesubids[self.depidx.shape[0] + i] = jobid
304306
self.procs.extend(mapnodesubids)
305307
self.depidx = ssp.vstack((self.depidx,
306-
ssp.lil_matrix(np.zeros((numnodes,
307-
self.depidx.shape[1])))),
308+
ssp.lil_matrix(np.zeros(
309+
(numnodes, self.depidx.shape[1])))),
308310
'lil')
309311
self.depidx = ssp.hstack((self.depidx,
310312
ssp.lil_matrix(
@@ -349,16 +351,19 @@ def _send_procs_to_workers(self, updatehash=False, slots=None, graph=None):
349351
if self._status_callback:
350352
self._status_callback(self.procs[jobid], 'start')
351353
continue_with_submission = True
352-
if str2bool(self.procs[jobid].config['execution']['local_hash_check']):
354+
if str2bool(self.procs[jobid].config['execution']
355+
['local_hash_check']):
353356
logger.debug('checking hash locally')
354357
try:
355358
hash_exists, _, _, _ = self.procs[
356359
jobid].hash_exists()
357360
logger.debug('Hash exists %s' % str(hash_exists))
358361
if (hash_exists and
359-
(self.procs[jobid].overwrite == False or
360-
(self.procs[jobid].overwrite == None and
361-
not self.procs[jobid]._interface.always_run))):
362+
(self.procs[jobid].overwrite == False or
363+
(self.procs[jobid].overwrite == None and
364+
not self.procs[jobid]._interface.always_run)
365+
)
366+
):
362367
continue_with_submission = False
363368
self._task_finished_cb(jobid)
364369
self._remove_node_dirs()
@@ -436,7 +441,8 @@ def _remove_node_dirs(self):
436441
"""Removes directories whose outputs have already been used up
437442
"""
438443
if str2bool(self._config['execution']['remove_node_directories']):
439-
for idx in np.nonzero((self.refidx.sum(axis=1) == 0).__array__())[0]:
444+
for idx in np.nonzero(
445+
(self.refidx.sum(axis=1) == 0).__array__())[0]:
440446
if idx in self.mapnodesubids:
441447
continue
442448
if self.proc_done[idx] and (not self.proc_pending[idx]):
@@ -506,9 +512,13 @@ def _get_result(self, taskid):
506512
'traceback': None}
507513
results_file = None
508514
try:
509-
raise IOError(('Job (%s) finished or terminated, but results file '
510-
'does not exist. Batch dir contains crashdump '
511-
'file if node raised an exception' % node_dir))
515+
error_message = ('Job id ({0}) finished or terminated, but '
516+
'results file does not exist after ({1}) '
517+
'seconds. Batch dir contains crashdump file '
518+
'if node raised an exception.\n'
519+
'Node working directory: ({2}) '.format(
520+
taskid,timeout,node_dir) )
521+
raise IOError(error_message)
512522
except IOError, e:
513523
result_data['traceback'] = format_exc()
514524
else:
@@ -582,13 +592,17 @@ def _get_args(self, node, keywords):
582592
value = getattr(self, "_" + keyword)
583593
if keyword == "template" and os.path.isfile(value):
584594
value = open(value).read()
585-
if hasattr(node, "plugin_args") and isinstance(node.plugin_args, dict) and keyword in node.plugin_args:
586-
if keyword == "template" and os.path.isfile(node.plugin_args[keyword]):
595+
if (hasattr(node, "plugin_args") and
596+
isinstance(node.plugin_args, dict) and
597+
keyword in node.plugin_args):
598+
if (keyword == "template" and
599+
os.path.isfile(node.plugin_args[keyword])):
587600
tmp_value = open(node.plugin_args[keyword]).read()
588601
else:
589602
tmp_value = node.plugin_args[keyword]
590603

591-
if 'overwrite' in node.plugin_args and node.plugin_args['overwrite']:
604+
if ('overwrite' in node.plugin_args and
605+
node.plugin_args['overwrite']):
592606
value = tmp_value
593607
else:
594608
value += tmp_value

0 commit comments

Comments
 (0)