From 645e3f2cbb5f10f80cd3de7082e8380551fd4b2d Mon Sep 17 00:00:00 2001 From: DOHMATOB Elvis Date: Wed, 14 Feb 2018 16:39:19 +0100 Subject: [PATCH 1/5] codecov and other ci stuff --- .travis.yml | 14 +++++--------- continuous_integration/after_success.sh | 12 ++++++++++++ continuous_integration/test_script.sh | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) create mode 100755 continuous_integration/after_success.sh create mode 100755 continuous_integration/test_script.sh diff --git a/.travis.yml b/.travis.yml index 9b2145fc..f0bea77b 100755 --- a/.travis.yml +++ b/.travis.yml @@ -39,12 +39,8 @@ virtualenv: install: source continuous_integration/install.sh -script: - - make clean - - make test-code - -after_success: - - | - if [[ "$COVERAGE" == "true" ]]; then - coveralls || echo "coverage upload failed" - fi +before_script: make clean + +script: source continuous_integration/test_script.sh + +after_success: source continuous_integration/after_success.sh diff --git a/continuous_integration/after_success.sh b/continuous_integration/after_success.sh new file mode 100755 index 00000000..a7475f62 --- /dev/null +++ b/continuous_integration/after_success.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +# Ignore codecov failures because we don't want travis to report a failure +# in the github UI just because the coverage report failed to be published. +# codecov needs to be run from the git checkout +# so we need to copy the coverage results from TEST_RUN_FOLDER +if [[ "$SKIP_TESTS" != "true" && "$COVERAGE" == "true" ]]; then + cp "$TEST_RUN_FOLDER/.coverage" . + codecov || echo "Codecov upload failed" +fi diff --git a/continuous_integration/test_script.sh b/continuous_integration/test_script.sh new file mode 100755 index 00000000..1dfa2578 --- /dev/null +++ b/continuous_integration/test_script.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +if [[ -n "$FLAKE8_VERSION" ]]; then + source continuous_integration/flake8_diff.sh +fi + +if [[ "$SKIP_TESTS" != "true" ]]; then + python continuous_integration/show-python-packages-versions.py + # Copy setup.cfg to TEST_RUN_FOLDER where we are going to run the tests from + # Mainly for nose config settings + cp setup.cfg "$TEST_RUN_FOLDER" + # We want to back out of the current working directory to make + # sure we are using nilearn installed in site-packages rather + # than the one from the current working directory + # Parentheses (run in a subshell) are used to leave + # the current directory unchanged + (cd "$TEST_RUN_FOLDER" && make -f $OLDPWD/Makefile test-code) + test "$MATPLOTLIB_VERSION" == "" || make test-doc +fi From 859bf1ddae4212eb8aca649435712f3a3c91109e Mon Sep 17 00:00:00 2001 From: DOHMATOB Elvis Date: Thu, 15 Feb 2018 10:55:28 +0100 Subject: [PATCH 2/5] fixing some issues with spm install stuff --- circle.yml | 1 + continuous_integration/{install_spm.sh => install_spm8.sh} | 0 continuous_integration/setup_spm.sh | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) rename continuous_integration/{install_spm.sh => install_spm8.sh} (100%) diff --git a/circle.yml b/circle.yml index 15af788b..c94f01cf 100644 --- a/circle.yml +++ b/circle.yml @@ -24,6 +24,7 @@ dependencies: SCIKIT_LEARN_VERSION: "*" MATPLOTLIB_VERSION: "*" NIPYPE_VERSION: "0.13.0" + SPM_VERSION: "8" test: override: diff --git a/continuous_integration/install_spm.sh b/continuous_integration/install_spm8.sh similarity index 100% rename from continuous_integration/install_spm.sh rename to continuous_integration/install_spm8.sh diff --git a/continuous_integration/setup_spm.sh b/continuous_integration/setup_spm.sh index a69d88ba..10bd597c 100644 --- a/continuous_integration/setup_spm.sh +++ b/continuous_integration/setup_spm.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -SPM_INSTALL_SCRIPT=continuous_integration/install_spm12.sh +SPM_INSTALL_SCRIPT=continuous_integration/install_spm${SPM_VERSION}.sh echo "" echo "SPM_INSTALL_SCRIPT: $SPM_INSTALL_SCRIPT" sudo bash $SPM_INSTALL_SCRIPT From 4680e01f4b5f4e501b90971ab931eac9f720d643 Mon Sep 17 00:00:00 2001 From: DOHMATOB Elvis Date: Thu, 15 Feb 2018 11:10:34 +0100 Subject: [PATCH 3/5] HOTFIX: default SPM=8 in circle-ci (fixing spm12 issues) --- circle.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index c94f01cf..05a64e94 100644 --- a/circle.yml +++ b/circle.yml @@ -1,7 +1,9 @@ machine: environment: - SPM_DIR: /home/ubuntu/opt/spm12/spm12_mcr/spm12 - SPM_MCR: /home/ubuntu/opt/spm12/spm12.sh + SPM_DIR: /home/ubuntu/opt/spm8/spm8/spm8_mcr/spm8 + SPM_MCR: /home/ubuntu/opt/spm8/spm8.sh + # SPM_DIR: /home/ubuntu/opt/spm12/spm12_mcr/spm12 + # SPM_MCR: /home/ubuntu/opt/spm12/spm12.sh dependencies: cache_directories: @@ -24,14 +26,13 @@ dependencies: SCIKIT_LEARN_VERSION: "*" MATPLOTLIB_VERSION: "*" NIPYPE_VERSION: "0.13.0" - SPM_VERSION: "8" test: override: - make clean - make test-code - cd examples/easy_start && python nipype_preproc_spm_auditory.py - - cd examples/pipelining && python nipype_preproc_spm_multimodal_faces.py + # - cd examples/pipelining && python nipype_preproc_spm_multimodal_faces.py - cd examples/pipelining && python nistats_glm_fsl_feeds_fmri.py - sh continuous_integration/clean_output.sh From 0ac38a1bb89623b4a50c870a7c5a7d05c9d1d0a0 Mon Sep 17 00:00:00 2001 From: DOHMATOB Elvis Date: Thu, 15 Feb 2018 11:11:15 +0100 Subject: [PATCH 4/5] IMPR: forced ignore_exception=False in nipype nodes --- .travis.yml | 4 +++ Makefile | 24 +++++++++++++++--- circle.yml | 13 ++++++++++ .../show-python-packages-versions.py | 25 +++++++++++++++++++ pypreprocess/nipype_preproc_spm_utils.py | 22 ++++++++-------- 5 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 continuous_integration/show-python-packages-versions.py diff --git a/.travis.yml b/.travis.yml index f0bea77b..fc97d873 100755 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ sudo: required language: python +env: + global: + - TEST_RUN_FOLDER="/tmp" # folder where the tests are run from + matrix: # Do not wait for the allowed_failures entry to finish before # setting the status diff --git a/Makefile b/Makefile index aa63d7e7..b6beba7f 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,15 @@ PYTHON ?= python CYTHON ?= cython NOSETESTS ?= nosetests +NOSETESTS_OPTIONS := $(shell pip list | grep nose-timer > /dev/null && \ + echo '--with-timer --timer-top-n 50') CTAGS ?= ctags -all: clean test +all: clean test doc-noplot clean-pyc: find . -name "*.pyc" | xargs rm -f + find . -name "__pycache__" | xargs rm -rf clean-so: find . -name "*.so" | xargs rm -f @@ -29,14 +32,17 @@ inplace: $(PYTHON) setup.py build_ext -i test-code: - $(NOSETESTS) -s pypreprocess + $(NOSETESTS) -s pypreprocess $(NOSETESTS_OPTIONS) +test-doc: + $(NOSETESTS) -s --with-doctest --doctest-tests --doctest-extension=rst \ + --doctest-extension=inc --doctest-fixtures=_fixture `find doc/ -name '*.rst'` test-coverage: rm -rf coverage .coverage $(NOSETESTS) -s --with-coverage --cover-html --cover-html-dir=coverage \ --cover-package=pypreprocess pypreprocess -test: test-code +test: test-code test-doc trailing-spaces: find . -name "*.py" | xargs perl -pi -e 's/[ \t]*$$//' @@ -48,3 +54,15 @@ ctags: # make tags for symbol based navigation in emacs and vim # Install with: sudo apt-get install exuberant-ctags $(CTAGS) -R * + +.PHONY : doc-plot +doc-plot: + make -C doc html + +.PHONY : doc +doc: + make -C doc html-noplot + +.PHONY : pdf +pdf: + make -C doc pdf diff --git a/circle.yml b/circle.yml index 05a64e94..93c02aae 100644 --- a/circle.yml +++ b/circle.yml @@ -1,5 +1,6 @@ machine: environment: + SPM_VERSION: 8 SPM_DIR: /home/ubuntu/opt/spm8/spm8/spm8_mcr/spm8 SPM_MCR: /home/ubuntu/opt/spm8/spm8.sh # SPM_DIR: /home/ubuntu/opt/spm12/spm12_mcr/spm12 @@ -7,6 +8,7 @@ machine: dependencies: cache_directories: + - "~/opt/spm8" - "~/opt/spm12" - "~/nilearn_data" @@ -14,8 +16,19 @@ dependencies: # Installing dependencies for SPM and MCR - sudo apt-get update - sudo apt-get install libxp6 libxpm4 libxmu6 libxt6 bc + # Installing SPM + - source continuous_integration/setup_spm.sh + # Installing Pypreprocess dependencies + - pip install --upgrade pip + - pip install nose coverage configparser override: + # Installing pypreprocess + - pip install -e . + # Fetching Auditory and Multimodal datasets in order to be cached in the future + - python -c "from pypreprocess import datasets; datasets.fetch_spm_auditory(); datasets.fetch_spm_multimodal_fmri(); datasets.fetch_fsl_feeds()" + # Caching terminates here. Outputs from test won't be saved. + # Moving to nilearn directory before performing the installation. - source continuous_integration/install.sh: environment: diff --git a/continuous_integration/show-python-packages-versions.py b/continuous_integration/show-python-packages-versions.py new file mode 100644 index 00000000..1822dd17 --- /dev/null +++ b/continuous_integration/show-python-packages-versions.py @@ -0,0 +1,25 @@ +import sys + +DEPENDENCIES = ['numpy', 'scipy', 'sklearn', 'matplotlib', 'nibabel'] + + +def print_package_version(package_name, indent=' '): + try: + package = __import__(package_name) + version = getattr(package, '__version__', None) + package_file = getattr(package, '__file__', ) + provenance_info = '{0} from {1}'.format(version, package_file) + except ImportError: + provenance_info = 'not installed' + + print('{0}{1}: {2}'.format(indent, package_name, provenance_info)) + +if __name__ == '__main__': + print('=' * 120) + print('Python %s' % str(sys.version)) + print('from: %s\n' % sys.executable) + + print('Dependencies versions') + for package_name in DEPENDENCIES: + print_package_version(package_name) + print('=' * 120) diff --git a/pypreprocess/nipype_preproc_spm_utils.py b/pypreprocess/nipype_preproc_spm_utils.py index 7d7410dc..7849a88e 100644 --- a/pypreprocess/nipype_preproc_spm_utils.py +++ b/pypreprocess/nipype_preproc_spm_utils.py @@ -196,7 +196,7 @@ def _do_subject_slice_timing(subject_data, TR, TA=None, spm_dir=None, time_acquisition=TA, num_slices=nslices, ref_slice=ref_slice + 1, slice_order=list(slice_order + 1), # SPM - ignore_exception=True + ignore_exception=False ) if stc_result.outputs is None: subject_data.failed = True @@ -495,7 +495,7 @@ def _do_subject_coregister(subject_data, reslice=False, spm_dir=None, source=coreg_source, apply_to_files=apply_to_files, jobtype=jobtype, - ignore_exception=True + ignore_exception=False ) # failed node ? @@ -633,7 +633,7 @@ def _do_subject_segment(subject_data, output_modulated_tpms=True, spm_dir=None, wm_output_type=gm_output_type, csf_output_type=csf_output_type, tissue_prob_maps=[GM_TEMPLATE, WM_TEMPLATE, CSF_TEMPLATE], - ignore_exception=True + ignore_exception=False ) # failed node @@ -746,7 +746,7 @@ def _do_subject_normalize(subject_data, fwhm=0., anat_fwhm=0., caching=True, output_dir=subject_data.scratch) normalize_result = normalize( source=subject_data.anat, template=t1_template, - write_preserve=False, ignore_exception=True) + write_preserve=False, ignore_exception=False) parameter_file = normalize_result.outputs.normalization_parameters else: parameter_file = subject_data.nipype_results[ @@ -777,7 +777,7 @@ def _do_subject_normalize(subject_data, fwhm=0., anat_fwhm=0., caching=True, apply_to_files=apply_to_files, write_voxel_sizes=list(write_voxel_sizes), # write_bounding_box=[[-78, -112, -50], [78, 76, 85]], - write_interp=1, jobtype='write', ignore_exception=True) + write_interp=1, jobtype='write', ignore_exception=False) # failed node ? if normalize_result.outputs is None: @@ -808,7 +808,7 @@ def _do_subject_normalize(subject_data, fwhm=0., anat_fwhm=0., caching=True, write_wrap=[0, 0, 0], write_interp=1, jobtype='write', - ignore_exception=True + ignore_exception=False ) # failed node @@ -933,7 +933,7 @@ def _do_subject_smooth(subject_data, fwhm, anat_fwhm=None, spm_dir=None, in_files = [getattr(subject_data, x) for x in anat_like] smooth_result = smooth( - in_files=in_files, fwhm=width, ignore_exception=True) + in_files=in_files, fwhm=width, ignore_exception=False) # failed node ? subject_data.nipype_results['smooth'][brain_name] = smooth_result @@ -1025,7 +1025,7 @@ def _do_subject_dartelnorm2mni(subject_data, flowfield_files=subject_data.dartel_flow_fields, template_file=template_file, modulate=output_modulated_tpms, # don't modulate - fwhm=anat_fwhm, ignore_exception=True, **tricky_kwargs) + fwhm=anat_fwhm, ignore_exception=False, **tricky_kwargs) setattr(subject_data, "mw" + tissue, dartelnorm2mni_result.outputs.normalized_files) @@ -1034,7 +1034,7 @@ def _do_subject_dartelnorm2mni(subject_data, apply_to_files=subject_data.anat, flowfield_files=subject_data.dartel_flow_fields, template_file=template_file, - ignore_exception=True, + ignore_exception=False, modulate=output_modulated_tpms, fwhm=anat_fwhm, **tricky_kwargs @@ -1049,7 +1049,7 @@ def _do_subject_dartelnorm2mni(subject_data, createwarped_result = createwarped( image_files=subject_data.func, flowfield_files=subject_data.dartel_flow_fields, - ignore_exception=True + ignore_exception=False ) subject_data.func = createwarped_result.outputs.warped_files @@ -1408,7 +1408,7 @@ def _do_subjects_newsegment( newsegment_result = newsegment( channel_files=[subject_data.anat for subject_data in subjects], tissues=TISSUES, - ignore_exception=True) + ignore_exception=False) if newsegment_result.outputs is None: return else: From 7884ca51f71ad76c7b86dbe34782d72a223732cc Mon Sep 17 00:00:00 2001 From: DOHMATOB Elvis Date: Tue, 20 Feb 2018 15:38:49 +0100 Subject: [PATCH 5/5] ENH: logging nipype version --- continuous_integration/install.sh | 6 +++--- pypreprocess/configure_spm.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index fb67a91f..43544afe 100644 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -100,13 +100,13 @@ elif [[ "$DISTRIB" == "conda" ]]; then # useful if you happen to want a specific nibabel version rather # than the latest available one. if [ -n "$NIBABEL_VERSION" ]; then - pip install nibabel=="$NIBABEL_VERSION" + pip install nibabel=="$NIBABEL_VERSION" --upgrade fi if [ -n "$NIPYPE_VERSION" ]; then - pip install nipype=="$NIPYPE_VERSION" + pip install nipype=="$NIPYPE_VERSION" --upgrade fi if [ -n "$NILEARN_VERSION" ]; then - pip install nilearn=="$NILEARN_VERSION" + pip install nilearn=="$NILEARN_VERSION" --upgrade fi else diff --git a/pypreprocess/configure_spm.py b/pypreprocess/configure_spm.py index c5159690..b697d2ed 100644 --- a/pypreprocess/configure_spm.py +++ b/pypreprocess/configure_spm.py @@ -458,7 +458,7 @@ def _find_spm_mcr_and_spm_dir(cli_spm_mcr, config_spm_mcr, 'nipype version {} too old.' ' No support for precompiled SPM'.format(nipype.__version__)) return None - + _logger.info("using nipype version {}".format(nipype.__version__)) check_mcr = _IsValidMCR(cli_spm_dir, config_spm_dir, defaults) spm_mcr_envs = _get_exported('spm_mcr_env_template', templates_dict=defaults)