Skip to content

Suggestion: cmake system for downloading datasets #324

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anderkve wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Suggestion: cmake system for downloading datasets #324

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,9 @@ message("${Yellow}-- Updating GAMBIT module, backend, and printer CMake files -
# Add backends and scanners
include(cmake/externals.cmake)

# Add datasets
include(cmake/datasets.cmake)

# Add GAMBIT subdirectories.
add_subdirectory(Logs)
add_subdirectory(Utils)
Expand Down
3 changes: 2 additions & 1 deletion cmake/cleaning.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,5 @@ add_custom_target(nuke-BOSS COMMAND ${CMAKE_COMMAND} -E remove_directory Backend
COMMAND ${CMAKE_COMMAND} -E remove Backends/scripts/BOSS/castxml*.tar.gz)
add_custom_target(nuke-backends DEPENDS clean-backend-download clean-backend-install)
add_custom_target(nuke-scanners DEPENDS clean-scanner-download clean-scanner-install)
add_custom_target(nuke-all DEPENDS distclean nuke-contrib nuke-backends nuke-scanners nuke-BOSS)
add_custom_target(nuke-datasets)
add_custom_target(nuke-all DEPENDS distclean nuke-contrib nuke-backends nuke-scanners nuke-datasets nuke-BOSS)
77 changes: 77 additions & 0 deletions cmake/datasets.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# GAMBIT: Global and Modular BSM Inference Tool
#************************************************
# \file
#
# CMake configuration scripts for downloading
# datasets required by GAMBIT modules
#
#************************************************
#
# Authors (add name and date if you modify):
#
# \author Anders Kvellestad
# (anderkve@fys.uio.no)
# \date 2022 Feb
#
#************************************************

# Define the download command to use for datasets
set(DL_DATASET "${PROJECT_SOURCE_DIR}/cmake/scripts/safe_dl.sh" "${CMAKE_BINARY_DIR}" "${CMAKE_COMMAND}" "${CMAKE_DOWNLOAD_FLAGS}")

# Define a macro and function for adding nuke targets for downloaded datasets
macro(get_dataset_paths dataset_name build_path nuke_stamps)
set(stamp_path "${CMAKE_BINARY_DIR}/${dataset_name}-prefix/src/${dataset_name}-stamp/${dataset_name}")
set(${build_path} "${CMAKE_BINARY_DIR}/${dataset_name}-prefix/src/${dataset_name}-build")
set(${nuke_stamps} ${stamp_path}-download ${stamp_path}-mkdir ${stamp_path}-patch ${stamp_path}-update)
endmacro()

function(add_dataset_nuke dataset_name dir)
get_dataset_paths(${dataset_name} build_path nuke-stamps)
add_custom_target(nuke-${dataset_name} COMMAND ${CMAKE_COMMAND} -E remove -f ${nuke-stamps}
COMMAND ${CMAKE_COMMAND} -E remove_directory "${build_path}"
COMMAND ${CMAKE_COMMAND} -E remove_directory "${dir}")
add_dependencies(nuke-datasets nuke-${dataset_name})
add_dependencies(nuke-all nuke-${dataset_name})
endfunction()

# A function to check whether or not a given GAMBIT module has been ditched
function(check_module_ditch_status module_name)
foreach(ditch_command ${itch})
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "print(\"${module_name}\".lower().startswith(\"${ditch_command}\".lower()))"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
RESULT_VARIABLE result
OUTPUT_VARIABLE output)
if (output STREQUAL "True\n")
if(NOT ditched_${module_name})
set(ditched_${module_name} TRUE)
set(ditched_${module_name} TRUE PARENT_SCOPE)
endif()
endif()
endforeach()
endfunction()


# ===== Add entries for downloadable datasets below ====


# Best-fit SLHA files from the GAMBIT GUT-SUSY study in https://arxiv.org/abs/1705.07935
set(name "best_fits_SLHA_1705_07935")
set(ver "none")
set(dl "https://zenodo.org/record/843496/files/best_fits_SLHA.tar.gz")
set(md5 "none")
set(for_module "ExampleBit_A")
set(dir "${PROJECT_SOURCE_DIR}/${for_module}/data/${name}")
set(dataset_name "dataset-${name}")
check_module_ditch_status(${for_module})
if(NOT ditched_${for_module})
ExternalProject_Add(${dataset_name}
DOWNLOAD_COMMAND ${DL_DATASET} ${dl} ${md5} ${dir} ${name} ${ver}
SOURCE_DIR ${dir}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
)
add_dataset_nuke(${dataset_name} ${dir})
endif()


14 changes: 12 additions & 2 deletions cmake/scripts/safe_dl.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@
# (stoecker@physik.rwth-aachen.de)
# \date 2020 Aug
#
# \author Anders Kvellestad
# (anders.kvellestad@fys.uio.no)
# \date 2022 Feb
#
#************************************************

# Constants
Expand All @@ -62,13 +66,19 @@ if [ $(( ${force_axel} + ${force_wget} + ${force_curl})) -gt 1 ]; then
exit 1
fi

# If version number ($8) is set to "none", don't add "_version-number" to the file and path names
version=_$8
if [ "$8" = "none" ]; then
version=""
fi

# Download
axel_worked=0
suffix=$($2 -E echo $4 | grep -o '\(zip\|tar.gz\|tgz\)')
if [ ! -z ${sufix} ]; then
suffix=$($2 -E echo $4 | sed 's#.*\.##g')
fi
filename=$7_$8.${suffix}
filename=$7${version}.${suffix}
$2 -E make_directory $1 >/dev/null

# Perform download only if the tarball does not already exist (e.g. it was moved there manually)
Expand Down Expand Up @@ -150,7 +160,7 @@ if [ "$5" != "none" ]; then
$2 -E cmake_echo_color --red --bold "Deleting downloaded file."
# Delete the file if the md5 is bad, and make a stamp saying so, as cmake does not actually check if DOWNLOAD_COMMAND fails.
$2 -E remove $1/${filename}
$2 -E touch $7_$8-stamp/$7_$8-download-failed
$2 -E touch $7${version}-stamp/$7${version}-download-failed
exit 1
fi
}
Expand Down