diff --git a/CMakeLists.txt b/CMakeLists.txt index 707f8caaca3..af1951e0269 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,7 @@ if(ENABLE_RAPIDJSON) include(FetchContent) FetchContent_Declare( rapidjson - URL https://codeload.github.com/Tencent/rapidjson/tar.gz/24b5e7a8b27f42fa16b96fc70aade9106cf7102f + URL https://codeload.github.com/Tencent/rapidjson/tar.gz/24b5e7a ) set(RAPIDJSON_BUILD_TESTS OFF diff --git a/cmake/FindCereal.cmake b/cmake/FindCereal.cmake index f6c5f398776..204a1233f24 100644 --- a/cmake/FindCereal.cmake +++ b/cmake/FindCereal.cmake @@ -15,7 +15,7 @@ if(NOT CEREAL_INCLUDE_DIR) include(FetchContent) FetchContent_Declare( cereal - URL https://codeload.github.com/USCiLab/cereal/tar.gz/22a1b369f39be918ca79206a83c4facd759f9105 + URL https://codeload.github.com/USCiLab/cereal/tar.gz/22a1b36 ) FetchContent_Populate(cereal) set(CEREAL_INCLUDE_DIR ${cereal_SOURCE_DIR}/include) diff --git a/cmake/FindLibComm.cmake b/cmake/FindLibComm.cmake index 28016bfaa7b..64b34333998 100644 --- a/cmake/FindLibComm.cmake +++ b/cmake/FindLibComm.cmake @@ -16,7 +16,7 @@ if(NOT LIBCOMM_DIR) include(FetchContent) FetchContent_Declare( LibComm - URL https://codeload.github.com/abacusmodeling/LibComm/tar.gz/965bf90713ce5598bae4fd957d417065f77ceb42 + URL https://codeload.github.com/abacusmodeling/LibComm/tar.gz/965bf90 ) FetchContent_Populate(LibComm) set(LIBCOMM_DIR ${libcomm_SOURCE_DIR}) diff --git a/cmake/FindLibRI.cmake b/cmake/FindLibRI.cmake index 3208a862190..23b5426fbdd 100644 --- a/cmake/FindLibRI.cmake +++ b/cmake/FindLibRI.cmake @@ -16,7 +16,7 @@ if(NOT LIBRI_DIR) include(FetchContent) FetchContent_Declare( LibRI - URL https://codeload.github.com/abacusmodeling/LibRI/tar.gz/e6d78e0504693d2a464c72b87d3240ce183fd881 + URL https://codeload.github.com/abacusmodeling/LibRI/tar.gz/e6d78e0 ) FetchContent_Populate(LibRI) set(LIBRI_DIR ${libri_SOURCE_DIR}) diff --git a/toolchain/scripts/get_openblas_arch.sh b/toolchain/scripts/get_openblas_arch.sh index 6f61db4612f..79f40805e97 100755 --- a/toolchain/scripts/get_openblas_arch.sh +++ b/toolchain/scripts/get_openblas_arch.sh @@ -49,13 +49,8 @@ source "${INSTALLDIR}"/toolchain.env cd "${BUILDDIR}" echo "==================== Getting proc arch info using OpenBLAS tools ====================" -if [ -f ${openblas_pkg} ]; then - echo "${openblas_pkg} is found" -else -# using codeload.github - url="https://codeload.github.com/OpenMathLib/OpenBLAS/tar.gz/v${openblas_ver}" - download_pkg_from_url "${openblas_sha256}" "${openblas_pkg}" "${url}" -fi +url="https://codeload.github.com/OpenMathLib/OpenBLAS/tar.gz/v${openblas_ver}" +retrieve_package "${openblas_sha256}" "${openblas_pkg}" "${url}" # if toolchain run in pack-run mode, do exit if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip arch detection" diff --git a/toolchain/scripts/package_versions.sh b/toolchain/scripts/package_versions.sh index 713751037fd..d758c8ea0f3 100644 --- a/toolchain/scripts/package_versions.sh +++ b/toolchain/scripts/package_versions.sh @@ -18,12 +18,12 @@ gcc_alt_sha256="af828619dd1970734dda3cfb792ea3f2cba61b5a00170ba8bce4910749d73c07 # CMake (supports dual versions) - main=3.31.7, alt=3.30.5 cmake_main_ver="3.31.7" -cmake_main_sha256_x86_64="b7a5c909cdafc36042c8c9bd5765e92ff1f2528cf01720aa6dc4df294ec7e1a0" -cmake_main_sha256_aarch64="ce8e32b2c1c497dd7f619124c043ac5c28a88677e390c58748dd62fe460c62a2" +cmake_main_sha256_x86_64="14e15d0b445dbeac686acc13fe13b3135e8307f69ccf4c5c91403996ce5aa2d4" +cmake_main_sha256_aarch64="e5b2dc2aefdca10afe09c8fa4ee2bbb4e732665943a94322f99c118781910c3c" cmake_main_sha256_macos="1cb11aa2edae8551bb0f22807c6f5246bd0eb60ae9fa1474781eb4095d299aca" cmake_alt_ver="3.30.5" -cmake_alt_sha256_x86_64="83de8839f3fb0d9caf982a0435da3fa8c4fbe2c817dfec99def310dc7e6a8404" -cmake_alt_sha256_aarch64="93c3b8920379585dece1314f113c6c9008eaedfe56023c78d856fc86dad5b8e2" +cmake_alt_sha256_x86_64="f747d9b23e1a252a8beafb4ed2bc2ddf78cff7f04a8e4de19f4ff88e9b51dc9d" +cmake_alt_sha256_aarch64="da7dead2c92c1747b40d506d7f7d68590f5bab175316d2e7af73e48a2e417e48" cmake_alt_sha256_macos="3d603e507c7579b13518ef752b4ffcf3ed479fba80ee171d7d85da8153e869d0" # ============================================================================= @@ -100,21 +100,21 @@ libnpy_alt_ver="1.0.1" libnpy_alt_sha256="43452a4db1e8c1df606c64376ea1e32789124051d7640e7e4e8518ab4f0fba44" # Branch packages cut with specific commits -cereal_ver="22a1b369f39be918ca79206a83c4facd759f9105" -cereal_sha256="ca3d544456f93ea3176eab85e2e7892063af45b20467bb4ad9c1c51068e521c6" +cereal_ver="22a1b36" +cereal_sha256="a8171736e6b553dd6cd37919c13433b01f499d24d45af502975a9439728803e0" -libcomm_ver="965bf90713ce5598bae4fd957d417065f77ceb42" -libcomm_sha256="1d1626b833c51efa7ab844af140e022e063ea64389c9e47b168c24fcc05b7323" +libcomm_ver="965bf90" +libcomm_sha256="d7b991465d98d7b715b484d86880bf3525b9bf0cc62c3e5d38b0f6d140f6b9d4" -libri_ver="e6d78e0504693d2a464c72b87d3240ce183fd881" -libri_sha256="ead14f05b645f3d028e09f715abaa0e8fca3d1d6d61804239c4858b33397286a" +libri_ver="e6d78e0" +libri_sha256="619b49a14047d7a167515d1f1d0fa2d82fbebd63b8cbd3181e07df6ed993a22c" -rapidjson_ver="24b5e7a8b27f42fa16b96fc70aade9106cf7102f" -rapidjson_sha256="2d2601a82d2d3b7e143a3c8d43ef616671391034bc46891a9816b79cf2d3e7a8" +rapidjson_ver="24b5e7a" +rapidjson_sha256="dcb57b11036cb8fc6b2a57a6aded68d52e9cfe543811bf4fa8941087f84e72d0" # NEP (Neural Evolution Potential) - CPU version -nep_ver="629ec5dda3127e046fde3f8082aedfca8deb915e" -nep_sha256="57803d2e98ca5c8ced09e3e8d21ac1a0dbd61dda1977997864195f8c54b9b1f9" +nep_ver="629ec5d" +nep_sha256="4d4d3c64211a2a39e5d5c795b77befbba987cc809786e0cd6abfa46d0f3bf8cb" # ============================================================================= # Package Variable Loading Function diff --git a/toolchain/scripts/stage0/install_cmake.sh b/toolchain/scripts/stage0/install_cmake.sh index b3ef274c0e7..482c75d40af 100755 --- a/toolchain/scripts/stage0/install_cmake.sh +++ b/toolchain/scripts/stage0/install_cmake.sh @@ -71,11 +71,9 @@ cd "${BUILDDIR}" case "${with_cmake}" in __INSTALL__) echo "==================== Installing CMake ====================" - cmake_ext="sh" if [ "${OPENBLAS_ARCH}" = "arm64" ]; then if [ "$(uname -s)" = "Darwin" ]; then cmake_arch="macos-universal" - cmake_ext="tar.gz" elif [ "$(uname -s)" = "Linux" ]; then cmake_arch="linux-aarch64" else @@ -92,17 +90,12 @@ case "${with_cmake}" in pkg_install_dir="${INSTALLDIR}/cmake-${cmake_ver}" #pkg_install_dir="${HOME}/apps/cmake/${cmake_ver}" install_lock_file="${pkg_install_dir}/install_successful" - cmake_pkg="cmake-${cmake_ver}-${cmake_arch}.${cmake_ext}" + cmake_pkg="cmake-${cmake_ver}-${cmake_arch}.tar.gz" if verify_checksums "${install_lock_file}"; then echo "cmake-${cmake_ver} is already installed, skipping it." else - if [ -f $cmake_pkg ]; then - echo "$cmake_pkg is found" - else - #download_pkg_from_ABACUS_org "${cmake_sha256}" "$cmake_pkg" - url="https://cmake.org/files/v${cmake_ver%.*}/${cmake_pkg}" - download_pkg_from_url "${cmake_sha256}" "${cmake_pkg}" "${url}" - fi + url="https://cmake.org/files/v${cmake_ver%.*}/${cmake_pkg}" + retrieve_package "${cmake_sha256}" "${cmake_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -110,10 +103,11 @@ case "${with_cmake}" in echo "Installing from scratch into ${pkg_install_dir}" mkdir -p ${pkg_install_dir} if [ "${cmake_arch}" = "macos-universal" ]; then - tar --strip-components=3 -xvf $cmake_pkg -C ${pkg_install_dir} > install.log 2>&1 || tail -n ${LOG_LINES} install.log + strip_components=3 else - /bin/sh $cmake_pkg --prefix=${pkg_install_dir} --skip-license > install.log 2>&1 || tail -n ${LOG_LINES} install.log + strip_components=1 fi + tar --strip-components=${strip_components} -xvf ${cmake_pkg} -C ${pkg_install_dir} > install.log 2>&1 || tail_excerpt install.log write_checksums "${install_lock_file}" "${SCRIPT_DIR}/stage0/$(basename ${SCRIPT_NAME})" fi ;; diff --git a/toolchain/scripts/stage0/install_gcc.sh b/toolchain/scripts/stage0/install_gcc.sh index fe4b4c44ad5..6091b8c450e 100755 --- a/toolchain/scripts/stage0/install_gcc.sh +++ b/toolchain/scripts/stage0/install_gcc.sh @@ -59,13 +59,8 @@ case "${with_gcc}" in tar -xzf "${repack_filename}" echo "Successfully extracted ${repack_filename}" else - if [ -f gcc-${gcc_ver}.tar.gz ]; then - echo "gcc-${gcc_ver}.tar.gz is found" - else - #download_pkg_from_ABACUS_org "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz" - url=https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-${gcc_ver}/gcc-${gcc_ver}.tar.gz - download_pkg_from_url "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz" "${url}" - fi + url=https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-${gcc_ver}/gcc-${gcc_ver}.tar.gz + retrieve_package "${gcc_sha256}" "gcc-${gcc_ver}.tar.gz" "${url}" fi [ -d gcc-${gcc_ver} ] && rm -rf gcc-${gcc_ver} tar -xzf gcc-${gcc_ver}.tar.gz diff --git a/toolchain/scripts/stage1/install_mpich.sh b/toolchain/scripts/stage1/install_mpich.sh index b171ac1a746..fc48b54080d 100755 --- a/toolchain/scripts/stage1/install_mpich.sh +++ b/toolchain/scripts/stage1/install_mpich.sh @@ -50,12 +50,7 @@ case "${with_mpich}" in if verify_checksums "${install_lock_file}"; then echo "mpich-${mpich_ver} is already installed, skipping it." else - if [ -f ${mpich_pkg} ]; then - echo "${mpich_pkg} is found" - else - #download_pkg_from_ABACUS_org "${mpich_sha256}" "${mpich_pkg}" - download_pkg_from_url "${mpich_sha256}" "${mpich_pkg}" "${url}" - fi + retrieve_package "${mpich_sha256}" "${mpich_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 diff --git a/toolchain/scripts/stage1/install_openmpi.sh b/toolchain/scripts/stage1/install_openmpi.sh index 9b0836d1171..a3a4583a956 100755 --- a/toolchain/scripts/stage1/install_openmpi.sh +++ b/toolchain/scripts/stage1/install_openmpi.sh @@ -52,11 +52,7 @@ case "${with_openmpi}" in if verify_checksums "${install_lock_file}"; then echo "openmpi-${openmpi_ver} is already installed, skipping it." else - if [ -f ${openmpi_pkg} ]; then - echo "${openmpi_pkg} is found" - else - download_pkg_from_url "${openmpi_sha256}" "${openmpi_pkg}" "${url}" - fi + retrieve_package "${openmpi_sha256}" "${openmpi_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 diff --git a/toolchain/scripts/stage2/install_aocl.sh b/toolchain/scripts/stage2/install_aocl.sh index a7f9b377915..01701171930 100755 --- a/toolchain/scripts/stage2/install_aocl.sh +++ b/toolchain/scripts/stage2/install_aocl.sh @@ -66,6 +66,7 @@ export MATH_CFLAGS="\${MATH_CFLAGS} ${AOCL_CFLAGS}" export MATH_LDFLAGS="\${MATH_LDFLAGS} ${AOCL_LDFLAGS}" export MATH_LIBS="\${MATH_LIBS} ${AOCL_LIBS}" EOF + cat "${BUILDDIR}/setup_aocl" >> $SETUPFILE fi load "${BUILDDIR}/setup_aocl" diff --git a/toolchain/scripts/stage2/install_openblas.sh b/toolchain/scripts/stage2/install_openblas.sh index 3b5800f54f2..45d85e94939 100755 --- a/toolchain/scripts/stage2/install_openblas.sh +++ b/toolchain/scripts/stage2/install_openblas.sh @@ -50,13 +50,8 @@ case "${with_openblas}" in if verify_checksums "${install_lock_file}"; then echo "openblas-${openblas_ver} is already installed, skipping it." else - if [ -f ${openblas_pkg} ]; then - echo "${openblas_pkg} is found" - else - # using codeload.github - url="https://codeload.github.com/OpenMathLib/OpenBLAS/tar.gz/v${openblas_ver}" - download_pkg_from_url "${openblas_sha256}" "${openblas_pkg}" "${url}" - fi + url="https://codeload.github.com/OpenMathLib/OpenBLAS/tar.gz/v${openblas_ver}" + retrieve_package "${openblas_sha256}" "${openblas_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -177,7 +172,6 @@ prepend_path LD_RUN_PATH "${pkg_install_dir}/lib" prepend_path LIBRARY_PATH "${pkg_install_dir}/lib" prepend_path CPATH "${pkg_install_dir}/include" EOF - cat "${BUILDDIR}/setup_openblas" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_openblas" export OPENBLAS_ROOT="${pkg_install_dir}" @@ -190,6 +184,7 @@ export MATH_LIBS="\${MATH_LIBS} ${OPENBLAS_LIBS}" prepend_path PKG_CONFIG_PATH "${pkg_install_dir}/lib/pkgconfig" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_openblas" >> $SETUPFILE fi load "${BUILDDIR}/setup_openblas" diff --git a/toolchain/scripts/stage3/install_elpa.sh b/toolchain/scripts/stage3/install_elpa.sh index ab343c1a793..7d72e33e58e 100755 --- a/toolchain/scripts/stage3/install_elpa.sh +++ b/toolchain/scripts/stage3/install_elpa.sh @@ -74,11 +74,7 @@ case "$with_elpa" in else require_env MATH_LIBS url="https://elpa.mpcdf.mpg.de/software/tarball-archive/Releases/${elpa_ver}/${elpa_pkg}" - if [ -f ${elpa_pkg} ]; then - echo "${elpa_pkg} is found" - else - download_pkg_from_url "${elpa_sha256}" "${elpa_pkg}" "${url}" - fi + retrieve_package "${elpa_sha256}" "${elpa_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -258,7 +254,6 @@ prepend_path LIBRARY_PATH "${pkg_install_dir}/lib" prepend_path PKG_CONFIG_PATH "${pkg_install_dir}/lib/pkgconfig" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF - cat "${BUILDDIR}/setup_elpa" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_elpa" export ELPA_ROOT="${pkg_install_dir}" @@ -271,7 +266,7 @@ export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(${ELPA_LDFLAGS}|)" export CP_LIBS="IF_MPI(${ELPA_LIBS}|) \${CP_LIBS}" export ELPA_VERSION="${elpa_ver}" EOF - + cat "${BUILDDIR}/setup_elpa" >> $SETUPFILE fi load "${BUILDDIR}/setup_elpa" diff --git a/toolchain/scripts/stage3/install_fftw.sh b/toolchain/scripts/stage3/install_fftw.sh index 31396455d62..a40f514ca44 100755 --- a/toolchain/scripts/stage3/install_fftw.sh +++ b/toolchain/scripts/stage3/install_fftw.sh @@ -49,12 +49,8 @@ case "$with_fftw" in if verify_checksums "${install_lock_file}"; then echo "fftw-${fftw_ver} is already installed, skipping it." else - if [ -f ${fftw_pkg} ]; then - echo "${fftw_pkg} is found" - else - url="http://www.fftw.org/${fftw_pkg}" - download_pkg_from_url "${fftw_sha256}" "${fftw_pkg}" "${url}" - fi + url="http://www.fftw.org/${fftw_pkg}" + retrieve_package "${fftw_sha256}" "${fftw_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 diff --git a/toolchain/scripts/stage3/install_libxc.sh b/toolchain/scripts/stage3/install_libxc.sh index 3afb4b4fee2..e3efa166a67 100755 --- a/toolchain/scripts/stage3/install_libxc.sh +++ b/toolchain/scripts/stage3/install_libxc.sh @@ -51,13 +51,8 @@ case "$with_libxc" in if verify_checksums "${install_lock_file}"; then echo "libxc-${libxc_ver} is already installed, skipping it." else - if [ -f ${libxc_pkg} ]; then - echo "${libxc_pkg} is found" - else - #download_pkg_from_ABACUS_org "${libxc_sha256}" "${libxc_pkg}" - libxc_url="https://gitlab.com/libxc/libxc/-/archive/${libxc_ver}/${libxc_pkg}" - download_pkg_from_url "${libxc_sha256}" "${libxc_pkg}" "${libxc_url}" - fi + libxc_url="https://gitlab.com/libxc/libxc/-/archive/${libxc_ver}/${libxc_pkg}" + retrieve_package "${libxc_sha256}" "${libxc_pkg}" "${libxc_url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -121,7 +116,6 @@ prepend_path CPATH "${pkg_install_dir}/include" prepend_path PKG_CONFIG_PATH "${pkg_install_dir}/lib/pkgconfig" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF - cat "${BUILDDIR}/setup_libxc" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_libxc" export LIBXC_CFLAGS="${LIBXC_CFLAGS}" @@ -133,6 +127,7 @@ export CP_LDFLAGS="\${CP_LDFLAGS} ${LIBXC_LDFLAGS}" export CP_LIBS="${LIBXC_LIBS} \${CP_LIBS}" export LIBXC_ROOT="${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_libxc" >> $SETUPFILE fi load "${BUILDDIR}/setup_libxc" diff --git a/toolchain/scripts/stage3/install_scalapack.sh b/toolchain/scripts/stage3/install_scalapack.sh index e03742efa2e..9151e7dfebe 100755 --- a/toolchain/scripts/stage3/install_scalapack.sh +++ b/toolchain/scripts/stage3/install_scalapack.sh @@ -49,13 +49,8 @@ case "${with_scalapack}" in echo "scalapack-${scalapack_ver} is already installed, skipping it." else require_env MATH_LIBS - if [ -f ${scalapack_pkg} ]; then - echo "${scalapack_pkg} is found" - else - url="https://codeload.github.com/Reference-ScaLAPACK/scalapack/tar.gz/v${scalapack_ver}" - download_pkg_from_url "${scalapack_sha256}" "${scalapack_pkg}" "${url}" - #download_pkg_from_ABACUS_org "${scalapack_sha256}" "${scalapack_pkg}" - fi + url="https://codeload.github.com/Reference-ScaLAPACK/scalapack/tar.gz/v${scalapack_ver}" + retrieve_package "${scalapack_sha256}" "${scalapack_pkg}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -120,7 +115,6 @@ prepend_path LIBRARY_PATH "${pkg_install_dir}/lib" prepend_path PKG_CONFIG_PATH "${pkg_install_dir}/lib/pkgconfig" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF - cat "${BUILDDIR}/setup_scalapack" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_scalapack" export SCALAPACK_ROOT="${pkg_install_dir}" @@ -130,6 +124,7 @@ export CP_DFLAGS="\${CP_DFLAGS} IF_MPI(-D__SCALAPACK|)" export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(${SCALAPACK_LDFLAGS}|)" export CP_LIBS="IF_MPI(-lscalapack|) \${CP_LIBS}" EOF + cat "${BUILDDIR}/setup_scalapack" >> $SETUPFILE fi cd "${ROOTDIR}" diff --git a/toolchain/scripts/stage4/install_cereal.sh b/toolchain/scripts/stage4/install_cereal.sh index b03861d4c19..e442a2413d9 100755 --- a/toolchain/scripts/stage4/install_cereal.sh +++ b/toolchain/scripts/stage4/install_cereal.sh @@ -30,13 +30,8 @@ if [[ -z "$version_suffix" && -n "${ABACUS_TOOLCHAIN_VERSION_SUFFIX}" ]]; then fi # Load package variables with appropriate version load_package_vars "cereal" "$version_suffix" -if [[ "${cereal_ver}" =~ ^[0-9a-f]{40}$ ]]; then - short_ver="${cereal_ver:0:7}" -else - short_ver="${cereal_ver}" -fi -dirname="cereal-${short_ver}" -filename="cereal-${short_ver}.tar.gz" +dirname="cereal-${cereal_ver}" +filename="cereal-${cereal_ver}.tar.gz" source "${INSTALLDIR}"/toolchain.conf source "${INSTALLDIR}"/toolchain.env @@ -55,7 +50,7 @@ case "$with_cereal" in # url construction rules: # - Branch names (master, main, develop) without v prefix # - Version tags (e.g., 1.0.0) with v prefix - if [[ "${cereal_ver}" =~ ^[0-9a-f]{40}$ ]]; then + if [[ "${cereal_ver}" =~ ^([0-9a-f]{7}|[0-9a-f]{40})$ ]]; then url="https://codeload.github.com/USCiLab/cereal/tar.gz/${cereal_ver}" else url="https://codeload.github.com/USCiLab/cereal/tar.gz/v${cereal_ver}" @@ -63,12 +58,7 @@ case "$with_cereal" in if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - # download from github.com and checksum - download_pkg_from_url "${cereal_sha256}" "${filename}" "${url}" - fi + retrieve_package "${cereal_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -120,7 +110,6 @@ prepend_path CPATH "${pkg_install_dir}/include" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF fi - cat "${BUILDDIR}/setup_cereal" >> $SETUPFILE cat << EOF >> "${BUILDDIR}/setup_cereal" export CEREAL_ROOT="${pkg_install_dir}" export CEREAL_CFLAGS="${CEREAL_CFLAGS}" @@ -128,6 +117,7 @@ export CP_DFLAGS="\${CP_DFLAGS} -D__CEREAL" export CP_CFLAGS="\${CP_CFLAGS} ${CEREAL_CFLAGS}" export CEREAL_VERSION="${cereal_ver}" EOF + cat "${BUILDDIR}/setup_cereal" >> $SETUPFILE fi load "${BUILDDIR}/setup_cereal" diff --git a/toolchain/scripts/stage4/install_libcomm.sh b/toolchain/scripts/stage4/install_libcomm.sh index c7b14f075e6..0d2c9ae24aa 100755 --- a/toolchain/scripts/stage4/install_libcomm.sh +++ b/toolchain/scripts/stage4/install_libcomm.sh @@ -34,13 +34,8 @@ if [[ -z "$version_suffix" && -n "${ABACUS_TOOLCHAIN_VERSION_SUFFIX}" ]]; then fi # Load package variables with appropriate version load_package_vars "libcomm" "$version_suffix" -if [[ "${libcomm_ver}" =~ ^[0-9a-f]{40}$ ]]; then - short_ver="${libcomm_ver:0:7}" -else - short_ver="${libcomm_ver}" -fi -dirname="LibComm-${short_ver}" -filename="LibComm-${short_ver}.tar.gz" +dirname="LibComm-${libcomm_ver}" +filename="LibComm-${libcomm_ver}.tar.gz" source "${INSTALLDIR}"/toolchain.conf source "${INSTALLDIR}"/toolchain.env @@ -59,7 +54,7 @@ case "$with_libcomm" in # url construction rules: # - Branch names (master, main, develop) without v prefix # - Version tags (e.g., 1.0.0) with v prefix - if [[ "${libcomm_ver}" =~ ^[0-9a-f]{40}$ ]]; then + if [[ "${libcomm_ver}" =~ ^([0-9a-f]{7}|[0-9a-f]{40})$ ]]; then url="https://codeload.github.com/abacusmodeling/LibComm/tar.gz/${libcomm_ver}" else url="https://codeload.github.com/abacusmodeling/LibComm/tar.gz/v${libcomm_ver}" @@ -67,12 +62,7 @@ case "$with_libcomm" in if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - # download from github.com and checksum - download_pkg_from_url "${libcomm_sha256}" "${filename}" "${url}" - fi + retrieve_package "${libcomm_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -120,12 +110,12 @@ if [ "$with_libcomm" != "__DONTUSE__" ]; then cat << EOF > "${BUILDDIR}/setup_libcomm" prepend_path CPATH "${pkg_install_dir}/include" EOF - cat "${BUILDDIR}/setup_libcomm" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_libcomm" export LIBCOMM_CFLAGS="${LIBCOMM_CFLAGS}" export LIBCOMM_ROOT="${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_libcomm" >> $SETUPFILE fi load "${BUILDDIR}/setup_libcomm" diff --git a/toolchain/scripts/stage4/install_libnpy.sh b/toolchain/scripts/stage4/install_libnpy.sh index bdd7c2b3959..8a0a83b5053 100755 --- a/toolchain/scripts/stage4/install_libnpy.sh +++ b/toolchain/scripts/stage4/install_libnpy.sh @@ -54,12 +54,7 @@ case "$with_libnpy" in if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - # download from github.com and checksum - download_pkg_from_url "${libnpy_sha256}" "${filename}" "${url}" - fi + retrieve_package "${libnpy_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -107,12 +102,12 @@ if [ "$with_libnpy" != "__DONTUSE__" ]; then cat << EOF > "${BUILDDIR}/setup_libnpy" prepend_path CPATH "${pkg_install_dir}/include" EOF - cat "${BUILDDIR}/setup_libnpy" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_libnpy" export LIBNPY_CFLAGS="${LIBNPY_CFLAGS}" export LIBNPY_ROOT="${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_libnpy" >> $SETUPFILE fi load "${BUILDDIR}/setup_libnpy" diff --git a/toolchain/scripts/stage4/install_libri.sh b/toolchain/scripts/stage4/install_libri.sh index a3dd10422d6..a0454e08e2e 100755 --- a/toolchain/scripts/stage4/install_libri.sh +++ b/toolchain/scripts/stage4/install_libri.sh @@ -34,11 +34,6 @@ if [[ -z "$version_suffix" && -n "${ABACUS_TOOLCHAIN_VERSION_SUFFIX}" ]]; then fi # Load package variables with appropriate version load_package_vars "libri" "$version_suffix" -if [[ "${libri_ver}" =~ ^[0-9a-f]{40}$ ]]; then - short_ver="${libri_ver:0:7}" -else - short_ver="${libri_ver}" -fi source "${INSTALLDIR}"/toolchain.conf source "${INSTALLDIR}"/toolchain.env @@ -50,28 +45,23 @@ cd "${BUILDDIR}" case "$with_libri" in __INSTALL__) echo "==================== Installing LIBRI ====================" - dirname="LibRI-${short_ver}" + dirname="LibRI-${libri_ver}" pkg_install_dir="${INSTALLDIR}/$dirname" #pkg_install_dir="${HOME}/lib/libri/${libri_ver}" install_lock_file="${pkg_install_dir}/install_successful" # url construction rules: # - Branch names (master, main, develop) without v prefix # - Version tags (e.g., 1.0.0) with v prefix - if [[ "${libri_ver}" =~ ^[0-9a-f]{40}$ ]]; then + if [[ "${libri_ver}" =~ ^([0-9a-f]{7}|[0-9a-f]{40})$ ]]; then url="https://codeload.github.com/abacusmodeling/LibRI/tar.gz/${libri_ver}" else url="https://codeload.github.com/abacusmodeling/LibRI/tar.gz/v${libri_ver}" fi - filename="LibRI-${short_ver}.tar.gz" + filename="LibRI-${libri_ver}.tar.gz" if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - # download from github.com and checksum - download_pkg_from_url "${libri_sha256}" "${filename}" "${url}" - fi + retrieve_package "${libri_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -119,12 +109,12 @@ if [ "$with_libri" != "__DONTUSE__" ]; then cat << EOF > "${BUILDDIR}/setup_libri" prepend_path CPATH "${pkg_install_dir}/include" EOF - cat "${BUILDDIR}/setup_libri" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_libri" export LIBRI_CFLAGS="${LIBRI_CFLAGS}" export LIBRI_ROOT="${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_libri" >> $SETUPFILE fi load "${BUILDDIR}/setup_libri" diff --git a/toolchain/scripts/stage4/install_libtorch.sh b/toolchain/scripts/stage4/install_libtorch.sh index 2d46178a244..4b821315e57 100755 --- a/toolchain/scripts/stage4/install_libtorch.sh +++ b/toolchain/scripts/stage4/install_libtorch.sh @@ -53,13 +53,8 @@ case "${with_libtorch}" in if verify_checksums "${install_lock_file}"; then echo "${dirname} is already installed, skipping it." else - if [ -f ${filename} ]; then - echo "${filename} is found" - else - # download from pytorch.com and checksum - url=https://download.pytorch.org/libtorch/cpu/${archive_file} - download_pkg_from_url "${libtorch_sha256}" "${filename}" "${url}" - fi + url=https://download.pytorch.org/libtorch/cpu/${archive_file} + retrieve_package "${libtorch_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -121,7 +116,6 @@ export CXXFLAGS="\${CXXFLAGS} ${LIBTORCH_CXXFLAGS}" export CP_LDFLAGS="\${CP_LDFLAGS} ${LIBTORCH_LDFLAGS}" export CP_LIBS="\${CP_LIBS} -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda -ltorch" EOF - cat "${BUILDDIR}/setup_libtorch" >> "${SETUPFILE}" cat << EOF >> "${BUILDDIR}/setup_libtorch" export CP_DFLAGS="\${CP_DFLAGS} -D__LIBTORCH" export CXXFLAGS="\${CXXFLAGS} ${LIBTORCH_CXXFLAGS}" diff --git a/toolchain/scripts/stage4/install_nep.sh b/toolchain/scripts/stage4/install_nep.sh index 0bcfef77ab4..0d510c21cfc 100755 --- a/toolchain/scripts/stage4/install_nep.sh +++ b/toolchain/scripts/stage4/install_nep.sh @@ -37,11 +37,7 @@ case "$with_nep" in if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - download_pkg_from_url "${nep_sha256}" "${filename}" "${url}" - fi + retrieve_package "${nep_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" @@ -130,7 +126,6 @@ prepend_path LIBRARY_PATH "${pkg_install_dir}/lib" prepend_path CPATH "${pkg_install_dir}/include" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF - cat "${BUILDDIR}/setup_nep" >> $SETUPFILE fi cat << EOF >> "${BUILDDIR}/setup_nep" export NEP_CFLAGS="${NEP_CFLAGS}" @@ -142,6 +137,7 @@ export CP_LDFLAGS="\${CP_LDFLAGS} \${NEP_LDFLAGS}" export CP_LIBS="\${NEP_LIBS} \${CP_LIBS}" export NEP_ROOT="${pkg_install_dir}" EOF + cat "${BUILDDIR}/setup_nep" >> $SETUPFILE fi load "${BUILDDIR}/setup_nep" diff --git a/toolchain/scripts/stage4/install_rapidjson.sh b/toolchain/scripts/stage4/install_rapidjson.sh index 6cb560af429..a92d5602e92 100755 --- a/toolchain/scripts/stage4/install_rapidjson.sh +++ b/toolchain/scripts/stage4/install_rapidjson.sh @@ -32,11 +32,6 @@ if [[ -z "$version_suffix" && -n "${ABACUS_TOOLCHAIN_VERSION_SUFFIX}" ]]; then fi # Load package variables with appropriate version load_package_vars "rapidjson" "$version_suffix" -if [[ "${rapidjson_ver}" =~ ^[0-9a-f]{40}$ ]]; then - short_ver="${rapidjson_ver:0:7}" -else - short_ver="${rapidjson_ver}" -fi source "${INSTALLDIR}"/toolchain.conf source "${INSTALLDIR}"/toolchain.env @@ -49,28 +44,23 @@ cd "${BUILDDIR}" case "$with_rapidjson" in __INSTALL__) echo "==================== Installing RapidJSON ====================" - dirname="rapidjson-${short_ver}" + dirname="rapidjson-${rapidjson_ver}" pkg_install_dir="${INSTALLDIR}/$dirname" #pkg_install_dir="${HOME}/lib/rapidjson/${rapidjson_ver}" install_lock_file="${pkg_install_dir}/install_successful" # url construction rules: # - Branch names (master, main, develop) without v prefix # - Version tags (e.g., 1.0.0) with v prefix - if [[ "${rapidjson_ver}" =~ ^[0-9a-f]{40}$ ]]; then + if [[ "${rapidjson_ver}" =~ ^([0-9a-f]{7}|[0-9a-f]{40})$ ]]; then url="https://codeload.github.com/Tencent/rapidjson/tar.gz/${rapidjson_ver}" else url="https://codeload.github.com/Tencent/rapidjson/tar.gz/v${rapidjson_ver}" fi - filename="rapidjson-${short_ver}.tar.gz" + filename="rapidjson-${rapidjson_ver}.tar.gz" if verify_checksums "${install_lock_file}"; then echo "$dirname is already installed, skipping it." else - if [ -f $filename ]; then - echo "$filename is found" - else - # download from github.com and checksum - download_pkg_from_url "${rapidjson_sha256}" "${filename}" "${url}" - fi + retrieve_package "${rapidjson_sha256}" "${filename}" "${url}" if [ "${PACK_RUN}" = "__TRUE__" ]; then echo "--pack-run mode specified, skip installation" exit 0 @@ -127,7 +117,6 @@ prepend_path CPATH "${pkg_install_dir}/include" prepend_path CMAKE_PREFIX_PATH "${pkg_install_dir}" EOF fi - cat "${BUILDDIR}/setup_rapidjson" >> $SETUPFILE cat << EOF >> "${BUILDDIR}/setup_rapidjson" export RAPIDJSON_ROOT="${pkg_install_dir}" export RAPIDJSON_CFLAGS="${RAPIDJSON_CFLAGS}" @@ -135,6 +124,7 @@ export CP_DFLAGS="\${CP_DFLAGS} -D__RAPIDJSON" export CP_CFLAGS="\${CP_CFLAGS} ${RAPIDJSON_CFLAGS}" export RAPIDJSON_VERSION="${rapidjson_ver}" EOF + cat "${BUILDDIR}/setup_rapidjson" >> $SETUPFILE fi load "${BUILDDIR}/setup_rapidjson" diff --git a/toolchain/scripts/tool_kit.sh b/toolchain/scripts/tool_kit.sh index b7d377cef9d..7e415e80c17 100755 --- a/toolchain/scripts/tool_kit.sh +++ b/toolchain/scripts/tool_kit.sh @@ -871,13 +871,7 @@ checksum() { # sha256sum, but has an equivalent with shasum -a 256 command -v "$__shasum_command" > /dev/null 2>&1 || __shasum_command="shasum -a 256" - if echo "$__sha256 $__filename" | ${__shasum_command} --check; then - echo "Checksum of $__filename Ok" - else - rm -v ${__filename} - report_error "Checksum of $__filename could not be verified, abort." - return 1 - fi + echo "$__sha256 $__filename" | ${__shasum_command} --check } # Enhanced checksum verification with multiple hash algorithms @@ -923,29 +917,6 @@ verify_file_integrity() { fi } -# downloader for the package tars, includes checksum -# backup and deprecated -download_pkg_from_org() { - # usage: download_pkg_from_org sha256 filename - echo "use cp2k mirror to download $__filename" - local __sha256="$1" - local __filename="$2" - local __url="https://www.cp2k.org/static/downloads/$__filename" - # download - #echo "wget ${DOWNLOADER_FLAGS} --quiet $__url" - #if ! wget ${DOWNLOADER_FLAGS} --quiet $__url; then - echo "wget ${DOWNLOADER_FLAGS} $__url" - if ! wget ${DOWNLOADER_FLAGS} $__url; then - report_error "failed to download $__url" - recommend_offline_installation $__filename $__url - if [ "${PACK_RUN}" != "__TRUE__" ]; then - return 1 - fi - fi - # checksum - checksum "$__filename" "$__sha256" -} - download_pkg_from_url() { # usage: download_pkg_from_url sha256 filename url local __sha256="$1" # if set to "--no-checksum", do not check checksum @@ -956,7 +927,7 @@ download_pkg_from_url() { case "${DOWNLOAD_CERT_POLICY:-smart}" in "strict") echo "Downloading with strict certificate validation: $__url" - if ! wget ${DOWNLOADER_FLAGS} "$__url" -O "$__filename"; then + if ! wget --quiet --show-progress ${DOWNLOADER_FLAGS} "$__url" -O "$__filename"; then rm -f "$__filename" report_error "failed to download $__url (strict certificate validation)" recommend_offline_installation "$__filename" "$__url" @@ -967,7 +938,7 @@ download_pkg_from_url() { ;; "skip") echo "Downloading with certificate validation disabled: $__url" - if ! wget ${DOWNLOADER_FLAGS} "$__url" -O "$__filename" --no-check-certificate; then + if ! wget --quiet --show-progress ${DOWNLOADER_FLAGS} "$__url" -O "$__filename" --no-check-certificate; then rm -f "$__filename" report_error "failed to download $__url" recommend_offline_installation "$__filename" "$__url" @@ -979,7 +950,7 @@ download_pkg_from_url() { "smart"|*) # Smart fallback: try with certificate validation first, then without echo "Attempting secure download: $__url" - if wget ${DOWNLOADER_FLAGS} "$__url" -O "$__filename"; then + if wget --quiet --show-progress ${DOWNLOADER_FLAGS} "$__url" -O "$__filename"; then echo "Download successful with certificate validation" else echo "Certificate validation failed, retrying without certificate check..." @@ -997,9 +968,34 @@ download_pkg_from_url() { ;; esac - # checksum validation (unchanged) - if [ "$__sha256" != "--no-checksum" ]; then - checksum "$__filename" "$__sha256" + # checksum + if checksum "$__filename" "$__sha256"; then + echo "Checksum of $__filename OK" + else + rm -vf "${__filename}" + report_error "Checksum of $__filename could not be verified, abort." + return 1 + fi +} + +# retrieve package under current directory with filename and checksum verification +# if file exists and checksum is correct, print the success message +# if file exists but checksum is incorrect, delete and re-download +# if file does not exist, download from corresponding websites +retrieve_package() { + local __sha256="$1" + local __filename="$2" + local __url="$3" + if ! [ -f "${__filename}" ]; then + download_pkg_from_url "${__sha256}" "${__filename}" "${__url}" + else + if ! checksum "$__filename" "$__sha256"; then + echo "$__filename is found but checksum is wrong; delete and re-download" + rm -vf "${__filename}" + download_pkg_from_url "${__sha256}" "${__filename}" "${__url}" + else + echo "$__filename is found and checksum is right" + fi fi }