Skip to content

feat: add iceberg_data library alongside iceberg #631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhjwpku wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add iceberg_data library alongside iceberg #631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions cmake_modules/IcebergBuildUtils.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@

include(CMakePackageConfigHelpers)

function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME)
function(iceberg_install_cmake_package PACKAGE_NAME)
set(EXPORT_NAME "${PACKAGE_NAME}_targets")
set(CONFIG_CMAKE "${PACKAGE_NAME}-config.cmake")
set(BUILT_CONFIG_CMAKE "${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_CMAKE}")
configure_package_config_file("${CONFIG_CMAKE}.in" "${BUILT_CONFIG_CMAKE}"
Expand All @@ -33,9 +34,11 @@ function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME)
install(FILES "${BUILT_CONFIG_CMAKE}" "${BUILT_CONFIG_VERSION_CMAKE}"
DESTINATION "${ICEBERG_INSTALL_CMAKEDIR}/${PACKAGE_NAME}")
set(TARGETS_CMAKE "${PACKAGE_NAME}-targets.cmake")
# Use iceberg:: for every export (not ${PACKAGE_NAME}::) so install INTERFACE
# link lines like iceberg::iceberg_data_static resolve across packages.
install(EXPORT ${EXPORT_NAME}
DESTINATION "${ICEBERG_INSTALL_CMAKEDIR}/${PACKAGE_NAME}"
NAMESPACE "${PACKAGE_NAME}::"
NAMESPACE "iceberg::"
FILE "${TARGETS_CMAKE}")
endfunction()

Expand Down Expand Up @@ -73,6 +76,8 @@ function(add_iceberg_lib LIB_NAME)
set(${ARG_OUTPUTS})
endif()

set(EXPORT_NAME "${LIB_NAME}_targets")

# Allow overriding ICEBERG_BUILD_SHARED and ICEBERG_BUILD_STATIC
if(DEFINED ARG_BUILD_SHARED)
set(BUILD_SHARED ${ARG_BUILD_SHARED})
Expand Down Expand Up @@ -162,7 +167,7 @@ function(add_iceberg_lib LIB_NAME)
endif()

install(TARGETS ${LIB_NAME}_shared
EXPORT iceberg_targets
EXPORT ${EXPORT_NAME}
ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR}
LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}
RUNTIME DESTINATION ${INSTALL_RUNTIME_DIR}
Expand Down Expand Up @@ -229,7 +234,7 @@ function(add_iceberg_lib LIB_NAME)
endif()

install(TARGETS ${LIB_NAME}_static
EXPORT iceberg_targets
EXPORT ${EXPORT_NAME}
ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR}
LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}
RUNTIME DESTINATION ${INSTALL_RUNTIME_DIR}
Expand Down
63 changes: 33 additions & 30 deletions cmake_modules/IcebergThirdpartyToolchain.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.

# Accumulate all dependencies to provide suitable static link parameters to the
# third party libraries.
# Per-package lists for *-config.cmake.in (iceberg avoids Arrow/Avro/cpr).
set(ICEBERG_SYSTEM_DEPENDENCIES)
set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES)
set(ICEBERG_REST_SYSTEM_DEPENDENCIES)
set(ICEBERG_ARROW_INSTALL_INTERFACE_LIBS)

# ----------------------------------------------------------------------
Expand Down Expand Up @@ -92,7 +93,7 @@ macro(prepare_fetchcontent)
endmacro()

# ----------------------------------------------------------------------
# Apache Arrow
# Apache Arrow (bundle-only)

function(resolve_arrow_dependency)
prepare_fetchcontent()
Expand Down Expand Up @@ -149,7 +150,7 @@ function(resolve_arrow_dependency)
set_target_properties(parquet_static PROPERTIES OUTPUT_NAME
"iceberg_vendored_parquet")
install(TARGETS arrow_static parquet_static
EXPORT iceberg_targets
EXPORT iceberg_bundle_targets
RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
Expand All @@ -167,28 +168,30 @@ function(resolve_arrow_dependency)
endif()

# Arrow's exported static target interface may reference system libraries
# (e.g. OpenSSL, CURL, ZLIB) that consumers need to find.
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB)
# (e.g. OpenSSL, CURL, ZLIB) that consumers need to find. ZLIB is also a
# core dependency (see resolve_zlib_dependency()), so it is intentionally
# placed in the lowest layer; OpenSSL/CURL are bundle-only because they
# are pulled in by Arrow's S3 filesystem.
if(ARROW_S3)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL)
list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES OpenSSL CURL)
endif()
else()
set(ARROW_VENDORED FALSE)
find_package(Arrow CONFIG REQUIRED)
find_package(Parquet CONFIG REQUIRED)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet)
list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Arrow Parquet)
endif()

set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES
${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
set(ARROW_VENDORED
${ARROW_VENDORED}
PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------
# Apache Avro
# Apache Avro (bundle-only)

function(resolve_avro_dependency)
prepare_fetchcontent()
Expand Down Expand Up @@ -249,31 +252,31 @@ function(resolve_avro_dependency)
set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp")
set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON)
install(TARGETS avrocpp_s
EXPORT iceberg_targets
EXPORT iceberg_bundle_targets
RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")

# TODO: add vendored ZLIB and Snappy support
find_package(Snappy CONFIG)
if(Snappy_FOUND)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy)
list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Snappy)
endif()
else()
set(AVRO_VENDORED FALSE)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro)
list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Avro)
endif()

set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES
${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
set(AVRO_VENDORED
${AVRO_VENDORED}
PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------
# Nanoarrow
# Nanoarrow (core)

# It is also possible to vendor nanoarrow using the bundled source code.
function(resolve_nanoarrow_dependency)
Expand Down Expand Up @@ -321,7 +324,7 @@ function(resolve_nanoarrow_dependency)
endfunction()

# ----------------------------------------------------------------------
# CRoaring
# CRoaring (core)

function(resolve_croaring_dependency)
prepare_fetchcontent()
Expand Down Expand Up @@ -375,7 +378,7 @@ function(resolve_croaring_dependency)
endfunction()

# ----------------------------------------------------------------------
# nlohmann-json
# nlohmann-json (core)

function(resolve_nlohmann_json_dependency)
prepare_fetchcontent()
Expand Down Expand Up @@ -437,7 +440,7 @@ function(resolve_nlohmann_json_dependency)
endfunction()

# ----------------------------------------------------------------------
# zlib
# zlib (core; lowest layer that needs it -- Arrow/Avro also pull it in)

function(resolve_zlib_dependency)
# use system zlib, zlib is required by arrow and avro
Expand All @@ -454,7 +457,7 @@ function(resolve_zlib_dependency)
endfunction()

# ----------------------------------------------------------------------
# cpr (C++ Requests)
# cpr (REST-only)

function(resolve_cpr_dependency)
prepare_fetchcontent()
Expand Down Expand Up @@ -493,34 +496,34 @@ function(resolve_cpr_dependency)
POSITION_INDEPENDENT_CODE ON)
add_library(iceberg::cpr ALIAS cpr)
install(TARGETS cpr
EXPORT iceberg_targets
EXPORT iceberg_rest_targets
RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL)
list(APPEND ICEBERG_REST_SYSTEM_DEPENDENCIES OpenSSL CURL)
else()
set(CPR_VENDORED FALSE)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES cpr)
list(APPEND ICEBERG_REST_SYSTEM_DEPENDENCIES cpr)
endif()

set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
set(ICEBERG_REST_SYSTEM_DEPENDENCIES
${ICEBERG_REST_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
set(CPR_VENDORED
${CPR_VENDORED}
PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------
# Zstd
# Zstd (bundle-only; pulled in via Arrow/Parquet)

function(resolve_zstd_dependency)
find_package(zstd CONFIG)
if(zstd_FOUND)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES zstd)
list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES zstd)
message(STATUS "Found zstd, version: ${zstd_VERSION}")
set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES
${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
endif()
endfunction()
Expand Down
6 changes: 3 additions & 3 deletions example/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ project(example)

set(CMAKE_CXX_STANDARD 23)

find_package(iceberg CONFIG REQUIRED)
# Demo uses iceberg_bundle (Arrow/Avro/Parquet; pulls core + data).
find_package(iceberg_bundle CONFIG REQUIRED)

add_executable(demo_example demo_example.cc)

target_link_libraries(demo_example PRIVATE iceberg::iceberg_bundle_static
iceberg::iceberg_rest_static)
target_link_libraries(demo_example PRIVATE iceberg::iceberg_bundle_static)
Loading
Loading