From 6b85b4f96a4fe15651cfd4e81f8a55459655844d Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sat, 25 Apr 2026 19:00:06 +0800 Subject: [PATCH] feat: add iceberg_data library alongside iceberg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move data writers, deletes/, and puffin/ into a separate `iceberg_data` library that links the existing `iceberg` target. `delete_file_index` stays in `iceberg` because manifest_group embeds DeleteFileIndex::Builder with only core dependencies. * `iceberg` — unchanged target name for metadata/planning, expressions, manifests, catalog (incl. in-memory), utilities, file I/O abstractions, and delete_file_index. * `iceberg_data` — data/, deletes/, puffin/; links `iceberg`. `iceberg_bundle` links `iceberg_data` when the bundle is built. `iceberg_rest` links `iceberg` and cpr only. CMake: per-package `iceberg_install_cmake_package` installs `iceberg-config.cmake`, `iceberg_data-config.cmake`, optional `iceberg_bundle-config.cmake`, and matching `*-targets.cmake` under `lib/cmake//`; install namespace remains `iceberg::`. Split `find_dependency` lists so `iceberg` consumers do not pull Arrow/Parquet/Avro/cpr. Vendored third-party exports are attached to the package that owns them (core vs bundle vs rest). Add `iceberg_rest-config.cmake.in` for the REST catalog package. Meson and tests link the split targets; the example uses `find_package(iceberg_bundle)` and `iceberg::iceberg_bundle_static`. Symbol visibility: `iceberg_export.h` documents `ICEBERG_EXPORT` for the main `iceberg` DLL; `iceberg_data_export.h` adds `ICEBERG_DATA_EXPORT` for data-layer public APIs. Shared builds define `${UPPER_LIB_NAME}_EXPORTING` (e.g. ICEBERG_DATA_EXPORTING) for MSVC. Types used across the boundary (e.g. WriterProperties, Arrow guards) remain exported from `iceberg`. --- cmake_modules/IcebergBuildUtils.cmake | 13 ++- .../IcebergThirdpartyToolchain.cmake | 63 +++++++------- example/CMakeLists.txt | 6 +- src/iceberg/CMakeLists.txt | 79 +++++++++++++---- src/iceberg/arrow_c_data_guard_internal.h | 12 ++- src/iceberg/catalog/rest/CMakeLists.txt | 3 + .../catalog/rest/iceberg_rest-config.cmake.in | 61 +++++++++++++ src/iceberg/catalog/rest/meson.build | 3 +- src/iceberg/data/data_writer.h | 6 +- src/iceberg/data/delete_loader.h | 4 +- src/iceberg/data/equality_delete_writer.h | 6 +- src/iceberg/data/position_delete_writer.h | 6 +- src/iceberg/data/writer.h | 6 +- src/iceberg/deletes/position_delete_index.h | 4 +- src/iceberg/deletes/roaring_position_bitmap.h | 4 +- src/iceberg/file_writer.h | 2 +- src/iceberg/iceberg-config.cmake.in | 51 +---------- src/iceberg/iceberg_bundle-config.cmake.in | 86 +++++++++++++++++++ src/iceberg/iceberg_data-config.cmake.in | 36 ++++++++ src/iceberg/iceberg_data_export.h | 50 +++++++++++ src/iceberg/iceberg_export.h | 32 ++++--- src/iceberg/meson.build | 66 +++++++++++--- src/iceberg/puffin/file_metadata.h | 20 ++--- src/iceberg/puffin/json_serde_internal.h | 16 ++-- src/iceberg/puffin/puffin_format.h | 8 +- src/iceberg/test/CMakeLists.txt | 26 ++++-- src/iceberg/test/meson.build | 4 +- src/iceberg/util/struct_like_set.cc | 4 +- 28 files changed, 497 insertions(+), 180 deletions(-) create mode 100644 src/iceberg/catalog/rest/iceberg_rest-config.cmake.in create mode 100644 src/iceberg/iceberg_bundle-config.cmake.in create mode 100644 src/iceberg/iceberg_data-config.cmake.in create mode 100644 src/iceberg/iceberg_data_export.h diff --git a/cmake_modules/IcebergBuildUtils.cmake b/cmake_modules/IcebergBuildUtils.cmake index 74b029708..2d6b09379 100644 --- a/cmake_modules/IcebergBuildUtils.cmake +++ b/cmake_modules/IcebergBuildUtils.cmake @@ -20,7 +20,8 @@ include(CMakePackageConfigHelpers) -function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME) +function(iceberg_install_cmake_package PACKAGE_NAME) + set(EXPORT_NAME "${PACKAGE_NAME}_targets") set(CONFIG_CMAKE "${PACKAGE_NAME}-config.cmake") set(BUILT_CONFIG_CMAKE "${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_CMAKE}") configure_package_config_file("${CONFIG_CMAKE}.in" "${BUILT_CONFIG_CMAKE}" @@ -33,9 +34,11 @@ function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME) install(FILES "${BUILT_CONFIG_CMAKE}" "${BUILT_CONFIG_VERSION_CMAKE}" DESTINATION "${ICEBERG_INSTALL_CMAKEDIR}/${PACKAGE_NAME}") set(TARGETS_CMAKE "${PACKAGE_NAME}-targets.cmake") + # Use iceberg:: for every export (not ${PACKAGE_NAME}::) so install INTERFACE + # link lines like iceberg::iceberg_data_static resolve across packages. install(EXPORT ${EXPORT_NAME} DESTINATION "${ICEBERG_INSTALL_CMAKEDIR}/${PACKAGE_NAME}" - NAMESPACE "${PACKAGE_NAME}::" + NAMESPACE "iceberg::" FILE "${TARGETS_CMAKE}") endfunction() @@ -73,6 +76,8 @@ function(add_iceberg_lib LIB_NAME) set(${ARG_OUTPUTS}) endif() + set(EXPORT_NAME "${LIB_NAME}_targets") + # Allow overriding ICEBERG_BUILD_SHARED and ICEBERG_BUILD_STATIC if(DEFINED ARG_BUILD_SHARED) set(BUILD_SHARED ${ARG_BUILD_SHARED}) @@ -162,7 +167,7 @@ function(add_iceberg_lib LIB_NAME) endif() install(TARGETS ${LIB_NAME}_shared - EXPORT iceberg_targets + EXPORT ${EXPORT_NAME} ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR} LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR} RUNTIME DESTINATION ${INSTALL_RUNTIME_DIR} @@ -229,7 +234,7 @@ function(add_iceberg_lib LIB_NAME) endif() install(TARGETS ${LIB_NAME}_static - EXPORT iceberg_targets + EXPORT ${EXPORT_NAME} ARCHIVE DESTINATION ${INSTALL_ARCHIVE_DIR} LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR} RUNTIME DESTINATION ${INSTALL_RUNTIME_DIR} diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index a0d418e05..9974c89c7 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -# Accumulate all dependencies to provide suitable static link parameters to the -# third party libraries. +# Per-package lists for *-config.cmake.in (iceberg avoids Arrow/Avro/cpr). set(ICEBERG_SYSTEM_DEPENDENCIES) +set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES) +set(ICEBERG_REST_SYSTEM_DEPENDENCIES) set(ICEBERG_ARROW_INSTALL_INTERFACE_LIBS) # ---------------------------------------------------------------------- @@ -92,7 +93,7 @@ macro(prepare_fetchcontent) endmacro() # ---------------------------------------------------------------------- -# Apache Arrow +# Apache Arrow (bundle-only) function(resolve_arrow_dependency) prepare_fetchcontent() @@ -149,7 +150,7 @@ function(resolve_arrow_dependency) set_target_properties(parquet_static PROPERTIES OUTPUT_NAME "iceberg_vendored_parquet") install(TARGETS arrow_static parquet_static - EXPORT iceberg_targets + EXPORT iceberg_bundle_targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") @@ -167,20 +168,22 @@ function(resolve_arrow_dependency) endif() # Arrow's exported static target interface may reference system libraries - # (e.g. OpenSSL, CURL, ZLIB) that consumers need to find. - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB) + # (e.g. OpenSSL, CURL, ZLIB) that consumers need to find. ZLIB is also a + # core dependency (see resolve_zlib_dependency()), so it is intentionally + # placed in the lowest layer; OpenSSL/CURL are bundle-only because they + # are pulled in by Arrow's S3 filesystem. if(ARROW_S3) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL) + list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES OpenSSL CURL) endif() else() set(ARROW_VENDORED FALSE) find_package(Arrow CONFIG REQUIRED) find_package(Parquet CONFIG REQUIRED) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet) + list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Arrow Parquet) endif() - set(ICEBERG_SYSTEM_DEPENDENCIES - ${ICEBERG_SYSTEM_DEPENDENCIES} + set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES + ${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES} PARENT_SCOPE) set(ARROW_VENDORED ${ARROW_VENDORED} @@ -188,7 +191,7 @@ function(resolve_arrow_dependency) endfunction() # ---------------------------------------------------------------------- -# Apache Avro +# Apache Avro (bundle-only) function(resolve_avro_dependency) prepare_fetchcontent() @@ -249,7 +252,7 @@ function(resolve_avro_dependency) set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp") set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON) install(TARGETS avrocpp_s - EXPORT iceberg_targets + EXPORT iceberg_bundle_targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") @@ -257,15 +260,15 @@ function(resolve_avro_dependency) # TODO: add vendored ZLIB and Snappy support find_package(Snappy CONFIG) if(Snappy_FOUND) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy) + list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Snappy) endif() else() set(AVRO_VENDORED FALSE) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro) + list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES Avro) endif() - set(ICEBERG_SYSTEM_DEPENDENCIES - ${ICEBERG_SYSTEM_DEPENDENCIES} + set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES + ${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES} PARENT_SCOPE) set(AVRO_VENDORED ${AVRO_VENDORED} @@ -273,7 +276,7 @@ function(resolve_avro_dependency) endfunction() # ---------------------------------------------------------------------- -# Nanoarrow +# Nanoarrow (core) # It is also possible to vendor nanoarrow using the bundled source code. function(resolve_nanoarrow_dependency) @@ -321,7 +324,7 @@ function(resolve_nanoarrow_dependency) endfunction() # ---------------------------------------------------------------------- -# CRoaring +# CRoaring (core) function(resolve_croaring_dependency) prepare_fetchcontent() @@ -375,7 +378,7 @@ function(resolve_croaring_dependency) endfunction() # ---------------------------------------------------------------------- -# nlohmann-json +# nlohmann-json (core) function(resolve_nlohmann_json_dependency) prepare_fetchcontent() @@ -437,7 +440,7 @@ function(resolve_nlohmann_json_dependency) endfunction() # ---------------------------------------------------------------------- -# zlib +# zlib (core; lowest layer that needs it -- Arrow/Avro also pull it in) function(resolve_zlib_dependency) # use system zlib, zlib is required by arrow and avro @@ -454,7 +457,7 @@ function(resolve_zlib_dependency) endfunction() # ---------------------------------------------------------------------- -# cpr (C++ Requests) +# cpr (REST-only) function(resolve_cpr_dependency) prepare_fetchcontent() @@ -493,18 +496,18 @@ function(resolve_cpr_dependency) POSITION_INDEPENDENT_CODE ON) add_library(iceberg::cpr ALIAS cpr) install(TARGETS cpr - EXPORT iceberg_targets + EXPORT iceberg_rest_targets RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL) + list(APPEND ICEBERG_REST_SYSTEM_DEPENDENCIES OpenSSL CURL) else() set(CPR_VENDORED FALSE) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES cpr) + list(APPEND ICEBERG_REST_SYSTEM_DEPENDENCIES cpr) endif() - set(ICEBERG_SYSTEM_DEPENDENCIES - ${ICEBERG_SYSTEM_DEPENDENCIES} + set(ICEBERG_REST_SYSTEM_DEPENDENCIES + ${ICEBERG_REST_SYSTEM_DEPENDENCIES} PARENT_SCOPE) set(CPR_VENDORED ${CPR_VENDORED} @@ -512,15 +515,15 @@ function(resolve_cpr_dependency) endfunction() # ---------------------------------------------------------------------- -# Zstd +# Zstd (bundle-only; pulled in via Arrow/Parquet) function(resolve_zstd_dependency) find_package(zstd CONFIG) if(zstd_FOUND) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES zstd) + list(APPEND ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES zstd) message(STATUS "Found zstd, version: ${zstd_VERSION}") - set(ICEBERG_SYSTEM_DEPENDENCIES - ${ICEBERG_SYSTEM_DEPENDENCIES} + set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES + ${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES} PARENT_SCOPE) endif() endfunction() diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 837ed7d7c..9276fed3f 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -22,9 +22,9 @@ project(example) set(CMAKE_CXX_STANDARD 23) -find_package(iceberg CONFIG REQUIRED) +# Demo uses iceberg_bundle (Arrow/Avro/Parquet; pulls core + data). +find_package(iceberg_bundle CONFIG REQUIRED) add_executable(demo_example demo_example.cc) -target_link_libraries(demo_example PRIVATE iceberg::iceberg_bundle_static - iceberg::iceberg_rest_static) +target_link_libraries(demo_example PRIVATE iceberg::iceberg_bundle_static) diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 603c35343..154f0f8bb 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -17,17 +17,13 @@ set(ICEBERG_INCLUDES "$" "$") + +# `iceberg`: metadata / planning. delete_file_index stays here because +# manifest_group embeds DeleteFileIndex::Builder (no iceberg_data deps). set(ICEBERG_SOURCES arrow_c_data_guard_internal.cc catalog/memory/in_memory_catalog.cc - data/data_writer.cc - data/delete_loader.cc - data/equality_delete_writer.cc - data/position_delete_writer.cc - data/writer.cc delete_file_index.cc - deletes/roaring_position_bitmap.cc - deletes/position_delete_index.cc expression/aggregate.cc expression/binder.cc expression/evaluator.cc @@ -66,9 +62,6 @@ set(ICEBERG_SOURCES partition_field.cc partition_spec.cc partition_summary.cc - puffin/file_metadata.cc - puffin/puffin_format.cc - puffin/json_serde.cc row/arrow_array_wrapper.cc row/manifest_wrapper.cc row/partition_values.cc @@ -125,6 +118,19 @@ set(ICEBERG_SOURCES util/url_encoder.cc util/uuid.cc) +# iceberg_data: writers, deletes, Puffin (split library). +set(ICEBERG_DATA_SOURCES + data/data_writer.cc + data/delete_loader.cc + data/equality_delete_writer.cc + data/position_delete_writer.cc + data/writer.cc + deletes/position_delete_index.cc + deletes/roaring_position_bitmap.cc + puffin/file_metadata.cc + puffin/json_serde.cc + puffin/puffin_format.cc) + set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_STATIC_INSTALL_INTERFACE_LIBS) @@ -169,6 +175,42 @@ add_iceberg_lib(iceberg OUTPUTS ICEBERG_LIBRARIES) +# iceberg_data links iceberg. +set(ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS) +set(ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS) +set(ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS) +set(ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS) + +list(APPEND ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>") +list(APPEND ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>") +list(APPEND + ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS + "$,iceberg::iceberg_static,iceberg::iceberg_shared>" +) +list(APPEND + ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS + "$,iceberg::iceberg_shared,iceberg::iceberg_static>" +) + +add_iceberg_lib(iceberg_data + SOURCES + ${ICEBERG_DATA_SOURCES} + EXTRA_INCLUDES + ${ICEBERG_INCLUDES} + SHARED_LINK_LIBS + ${ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS} + STATIC_LINK_LIBS + ${ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS} + STATIC_INSTALL_INTERFACE_LIBS + ${ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_INSTALL_INTERFACE_LIBS + ${ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS} + OUTPUTS + ICEBERG_DATA_LIBRARIES) + +# Top-level public headers; subdirs install their own via add_subdirectory. iceberg_install_all_headers(iceberg) add_subdirectory(catalog) @@ -202,7 +244,8 @@ if(ICEBERG_BUILD_BUNDLE) parquet/parquet_schema_util.cc parquet/parquet_writer.cc) - # Libraries to link with exported libiceberg_bundle.{so,a}. + # Bundle links iceberg_data so one find_package(iceberg_bundle) pulls writers + # and Puffin; adapters only include main iceberg headers at compile time. set(ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS) @@ -210,14 +253,14 @@ if(ICEBERG_BUILD_BUNDLE) list(APPEND ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS - "$,iceberg_static,iceberg_shared>" + "$,iceberg_data_static,iceberg_data_shared>" "$,Arrow::arrow_static,Arrow::arrow_shared>" "$,Parquet::parquet_static,Parquet::parquet_shared>" "$,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>" ) list(APPEND ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS - "$,iceberg_shared,iceberg_static>" + "$,iceberg_data_shared,iceberg_data_static>" "$,Arrow::arrow_shared,Arrow::arrow_static>" "$,Parquet::parquet_shared,Parquet::parquet_static>" "$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>" @@ -225,14 +268,14 @@ if(ICEBERG_BUILD_BUNDLE) list(APPEND ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_static,iceberg::iceberg_shared>" + "$,iceberg::iceberg_data_static,iceberg::iceberg_data_shared>" "$,iceberg::arrow_static,$,Arrow::arrow_static,Arrow::arrow_shared>>" "$,iceberg::parquet_static,$,Parquet::parquet_static,Parquet::parquet_shared>>" "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>>" ) list(APPEND ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_shared,iceberg::iceberg_static>" + "$,iceberg::iceberg_data_shared,iceberg::iceberg_data_static>" "$,iceberg::arrow_static,$,Arrow::arrow_shared,Arrow::arrow_static>>" "$,iceberg::parquet_static,$,Parquet::parquet_shared,Parquet::parquet_static>>" "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>" @@ -269,7 +312,11 @@ if(ICEBERG_BUILD_BUNDLE) add_subdirectory(parquet) endif() -iceberg_install_cmake_package(iceberg iceberg_targets) +iceberg_install_cmake_package(iceberg) +iceberg_install_cmake_package(iceberg_data) +if(ICEBERG_BUILD_BUNDLE) + iceberg_install_cmake_package(iceberg_bundle) +endif() if(ICEBERG_BUILD_TESTS) add_subdirectory(test) diff --git a/src/iceberg/arrow_c_data_guard_internal.h b/src/iceberg/arrow_c_data_guard_internal.h index 8bce14e57..cae9a5907 100644 --- a/src/iceberg/arrow_c_data_guard_internal.h +++ b/src/iceberg/arrow_c_data_guard_internal.h @@ -22,10 +22,14 @@ #include #include "iceberg/arrow_c_data.h" +#include "iceberg/iceberg_export.h" namespace iceberg::internal { -class ArrowArrayGuard { +// Used from iceberg_data; export dtors so they are visible in libiceberg +// with -fvisibility=hidden. + +class ICEBERG_EXPORT ArrowArrayGuard { public: explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {} ~ArrowArrayGuard(); @@ -34,7 +38,7 @@ class ArrowArrayGuard { ArrowArray* array_; }; -class ArrowSchemaGuard { +class ICEBERG_EXPORT ArrowSchemaGuard { public: explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {} ~ArrowSchemaGuard(); @@ -43,7 +47,7 @@ class ArrowSchemaGuard { ArrowSchema* schema_; }; -class ArrowArrayViewGuard { +class ICEBERG_EXPORT ArrowArrayViewGuard { public: explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {} ~ArrowArrayViewGuard(); @@ -52,7 +56,7 @@ class ArrowArrayViewGuard { ArrowArrayView* view_; }; -class ArrowArrayBufferGuard { +class ICEBERG_EXPORT ArrowArrayBufferGuard { public: explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {} ~ArrowArrayBufferGuard(); diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index b862bc869..f61eeddbe 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -39,6 +39,7 @@ set(ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS) +# REST client: iceberg + cpr only. list(APPEND ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS "$,iceberg_static,iceberg_shared>" cpr::cpr) list(APPEND ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS @@ -65,3 +66,5 @@ add_iceberg_lib(iceberg_rest ${ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS}) iceberg_install_all_headers(iceberg/catalog/rest) + +iceberg_install_cmake_package(iceberg_rest) diff --git a/src/iceberg/catalog/rest/iceberg_rest-config.cmake.in b/src/iceberg/catalog/rest/iceberg_rest-config.cmake.in new file mode 100644 index 000000000..b05a5b442 --- /dev/null +++ b/src/iceberg/catalog/rest/iceberg_rest-config.cmake.in @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This config sets the following variables in your project:: +# +# iceberg_rest_FOUND - true if iceberg_rest is found on the system +# iceberg_rest_VERSION - version of the found iceberg_rest +# +# This config sets the following targets (if built) in your project:: +# +# iceberg::iceberg_rest_shared +# iceberg::iceberg_rest_static + +@PACKAGE_INIT@ + +set(ICEBERG_REST_SYSTEM_DEPENDENCIES "@ICEBERG_REST_SYSTEM_DEPENDENCIES@") + +include(CMakeFindDependencyMacro) + +find_dependency(iceberg) + +macro(iceberg_rest_find_dependencies dependencies) + if(DEFINED CMAKE_MODULE_PATH) + set(ICEBERG_REST_CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) + else() + unset(ICEBERG_REST_CMAKE_MODULE_PATH_OLD) + endif() + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + + foreach(dependency ${dependencies}) + find_dependency(${dependency}) + endforeach() + + if(DEFINED ICEBERG_REST_CMAKE_MODULE_PATH_OLD) + set(CMAKE_MODULE_PATH ${ICEBERG_REST_CMAKE_MODULE_PATH_OLD}) + unset(ICEBERG_REST_CMAKE_MODULE_PATH_OLD) + else() + unset(CMAKE_MODULE_PATH) + endif() +endmacro() + +# cpr (+ OpenSSL/CURL when bundled). +iceberg_rest_find_dependencies("${ICEBERG_REST_SYSTEM_DEPENDENCIES}") + +include("${CMAKE_CURRENT_LIST_DIR}/iceberg_rest-targets.cmake") + +check_required_components(iceberg_rest) diff --git a/src/iceberg/catalog/rest/meson.build b/src/iceberg/catalog/rest/meson.build index a1f8ce973..54c74b674 100644 --- a/src/iceberg/catalog/rest/meson.build +++ b/src/iceberg/catalog/rest/meson.build @@ -40,6 +40,7 @@ cpr_needs_static = ( ) cpr_dep = dependency('cpr', static: cpr_needs_static) +# iceberg + cpr. iceberg_rest_build_deps = [iceberg_dep, cpr_dep] iceberg_rest_lib = library( 'iceberg_rest', @@ -59,7 +60,7 @@ iceberg_rest_dep = declare_dependency( dependencies: iceberg_rest_build_deps, compile_args: iceberg_rest_compile_args, ) -meson.override_dependency('iceberg-rest', iceberg_rest_dep) +meson.override_dependency('iceberg_rest', iceberg_rest_dep) pkg.generate(iceberg_rest_lib) install_headers( diff --git a/src/iceberg/data/data_writer.h b/src/iceberg/data/data_writer.h index 380c97e2e..9a2002f0d 100644 --- a/src/iceberg/data/data_writer.h +++ b/src/iceberg/data/data_writer.h @@ -31,7 +31,7 @@ #include "iceberg/arrow_c_data.h" #include "iceberg/data/writer.h" #include "iceberg/file_format.h" -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" #include "iceberg/row/partition_values.h" #include "iceberg/type_fwd.h" @@ -39,7 +39,7 @@ namespace iceberg { /// \brief Options for creating a DataWriter. -struct ICEBERG_EXPORT DataWriterOptions { +struct ICEBERG_DATA_EXPORT DataWriterOptions { std::string path; std::shared_ptr schema; std::shared_ptr spec; @@ -51,7 +51,7 @@ struct ICEBERG_EXPORT DataWriterOptions { }; /// \brief Writer for Iceberg data files. -class ICEBERG_EXPORT DataWriter : public FileWriter { +class ICEBERG_DATA_EXPORT DataWriter : public FileWriter { public: ~DataWriter() override; diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h index 0a1122f95..a422f0683 100644 --- a/src/iceberg/data/delete_loader.h +++ b/src/iceberg/data/delete_loader.h @@ -26,14 +26,14 @@ #include #include -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" namespace iceberg { /// \brief Loads delete files and constructs in-memory delete indexes. -class ICEBERG_EXPORT DeleteLoader { +class ICEBERG_DATA_EXPORT DeleteLoader { public: /// \brief Create a DeleteLoader. /// \param io FileIO instance for reading delete files diff --git a/src/iceberg/data/equality_delete_writer.h b/src/iceberg/data/equality_delete_writer.h index d1728a481..fc0083a81 100644 --- a/src/iceberg/data/equality_delete_writer.h +++ b/src/iceberg/data/equality_delete_writer.h @@ -32,7 +32,7 @@ #include "iceberg/arrow_c_data.h" #include "iceberg/data/writer.h" #include "iceberg/file_format.h" -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" #include "iceberg/row/partition_values.h" #include "iceberg/type_fwd.h" @@ -40,7 +40,7 @@ namespace iceberg { /// \brief Options for creating an EqualityDeleteWriter. -struct ICEBERG_EXPORT EqualityDeleteWriterOptions { +struct ICEBERG_DATA_EXPORT EqualityDeleteWriterOptions { std::string path; std::shared_ptr schema; std::shared_ptr spec; @@ -54,7 +54,7 @@ struct ICEBERG_EXPORT EqualityDeleteWriterOptions { }; /// \brief Writer for Iceberg equality delete files. -class ICEBERG_EXPORT EqualityDeleteWriter : public FileWriter { +class ICEBERG_DATA_EXPORT EqualityDeleteWriter : public FileWriter { public: ~EqualityDeleteWriter() override; diff --git a/src/iceberg/data/position_delete_writer.h b/src/iceberg/data/position_delete_writer.h index de7a2c3f3..c76e61720 100644 --- a/src/iceberg/data/position_delete_writer.h +++ b/src/iceberg/data/position_delete_writer.h @@ -31,7 +31,7 @@ #include "iceberg/arrow_c_data.h" #include "iceberg/data/writer.h" #include "iceberg/file_format.h" -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" #include "iceberg/row/partition_values.h" #include "iceberg/type_fwd.h" @@ -39,7 +39,7 @@ namespace iceberg { /// \brief Options for creating a PositionDeleteWriter. -struct ICEBERG_EXPORT PositionDeleteWriterOptions { +struct ICEBERG_DATA_EXPORT PositionDeleteWriterOptions { std::string path; std::shared_ptr schema; std::shared_ptr spec; @@ -51,7 +51,7 @@ struct ICEBERG_EXPORT PositionDeleteWriterOptions { }; /// \brief Writer for Iceberg position delete files. -class ICEBERG_EXPORT PositionDeleteWriter : public FileWriter { +class ICEBERG_DATA_EXPORT PositionDeleteWriter : public FileWriter { public: ~PositionDeleteWriter() override; diff --git a/src/iceberg/data/writer.h b/src/iceberg/data/writer.h index 82c1d0cdc..a6f5f063d 100644 --- a/src/iceberg/data/writer.h +++ b/src/iceberg/data/writer.h @@ -27,14 +27,14 @@ #include #include "iceberg/arrow_c_data.h" -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" namespace iceberg { /// \brief Base interface for data file writers. -class ICEBERG_EXPORT FileWriter { +class ICEBERG_DATA_EXPORT FileWriter { public: virtual ~FileWriter(); @@ -49,7 +49,7 @@ class ICEBERG_EXPORT FileWriter { virtual Status Close() = 0; /// \brief File metadata for all files produced by this writer. - struct ICEBERG_EXPORT WriteResult { + struct ICEBERG_DATA_EXPORT WriteResult { /// Usually a writer produces a single data or delete file. /// Position delete writer may produce multiple file-scoped delete files. /// In the future, multiple files can be produced if file rolling is supported. diff --git a/src/iceberg/deletes/position_delete_index.h b/src/iceberg/deletes/position_delete_index.h index 968d3ebed..5de82a591 100644 --- a/src/iceberg/deletes/position_delete_index.h +++ b/src/iceberg/deletes/position_delete_index.h @@ -26,7 +26,7 @@ #include #include "iceberg/deletes/roaring_position_bitmap.h" -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" namespace iceberg { @@ -35,7 +35,7 @@ namespace iceberg { /// This class provides a domain-specific API for position deletes /// in Iceberg MOR (merge-on-read) tables. Positions are 0-based /// row indices within a data file. -class ICEBERG_EXPORT PositionDeleteIndex { +class ICEBERG_DATA_EXPORT PositionDeleteIndex { public: PositionDeleteIndex() = default; ~PositionDeleteIndex() = default; diff --git a/src/iceberg/deletes/roaring_position_bitmap.h b/src/iceberg/deletes/roaring_position_bitmap.h index a6d7742be..7585a2c55 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.h +++ b/src/iceberg/deletes/roaring_position_bitmap.h @@ -28,7 +28,7 @@ #include #include -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" namespace iceberg { @@ -45,7 +45,7 @@ namespace iceberg { /// \note This class is used to represent deletion vectors. The Puffin puffin /// reader/write handle adding the additional required framing (length prefix, magic, /// magic bytes, CRC-32) for `deletion-vector-v1` persistence. -class ICEBERG_EXPORT RoaringPositionBitmap { +class ICEBERG_DATA_EXPORT RoaringPositionBitmap { public: /// \brief Maximum supported position (aligned with the Java implementation). static constexpr int64_t kMaxPosition = 0x7FFFFFFE80000000LL; diff --git a/src/iceberg/file_writer.h b/src/iceberg/file_writer.h index 0b1af7f93..f3352d8fd 100644 --- a/src/iceberg/file_writer.h +++ b/src/iceberg/file_writer.h @@ -36,7 +36,7 @@ namespace iceberg { -class WriterProperties : public ConfigBase { +class ICEBERG_EXPORT WriterProperties : public ConfigBase { public: template using Entry = const ConfigBase::Entry; diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in index 787fadcc6..afa72409f 100644 --- a/src/iceberg/iceberg-config.cmake.in +++ b/src/iceberg/iceberg-config.cmake.in @@ -17,21 +17,16 @@ # # This config sets the following variables in your project:: # -# Iceberg_FOUND - true if Iceberg found on the system -# Iceberg_VERSION - version of the found Iceberg +# iceberg_FOUND - true if the iceberg package is found +# iceberg_VERSION - version of the found package # # This config sets the following targets (if built) in your project:: # # iceberg::iceberg_shared # iceberg::iceberg_static -# iceberg::iceberg_bundle_shared -# iceberg::iceberg_bundle_static -# iceberg::iceberg_rest_shared -# iceberg::iceberg_rest_static @PACKAGE_INIT@ -set(ICEBERG_BUILD_STATIC "@ICEBERG_BUILD_STATIC@") set(ICEBERG_SYSTEM_DEPENDENCIES "@ICEBERG_SYSTEM_DEPENDENCIES@") include(CMakeFindDependencyMacro) @@ -56,50 +51,8 @@ macro(iceberg_find_dependencies dependencies) endif() endmacro() -macro(iceberg_find_components components) - foreach(comp ${components}) - string(TOLOWER "${comp}" _comp_lower_case) - if(TARGET "iceberg::iceberg_${_comp_lower_case}_shared" OR - TARGET "iceberg::iceberg_${_comp_lower_case}_static") - set(iceberg_${comp}_FOUND TRUE) - else() - set(iceberg_${comp}_FOUND FALSE) - set(iceberg_NOT_FOUND_MESSAGE "Component ${comp} was not installed") - endif() - endforeach() -endmacro() - -# Find system dependencies iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}") include("${CMAKE_CURRENT_LIST_DIR}/iceberg-targets.cmake") -if(TARGET iceberg::arrow_static) - add_library(Arrow::arrow_static ALIAS iceberg::arrow_static) - - add_library(Arrow::arrow_bundled_dependencies STATIC IMPORTED) - get_target_property(arrow_static_configurations iceberg::arrow_static - IMPORTED_CONFIGURATIONS) - foreach(CONFIGURATION ${arrow_static_configurations}) - string(TOUPPER "${CONFIGURATION}" CONFIGURATION) - get_target_property(arrow_static_location iceberg::arrow_static - LOCATION_${CONFIGURATION}) - get_filename_component(arrow_lib_dir "${arrow_static_location}" DIRECTORY) - set_property(TARGET Arrow::arrow_bundled_dependencies - APPEND - PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION}) - set_target_properties(Arrow::arrow_bundled_dependencies - PROPERTIES IMPORTED_LOCATION_${CONFIGURATION} - "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - endforeach() -endif() - -if(TARGET iceberg::parquet_static) - add_library(Parquet::parquet_static ALIAS iceberg::parquet_static) -endif() - -# Find required components -iceberg_find_components("${iceberg_FIND_COMPONENTS}") - check_required_components(iceberg) diff --git a/src/iceberg/iceberg_bundle-config.cmake.in b/src/iceberg/iceberg_bundle-config.cmake.in new file mode 100644 index 000000000..d210b228b --- /dev/null +++ b/src/iceberg/iceberg_bundle-config.cmake.in @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This config sets the following variables in your project:: +# +# iceberg_bundle_FOUND - true if iceberg_bundle is found on the system +# iceberg_bundle_VERSION - version of the found iceberg_bundle +# +# This config sets the following targets (if built) in your project:: +# +# iceberg::iceberg_bundle_shared +# iceberg::iceberg_bundle_static + +@PACKAGE_INIT@ + +set(ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES "@ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES@") + +include(CMakeFindDependencyMacro) + +find_dependency(iceberg_data) + +macro(iceberg_bundle_find_dependencies dependencies) + if(DEFINED CMAKE_MODULE_PATH) + set(ICEBERG_BUNDLE_CMAKE_MODULE_PATH_OLD ${CMAKE_MODULE_PATH}) + else() + unset(ICEBERG_BUNDLE_CMAKE_MODULE_PATH_OLD) + endif() + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + + foreach(dependency ${dependencies}) + find_dependency(${dependency}) + endforeach() + + if(DEFINED ICEBERG_BUNDLE_CMAKE_MODULE_PATH_OLD) + set(CMAKE_MODULE_PATH ${ICEBERG_BUNDLE_CMAKE_MODULE_PATH_OLD}) + unset(ICEBERG_BUNDLE_CMAKE_MODULE_PATH_OLD) + else() + unset(CMAKE_MODULE_PATH) + endif() +endmacro() + +# Arrow / Parquet / Avro / … (not required for iceberg or iceberg_data alone). +iceberg_bundle_find_dependencies("${ICEBERG_BUNDLE_SYSTEM_DEPENDENCIES}") + +include("${CMAKE_CURRENT_LIST_DIR}/iceberg_bundle-targets.cmake") + +if(TARGET iceberg::arrow_static) + add_library(Arrow::arrow_static ALIAS iceberg::arrow_static) + + add_library(Arrow::arrow_bundled_dependencies STATIC IMPORTED) + get_target_property(arrow_static_configurations iceberg::arrow_static + IMPORTED_CONFIGURATIONS) + foreach(CONFIGURATION ${arrow_static_configurations}) + string(TOUPPER "${CONFIGURATION}" CONFIGURATION) + get_target_property(arrow_static_location iceberg::arrow_static + LOCATION_${CONFIGURATION}) + get_filename_component(arrow_lib_dir "${arrow_static_location}" DIRECTORY) + set_property(TARGET Arrow::arrow_bundled_dependencies + APPEND + PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION}) + set_target_properties(Arrow::arrow_bundled_dependencies + PROPERTIES IMPORTED_LOCATION_${CONFIGURATION} + "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endforeach() +endif() + +if(TARGET iceberg::parquet_static) + add_library(Parquet::parquet_static ALIAS iceberg::parquet_static) +endif() + +check_required_components(iceberg_bundle) diff --git a/src/iceberg/iceberg_data-config.cmake.in b/src/iceberg/iceberg_data-config.cmake.in new file mode 100644 index 000000000..74deea624 --- /dev/null +++ b/src/iceberg/iceberg_data-config.cmake.in @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This config sets the following variables in your project:: +# +# iceberg_data_FOUND - true if the iceberg_data package is found +# iceberg_data_VERSION - version of the found package +# +# This config sets the following targets (if built) in your project:: +# +# iceberg::iceberg_data_shared +# iceberg::iceberg_data_static + +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +find_dependency(iceberg) + +include("${CMAKE_CURRENT_LIST_DIR}/iceberg_data-targets.cmake") + +check_required_components(iceberg_data) diff --git a/src/iceberg/iceberg_data_export.h b/src/iceberg/iceberg_data_export.h new file mode 100644 index 000000000..29743dd28 --- /dev/null +++ b/src/iceberg/iceberg_data_export.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +// Export macros for iceberg_data (data/, deletes/, puffin/ public API). + +#if defined(_WIN32) || defined(__CYGWIN__) +# ifdef ICEBERG_DATA_STATIC +# define ICEBERG_DATA_EXPORT +# elif defined(ICEBERG_DATA_EXPORTING) +# define ICEBERG_DATA_EXPORT __declspec(dllexport) +# else +# define ICEBERG_DATA_EXPORT __declspec(dllimport) +# endif + +# if defined(_MSC_VER) +# define ICEBERG_DATA_TEMPLATE_CLASS_EXPORT +# else +# define ICEBERG_DATA_TEMPLATE_CLASS_EXPORT ICEBERG_DATA_EXPORT +# endif + +# if defined(_MSC_VER) && defined(ICEBERG_DATA_EXPORTING) && \ + !defined(ICEBERG_DATA_STATIC) +# define ICEBERG_DATA_EXTERN_TEMPLATE_CLASS_EXPORT +# else +# define ICEBERG_DATA_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_DATA_EXPORT +# endif + +#else // Non-Windows +# define ICEBERG_DATA_EXPORT __attribute__((visibility("default"))) +# define ICEBERG_DATA_TEMPLATE_CLASS_EXPORT ICEBERG_DATA_EXPORT +# define ICEBERG_DATA_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_DATA_EXPORT +#endif diff --git a/src/iceberg/iceberg_export.h b/src/iceberg/iceberg_export.h index 8d7a35f27..cf7b570da 100644 --- a/src/iceberg/iceberg_export.h +++ b/src/iceberg/iceberg_export.h @@ -19,6 +19,10 @@ #pragma once +// Export macros for the main `iceberg` library (DLL). iceberg_data uses +// ICEBERG_DATA_* in iceberg_data_export.h — do not use one macro for both DLLs +// on MSVC. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_STATIC # define ICEBERG_EXPORT @@ -28,30 +32,32 @@ # define ICEBERG_EXPORT __declspec(dllimport) # endif -# define ICEBERG_TEMPLATE_EXPORT ICEBERG_EXPORT - -// For template class declarations. Empty on MSVC: dllexport on a class template -// declaration combined with extern template triggers C4910. +// dllexport on a class template declaration combined with extern template +// triggers MSVC C4910, so leave the class-template-decl macro empty there. # if defined(_MSC_VER) # define ICEBERG_TEMPLATE_CLASS_EXPORT # else # define ICEBERG_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT # endif -// For extern template declarations. Empty when building the DLL on MSVC: -// `extern` + `dllexport` is contradictory and triggers C4910. +// `extern template` + `dllexport` is contradictory on MSVC (also C4910). # if defined(_MSC_VER) && defined(ICEBERG_EXPORTING) && !defined(ICEBERG_STATIC) # define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT # else -# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_EXPORT +# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT # endif -#else // Non-Windows -# ifndef ICEBERG_EXPORT -# define ICEBERG_EXPORT __attribute__((visibility("default"))) -# endif +// Explicit template instantiation definitions in .cc files must be exported on +// MSVC so dependent DLLs can link imported symbols. +# define ICEBERG_TEMPLATE_INSTANTIATION_EXPORT ICEBERG_EXPORT -# define ICEBERG_TEMPLATE_EXPORT +#else // Non-Windows +# define ICEBERG_EXPORT __attribute__((visibility("default"))) +// Default visibility for explicit template instantiations (hidden inlines +// otherwise; needed when iceberg_data links iceberg). # define ICEBERG_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT -# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_TEMPLATE_EXPORT +# define ICEBERG_EXTERN_TEMPLATE_CLASS_EXPORT ICEBERG_EXPORT +// GCC/Clang can warn when attributes appear on explicit template +// instantiation definitions after the class is already defined. +# define ICEBERG_TEMPLATE_INSTANTIATION_EXPORT #endif diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 5e68def98..9029a7f47 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -39,17 +39,12 @@ configure_file( ) iceberg_include_dir = include_directories('..') + +# `iceberg`: metadata / planning / model layer. iceberg_sources = files( 'arrow_c_data_guard_internal.cc', 'catalog/memory/in_memory_catalog.cc', - 'data/data_writer.cc', - 'data/delete_loader.cc', - 'data/equality_delete_writer.cc', - 'data/position_delete_writer.cc', - 'data/writer.cc', 'delete_file_index.cc', - 'deletes/position_delete_index.cc', - 'deletes/roaring_position_bitmap.cc', 'expression/aggregate.cc', 'expression/binder.cc', 'expression/evaluator.cc', @@ -88,9 +83,6 @@ iceberg_sources = files( 'partition_field.cc', 'partition_spec.cc', 'partition_summary.cc', - 'puffin/file_metadata.cc', - 'puffin/json_serde.cc', - 'puffin/puffin_format.cc', 'row/arrow_array_wrapper.cc', 'row/manifest_wrapper.cc', 'row/partition_values.cc', @@ -148,6 +140,20 @@ iceberg_sources = files( 'util/uuid.cc', ) +# iceberg_data: data-file / delete / Puffin / execution layer. +iceberg_data_sources = files( + 'data/data_writer.cc', + 'data/delete_loader.cc', + 'data/equality_delete_writer.cc', + 'data/position_delete_writer.cc', + 'data/writer.cc', + 'deletes/position_delete_index.cc', + 'deletes/roaring_position_bitmap.cc', + 'puffin/file_metadata.cc', + 'puffin/json_serde.cc', + 'puffin/puffin_format.cc', +) + # CRoaring does not export symbols, so on Windows it must # be used as a static lib croaring_needs_static = ( @@ -173,7 +179,7 @@ iceberg_lib = library( ) iceberg_interface_args = [] -if get_option('default_library') != 'shared' +if get_option('default_library') == 'static' iceberg_interface_args += ['-DICEBERG_STATIC'] endif @@ -187,6 +193,32 @@ meson.override_dependency('iceberg', iceberg_dep) pkg = import('pkgconfig') pkg.generate(iceberg_lib) +# iceberg_data -> iceberg (declare_dependency below). +iceberg_data_lib = library( + 'iceberg_data', + sources: iceberg_data_sources, + dependencies: [iceberg_dep], + include_directories: iceberg_include_dir, + install: true, + gnu_symbol_visibility: 'inlineshidden', + cpp_shared_args: ['-DICEBERG_DATA_EXPORTING'], + cpp_static_args: ['-DICEBERG_DATA_STATIC'], +) + +iceberg_data_interface_args = [] +if get_option('default_library') == 'static' + iceberg_data_interface_args += ['-DICEBERG_DATA_STATIC'] +endif + +iceberg_data_dep = declare_dependency( + link_with: iceberg_data_lib, + dependencies: [iceberg_dep], + include_directories: iceberg_include_dir, + compile_args: iceberg_data_interface_args, +) +meson.override_dependency('iceberg_data', iceberg_data_dep) +pkg.generate(iceberg_data_lib) + install_headers( [ 'arrow_c_data.h', @@ -199,6 +231,7 @@ install_headers( 'file_io_registry.h', 'file_reader.h', 'file_writer.h', + 'iceberg_data_export.h', 'iceberg_export.h', 'inheritable_metadata.h', 'location_provider.h', @@ -232,6 +265,17 @@ install_headers( subdir: 'iceberg', ) +install_headers( + [ + 'data/data_writer.h', + 'data/delete_loader.h', + 'data/equality_delete_writer.h', + 'data/position_delete_writer.h', + 'data/writer.h', + ], + subdir: 'iceberg/data', +) + subdir('catalog') subdir('deletes') subdir('expression') diff --git a/src/iceberg/puffin/file_metadata.h b/src/iceberg/puffin/file_metadata.h index 17ddad77d..14eaae6c6 100644 --- a/src/iceberg/puffin/file_metadata.h +++ b/src/iceberg/puffin/file_metadata.h @@ -29,7 +29,7 @@ #include #include -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" namespace iceberg::puffin { @@ -41,12 +41,12 @@ enum class PuffinCompressionCodec { kZstd, }; -ICEBERG_EXPORT std::string_view CodecName(PuffinCompressionCodec codec); +ICEBERG_DATA_EXPORT std::string_view CodecName(PuffinCompressionCodec codec); -ICEBERG_EXPORT Result PuffinCompressionCodecFromName( +ICEBERG_DATA_EXPORT Result PuffinCompressionCodecFromName( std::string_view codec_name); -ICEBERG_EXPORT std::string ToString(PuffinCompressionCodec codec); +ICEBERG_DATA_EXPORT std::string ToString(PuffinCompressionCodec codec); /// \brief Standard blob types defined by the Iceberg specification. struct StandardBlobTypes { @@ -67,7 +67,7 @@ struct StandardPuffinProperties { }; /// \brief A blob in a Puffin file. -struct ICEBERG_EXPORT Blob { +struct ICEBERG_DATA_EXPORT Blob { /// See StandardBlobTypes for known types. std::string type; /// Ordered list of field IDs the blob was computed from. @@ -84,10 +84,10 @@ struct ICEBERG_EXPORT Blob { friend bool operator==(const Blob& lhs, const Blob& rhs) = default; }; -ICEBERG_EXPORT std::string ToString(const Blob& blob); +ICEBERG_DATA_EXPORT std::string ToString(const Blob& blob); /// \brief Metadata about a blob stored in a Puffin file footer. -struct ICEBERG_EXPORT BlobMetadata { +struct ICEBERG_DATA_EXPORT BlobMetadata { /// See StandardBlobTypes for known types. std::string type; /// Ordered list of field IDs the blob was computed from. @@ -105,16 +105,16 @@ struct ICEBERG_EXPORT BlobMetadata { friend bool operator==(const BlobMetadata& lhs, const BlobMetadata& rhs) = default; }; -ICEBERG_EXPORT std::string ToString(const BlobMetadata& blob_metadata); +ICEBERG_DATA_EXPORT std::string ToString(const BlobMetadata& blob_metadata); /// \brief Metadata about a Puffin file. -struct ICEBERG_EXPORT FileMetadata { +struct ICEBERG_DATA_EXPORT FileMetadata { std::vector blobs; std::unordered_map properties; friend bool operator==(const FileMetadata& lhs, const FileMetadata& rhs) = default; }; -ICEBERG_EXPORT std::string ToString(const FileMetadata& file_metadata); +ICEBERG_DATA_EXPORT std::string ToString(const FileMetadata& file_metadata); } // namespace iceberg::puffin diff --git a/src/iceberg/puffin/json_serde_internal.h b/src/iceberg/puffin/json_serde_internal.h index dbb5fbd09..9e003e168 100644 --- a/src/iceberg/puffin/json_serde_internal.h +++ b/src/iceberg/puffin/json_serde_internal.h @@ -27,30 +27,30 @@ #include -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/puffin/type_fwd.h" #include "iceberg/result.h" namespace iceberg::puffin { /// \brief Serialize a BlobMetadata to JSON. -ICEBERG_EXPORT nlohmann::json ToJson(const BlobMetadata& blob_metadata); +ICEBERG_DATA_EXPORT nlohmann::json ToJson(const BlobMetadata& blob_metadata); /// \brief Deserialize a BlobMetadata from JSON. -ICEBERG_EXPORT Result BlobMetadataFromJson(const nlohmann::json& json); +ICEBERG_DATA_EXPORT Result BlobMetadataFromJson(const nlohmann::json& json); /// \brief Serialize a FileMetadata to JSON. -ICEBERG_EXPORT nlohmann::json ToJson(const FileMetadata& file_metadata); +ICEBERG_DATA_EXPORT nlohmann::json ToJson(const FileMetadata& file_metadata); /// \brief Deserialize a FileMetadata from JSON. -ICEBERG_EXPORT Result FileMetadataFromJson(const nlohmann::json& json); +ICEBERG_DATA_EXPORT Result FileMetadataFromJson(const nlohmann::json& json); /// \brief Serialize a FileMetadata to a JSON string. -ICEBERG_EXPORT std::string ToJsonString(const FileMetadata& file_metadata, - bool pretty = false); +ICEBERG_DATA_EXPORT std::string ToJsonString(const FileMetadata& file_metadata, + bool pretty = false); /// \brief Deserialize a FileMetadata from a JSON string. -ICEBERG_EXPORT Result FileMetadataFromJsonString( +ICEBERG_DATA_EXPORT Result FileMetadataFromJsonString( std::string_view json_string); } // namespace iceberg::puffin diff --git a/src/iceberg/puffin/puffin_format.h b/src/iceberg/puffin/puffin_format.h index 857c2ba57..e5ecf9003 100644 --- a/src/iceberg/puffin/puffin_format.h +++ b/src/iceberg/puffin/puffin_format.h @@ -26,14 +26,14 @@ #include #include -#include "iceberg/iceberg_export.h" +#include "iceberg/iceberg_data_export.h" #include "iceberg/puffin/file_metadata.h" #include "iceberg/result.h" namespace iceberg::puffin { /// \brief Puffin file format constants. -struct ICEBERG_EXPORT PuffinFormat { +struct ICEBERG_DATA_EXPORT PuffinFormat { /// Magic bytes: "PFA1" (Puffin Fratercula arctica, version 1) static constexpr std::array kMagicV1 = {0x50, 0x46, 0x41, 0x31}; @@ -61,9 +61,9 @@ enum class PuffinFlag : uint8_t { }; /// \brief Check if a flag is set in the flags bytes. -ICEBERG_EXPORT bool IsFlagSet(std::span flags, PuffinFlag flag); +ICEBERG_DATA_EXPORT bool IsFlagSet(std::span flags, PuffinFlag flag); /// \brief Set a flag in the flags bytes. -ICEBERG_EXPORT void SetFlag(std::span flags, PuffinFlag flag); +ICEBERG_DATA_EXPORT void SetFlag(std::span flags, PuffinFlag flag); } // namespace iceberg::puffin diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index b0f03b527..f03ffb56c 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -31,7 +31,7 @@ set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources") configure_file("test_config.h.in" "test_config.h") function(add_iceberg_test test_name) - set(options USE_BUNDLE) + set(options USE_BUNDLE USE_DATA) set(oneValueArgs) set(multiValueArgs SOURCES) cmake_parse_arguments(ARG @@ -46,10 +46,18 @@ function(add_iceberg_test test_name) target_sources(${test_name} PRIVATE ${ARG_SOURCES}) if(ARG_USE_BUNDLE) - target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gmock_main) + set(test_link_target + "$,iceberg_bundle_static,iceberg_bundle_shared>" + ) + elseif(ARG_USE_DATA) + set(test_link_target + "$,iceberg_data_static,iceberg_data_shared>" + ) else() - target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gmock_main) + set(test_link_target + "$,iceberg_static,iceberg_shared>") endif() + target_link_libraries(${test_name} PRIVATE ${test_link_target} GTest::gmock_main) if(MSVC_TOOLCHAIN) target_compile_options(${test_name} PRIVATE /bigobj) @@ -109,6 +117,7 @@ add_iceberg_test(json_serde_test schema_json_test.cc) add_iceberg_test(util_test + USE_DATA SOURCES bucket_util_test.cc config_test.cc @@ -130,7 +139,11 @@ add_iceberg_test(util_test add_iceberg_test(roaring_test SOURCES roaring_test.cc) -add_iceberg_test(puffin_test SOURCES puffin_format_test.cc puffin_json_test.cc) +add_iceberg_test(puffin_test + USE_DATA + SOURCES + puffin_format_test.cc + puffin_json_test.cc) if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(avro_test @@ -226,7 +239,10 @@ if(ICEBERG_BUILD_REST) add_executable(${test_name}) target_include_directories(${test_name} PRIVATE "${CMAKE_BINARY_DIR}/iceberg/test/") target_sources(${test_name} PRIVATE ${ARG_SOURCES}) - target_link_libraries(${test_name} PRIVATE GTest::gmock_main iceberg_rest_static) + set(rest_test_link_target + "$,iceberg_rest_static,iceberg_rest_shared>" + ) + target_link_libraries(${test_name} PRIVATE GTest::gmock_main ${rest_test_link_target}) if(MSVC_TOOLCHAIN) target_compile_options(${test_name} PRIVATE /bigobj) endif() diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index ed9a5866a..d271b2100 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -101,10 +101,12 @@ iceberg_tests = { 'uuid_test.cc', 'visit_type_test.cc', ), + 'iceberg_lib': iceberg_data_dep, }, 'roaring_test': {'sources': files('roaring_test.cc')}, 'puffin_test': { 'sources': files('puffin_format_test.cc', 'puffin_json_test.cc'), + 'iceberg_lib': iceberg_data_dep, }, } @@ -143,7 +145,7 @@ foreach test_name, values : iceberg_tests exc = executable( test_name, sources: values['sources'], - dependencies: [iceberg_dep, gmock_main_dep] + values.get( + dependencies: [values.get('iceberg_lib', iceberg_dep), gmock_main_dep] + values.get( 'dependencies', [], ), diff --git a/src/iceberg/util/struct_like_set.cc b/src/iceberg/util/struct_like_set.cc index cc3de5293..38f5ef021 100644 --- a/src/iceberg/util/struct_like_set.cc +++ b/src/iceberg/util/struct_like_set.cc @@ -618,7 +618,7 @@ bool StructLikeSet::KeyEqual::operator()( return StructLikeEqualUnchecked(*lhs, rhs); } -template class ICEBERG_TEMPLATE_EXPORT StructLikeSet; -template class ICEBERG_TEMPLATE_EXPORT StructLikeSet; +template class ICEBERG_TEMPLATE_INSTANTIATION_EXPORT StructLikeSet; +template class ICEBERG_TEMPLATE_INSTANTIATION_EXPORT StructLikeSet; } // namespace iceberg