v1.16.0-rc0 (#158)

Co-authored-by: rtosholdings-bot <rtosholdings-bot@sig.com>

Author: OrestZborowski-SIG

.github/workflows/python_package.yml

@@ -192,6 +192,8 @@ jobs:
       - name: Build package
         run: |
           set -ex
+          conda config --set conda_build.pkg_format 2 # create .conda package format
+          conda config --set conda_build.zstd_compression_level 19 # set .conda package format compression level
           export BUILD_VERSION=$(python -c "from setuptools_scm import get_version; print(get_version(version_scheme='post-release'))")
           mkdir conda_pkgs_output
           echo "python: " ${{ matrix.zips.python }} > ./conda_variant.yaml
@@ -201,7 +203,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: conda-build-artifacts
-          path: conda_pkgs_output/*/riptide_cpp-*.tar.bz2
+          path: conda_pkgs_output/*/riptide_cpp-*.conda
           if-no-files-found: "error"
 
   conda_deploy:
@@ -240,4 +242,4 @@ jobs:
         shell: bash -l {0}
         run: |
           set -ex
-          anaconda --token "${ANACONDA_TOKEN}" upload --label main --user ${ANACONDA_USER} ./conda_pkgs_output/*/riptide_cpp-*.tar.bz2
+          anaconda --token "${ANACONDA_TOKEN}" upload --label main --user ${ANACONDA_USER} ./conda_pkgs_output/*/riptide_cpp-*.conda

CMakeLists.txt

@@ -56,10 +56,8 @@ if(PROJ_COMPILER_ID STREQUAL "MSVC")
     endif()
 elseif(PROJ_COMPILER_ID STREQUAL "GNU")
     # No toolset variants
-    # Remove warnings from absl compilation
 elseif(PROJ_COMPILER_ID STREQUAL "LLVM")
     # No toolset variants
-    # Remove warnings from absl compilation
 else()
     message(FATAL_ERROR "Unexpected proj compiler ID, ${PROJ_COMPILER_ID}")
 endif()
@@ -87,7 +85,6 @@ macro(verbose_find_package)
     message("Found package ${ARGV0} config: ${${ARGV0}_CONFIG}")
 endmacro()
 
-verbose_find_package(absl CONFIG REQUIRED)
 verbose_find_package(benchmark CONFIG REQUIRED)
 verbose_find_package(TBB CONFIG REQUIRED)
 verbose_find_package(zstd REQUIRED)

README.md

@@ -2,6 +2,4 @@
 
 Cross platform 64 bit c++ files for processing numpy arrays with stacking, multifile, and multithreaded SDS file format.
 
-## usage
-
-See examples in <https://github.com/rtosholdings/riptable-docdata>
+Part of the riptable package, <https://github.com/rtosholdings/riptable>.

bench/riptide_bench/riptide_bench/CMakeLists.txt

@@ -5,7 +5,9 @@ set(CMAKE_VERBOSE_MAKEFILE on)
 set(SOURCES main.cpp
     bench_one_input.cpp
     hash_linear_bench.cpp
-    memcmp_bench.cpp)
+    memcmp_bench.cpp
+    bench_logging.cpp
+    )
 
 add_executable(${TARGET_NAME} ${HEADERS} ${SOURCES})
 

bench/riptide_bench/riptide_bench/bench_logging.cpp

@@ -0,0 +1,61 @@
+#include "logging/logger.h"
+
+#include <benchmark/benchmark.h>
+
+namespace
+{
+    using namespace riptide::logging;
+    auto & logg = logger::get();
+
+    static void bench_logging(benchmark::State & state)
+    {
+        for (auto _ : state)
+        {
+            auto num_threads = state.range(0);
+            auto num_logs = state.range(1);
+            auto produce = [=](int id)
+            {
+                for (auto i = 0; i < num_logs; i++)
+                {
+                    logg.log(loglevel::debug, "Thread: {0} log number: {1}", id, i);
+                }
+            };
+
+            auto consume = [=]()
+            {
+                while (logg.active())
+                {
+                    auto curr{ logg.receive() };
+                    if (! curr)
+                        break;
+
+                    benchmark::DoNotOptimize(curr.value().level);
+                    benchmark::DoNotOptimize(curr.value().message);
+                }
+            };
+
+            logg.enable({ .max_size = 1'000'000'000, .level = loglevel::debug });
+
+            std::thread consumer{ consume };
+            std::vector<std::thread> threads;
+
+            for (auto i = 0; i < num_threads; i++)
+            {
+                threads.emplace_back(produce, i);
+            }
+
+            for (auto & t : threads)
+            {
+                t.join();
+            }
+
+            logg.disable();
+            consumer.join();
+        }
+    }
+
+    BENCHMARK(bench_logging)
+        ->Unit(benchmark::TimeUnit::kMillisecond)
+        ->ArgsProduct({ { 2, 4, 8, 16, 32, 64 }, { 1000, 10'000, 100'000 } })
+        ->UseRealTime();
+}

cmake/AddCommonSettings.cmake

@@ -36,9 +36,10 @@ if(PROJ_COMPILER_FRONTEND STREQUAL "MSVC")
         target_compile_options(rt_common_settings INTERFACE -Wno-duplicated-branches)
     endif()
 elseif(PROJ_COMPILER_FRONTEND STREQUAL "GNU")
-    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Debug>:-O0>")
-    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Release>:-O2>")
-    target_compile_options(rt_common_settings INTERFACE -ggdb -x c++ -mavx2 -mbmi2 -fpermissive -pthread)
+    target_compile_options(rt_common_settings INTERFACE -x c++ -gdwarf-4) # dwarf-4 (for gdb8); must come before -g1
+    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Debug>:-O0;-g>")
+    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Release>:-O2;-g1>")
+    target_compile_options(rt_common_settings INTERFACE -mavx2 -mbmi2 -fpermissive -pthread)
     target_compile_options(rt_common_settings INTERFACE -falign-functions=32 -fno-strict-aliasing)
     target_compile_options(rt_common_settings INTERFACE -Wall -Werror)
     target_compile_options(rt_common_settings INTERFACE -Wno-error=cast-qual)
@@ -59,9 +60,9 @@ elseif(PROJ_COMPILER_FRONTEND STREQUAL "GNU")
     target_compile_options(rt_common_settings INTERFACE -Wno-useless-cast)
     target_compile_options(rt_common_settings INTERFACE -Wno-old-style-cast)
 elseif(PROJ_COMPILER_FRONTEND STREQUAL "LLVM")
-    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Debug>:-O0>")
-    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Release>:-O2>")
-    target_compile_options(rt_common_settings INTERFACE -ggdb -x c++ -mavx2 -mbmi2 -fpermissive -pthread)
+    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Debug>:-O0;-g>")
+    target_compile_options(rt_common_settings INTERFACE "$<$<CONFIG:Release>:-O2;-g1>")
+    target_compile_options(rt_common_settings INTERFACE -x c++ -mavx2 -mbmi2 -fpermissive -pthread)
     target_compile_options(rt_common_settings INTERFACE -falign-functions=32 -fno-strict-aliasing)
     target_compile_options(rt_common_settings INTERFACE -Wall -Werror)
     target_compile_options(rt_common_settings INTERFACE -Wno-format) # TODO: Remove this and fix all the printf format mismatches

conda_recipe/conda_build_config.yaml

@@ -15,7 +15,7 @@ cxx_compiler:
   - vs2022 # [win]
 
 c_compiler_version:
-  - 10 # [linux]
+  - 13 # [linux]
 
 cxx_compiler_version:
-  - 10 # [linux]
+  - 13 # [linux]

conda_recipe/meta.yaml

@@ -13,7 +13,6 @@ build:
 requirements:
   build:
     - python {{ python }}
-    - abseil-cpp 20220623.*
     - benchmark >=1.7,<1.8
     - "{{ compiler('cxx') }}"
     - cmake >=3.21
@@ -28,7 +27,6 @@ requirements:
     - setuptools_scm
   run:
     - python {{ python }}
-    - abseil-cpp 20220623.*
     - "{{ pin_compatible('numpy', min_pin='x.x', max_pin='x.x') }}"
     - tbb 2021.6.*
     - zstd >=1.5,<1.6

dev_tools/gen_requirements.py

@@ -18,7 +18,6 @@ def is_python(major: int, minor: int) -> bool:
     return ver.major == major and ver.minor == minor
 
 
-_ABSEIL_REQ = "abseil-cpp==20220623.*"
 _BENCHMARK_REQ = "benchmark>=1.7,<1.8"
 _CMAKE_REQ = "cmake>=3.21"
 _NUMPY_REQ = "numpy>=1.23,<1.25"
@@ -33,8 +32,9 @@ def is_python(major: int, minor: int) -> bool:
     toolchain_reqs += [
         "binutils",
         "binutils_linux-64",
-        "gcc==10.*",
-        "gxx==10.*",
+        "gcc==13.*",
+        "gxx==13.*",
+        "libstdcxx-ng=13.*",
         "ninja",
     ]
 
@@ -50,7 +50,6 @@ def is_python(major: int, minor: int) -> bool:
 # Most everything else *should* be in pyproject.toml, but since we run
 # setup.py directly we need to set up build environment manually here.
 pypi_reqs = [
-    _ABSEIL_REQ,  # PyPI package doesn't exist
     _BENCHMARK_REQ,  # PyPI package doesn't exist
     _CMAKE_REQ,
     _TBB_DEVEL_REQ,  # needed because PyPI tbb-devel pkg doesn't contain CMake files yet
@@ -60,7 +59,6 @@ def is_python(major: int, minor: int) -> bool:
 
 # Native CMake build requirements.
 native_reqs = [
-    _ABSEIL_REQ,
     _BENCHMARK_REQ,
     _CMAKE_REQ,
     _NUMPY_REQ,
@@ -74,9 +72,6 @@ def is_python(major: int, minor: int) -> bool:
 # Runtime requirements for riptide_cpp.
 # Replicates runtime requirements in meta.yaml and setup.py.
 runtime_reqs = [
-    # Disable abseil-cpp requirements to avoid pip install failures (no PyPI package exists)
-    # Assume it's conda install'ed as part of build-level requirements.
-    # _ABSEIL_REQ,
     _NUMPY_REQ,
     _TBB_REQ,
     _ZSTD_REQ,

pyproject.toml

@@ -1,6 +1,5 @@
 [build-system]
 requires = [
-    #"abseil-cpp ==20220623.*", # DOES NOT EXIST! We must assume it exists in environment
     #"benchmark >=1.7,<1.8", # DOES NOT EXIST! We must assume it exists in environment
     "cmake >=3.21",
     "ninja; platform_system!='Windows'",