diff --git a/configuration/builders/definitions/connectors/conc.py b/configuration/builders/definitions/connectors/conc.py index abb415ef..714bd13d 100644 --- a/configuration/builders/definitions/connectors/conc.py +++ b/configuration/builders/definitions/connectors/conc.py @@ -1,5 +1,201 @@ +import os +from pathlib import PurePath + +from buildbot.plugins import util from configuration.builders.base import GenericBuilder -from configuration.builders.sequences.connectors.conc import deb, rpm, tarball +from configuration.builders.common import docker_config +from configuration.builders.infra.runtime import DockerConfig, Sidecar +from configuration.builders.sequences.connectors.conc import ( + bintar, + get_source_package, + git_clone_sq, + save_packages, + tarball, + windows, +) + +PACKAGES_DIR = f"{os.environ['CONNECTORS_PACKAGES_DIR']}/c" +BUILD_BASE_PATH = "build" +BINTAR_PATH = f"{BUILD_BASE_PATH}/bintar" +SOURCE_PATH = f"{BUILD_BASE_PATH}/source" +BINTAR_PACKAGES_TO_SAVE = [f"{BINTAR_PATH}/*.tar.gz"] + + +# MariaDB Server used for C/C++ tests +SIDECAR = Sidecar( + repository="docker.io/library/", + image_tag="mariadb:lts", + env_vars=[("MARIADB_ROOT_PASSWORD", "test"), ("MARIADB_DATABASE", "test")], + tmpfs=PurePath("/var/lib/mysql"), +) + + +TARBALL = GenericBuilder( + name="cc-tarball-docker", + sequences=[ + tarball( + config=docker_config( + image="debian13", + packages_dir=PACKAGES_DIR, + artifacts_url=f"{os.environ['ARTIFACTS_URL']}/connector-c/", + ), + ) + ], +) + + +def generate_bintar_sqs( + ops, + version, + build_environment: DockerConfig = None, + upload_packages_to_ci=True, + get_source_from_git=False, + with_asan_ubsan=False, + with_msan=False, +): + if not build_environment: + build_environment = docker_config( + image=f"{ops}{version}", + packages_dir=PACKAGES_DIR, + artifacts_url=f"{os.environ['ARTIFACTS_URL']}/connector-c/", + ) + + alma_linux_environment = None + rockylinux_environment = None + if ops == "rhel": + alma_linux_environment = DockerConfig( + repository="docker.io/library/", + image_tag=f"almalinux:{version}", + ) + rockylinux_environment = DockerConfig( + repository="rockylinux/", + image_tag=f"rockylinux:{version}", + ) + + test_environments = [ + build_environment, + alma_linux_environment, + rockylinux_environment, + ] + + source_sq = [ + get_source_package( + config=build_environment, + source_path=SOURCE_PATH, + ), + ] + + if get_source_from_git: + source_sq = [ + git_clone_sq( + config=build_environment, + source_path=SOURCE_PATH, + ) + ] + + return ( + source_sq + + [ + bintar( + config=build_environment, + test_environments=test_environments, + source_path=SOURCE_PATH, + bintar_path=BINTAR_PATH, + package_platform_suffix=f"{ops}{version}", + jobs=util.Property("jobs"), + with_asan_ubsan=with_asan_ubsan, + with_msan=with_msan, + ), + ] + + ( + [ + save_packages( + packages=BINTAR_PACKAGES_TO_SAVE, + config=build_environment, + ) + ] + if upload_packages_to_ci + else [] + ) + ) + -TARBALL = GenericBuilder(name="cc-tarball-docker", sequences=[tarball()]) RELEASE_BUILDERS_BY_ARCH = {"amd64": [], "aarch64": []} +for arch in ["amd64", "aarch64"]: + for ops, version in [ + ("fedora", "43"), + ("fedora", "44"), + ("sles", "1507"), + ("sles", "1600"), + ("rhel", "8"), + ("rhel", "9"), + ("rhel", "10"), + ]: + if ops == "sles" and arch != "amd64": + continue + builder = GenericBuilder( + name=f"cc-{arch}-{ops}-{version}", + sidecar=SIDECAR, + sequences=generate_bintar_sqs(ops=ops, version=version), + ) + RELEASE_BUILDERS_BY_ARCH[arch].append(builder) + + for ops, version in [ + ("debian", "11"), + ("debian", "12"), + ("debian", "13"), + ("ubuntu", "22.04"), + ("ubuntu", "24.04"), + ("ubuntu", "26.04"), + ]: + builder = GenericBuilder( + name=f"cc-{arch}-{ops}-{version}", + sidecar=SIDECAR, + sequences=generate_bintar_sqs(ops=ops, version=version), + ) + RELEASE_BUILDERS_BY_ARCH[arch].append(builder) + +# UBASAN_BUILDER = GenericBuilder( +# name="cc-debian-13-ubasan-clang-22", +# sidecar=SIDECAR, +# sequences=generate_bintar_sqs( +# build_environment=docker_config( +# image="debian13-msan-clang-22", +# artifacts_url=f"{os.environ['ARTIFACTS_URL']}/connector-c/", +# ), +# ops="debian", +# version="13", +# upload_packages_to_ci=False, +# with_asan_ubsan=True, +# get_source_from_git=True, +# ), +# ) + +WINDOWS_64_BUILDER = GenericBuilder( + name="cc-amd64-windows", + sequences=[ + windows(jobs=util.Property("jobs"), target_platform="64-bit"), + ], +) + +WINDOWS_32_BUILDER = GenericBuilder( + name="cc-x86-windows", + sequences=[ + windows(jobs=util.Property("jobs"), target_platform="32-bit"), + ], +) + +# MSAN_BUILDER = GenericBuilder( +# name="cc-debian-13-msan-clang-22", +# sidecar=SIDECAR, +# sequences=generate_bintar_sqs( +# build_environment=docker_config( +# image="debian13-msan-clang-22", +# artifacts_url=f"{os.environ['ARTIFACTS_URL']}/connector-c/", +# ), +# with_msan=True, +# ops="debian", +# version="13", +# upload_packages_to_ci=False, +# ), +# ) diff --git a/configuration/builders/sequences/connectors/conc.py b/configuration/builders/sequences/connectors/conc.py index fdd277b2..9226d4c0 100644 --- a/configuration/builders/sequences/connectors/conc.py +++ b/configuration/builders/sequences/connectors/conc.py @@ -1,35 +1,427 @@ +import os +from pathlib import PurePath + import configuration.steps.commands.trigger as trigger -from configuration.builders.infra.runtime import BuildSequence +from configuration.builders.infra.runtime import ( + BuildSequence, + DockerConfig, + InContainer, +) +from configuration.steps.base import StepOptions +from configuration.steps.commands.base import URL, BashCommand +from configuration.steps.commands.compile import MAKE, CompileCMakeCommand +from configuration.steps.commands.configure import ConfigureMariaDBCMake +from configuration.steps.commands.download import FetchTarball, GitInitFromCommit +from configuration.steps.commands.packages import ( + ArchiveSource, + InstallRPMPackages, + SavePackages, +) +from configuration.steps.commands.upload import FileUpload from configuration.steps.commands.util import PrintEnvironmentDetails -from configuration.steps.remote import ShellStep +from configuration.steps.generators.cmake.generator import CMakeGenerator +from configuration.steps.generators.cmake.options import ( + BUILDPLATFORM, + BUILDTOOLS, + CMAKE, + OTHER, + WITH, + BuildType, + CMakeOption, +) +from configuration.steps.remote import PropFromShellStep, ShellStep + + +def git_clone_step(step_wrapping_fn=lambda step: step, source_path: str = "."): + source_path = PurePath(source_path) + return step_wrapping_fn( + ShellStep( + command=GitInitFromCommit( + repo_url="%(prop:repository)s", + commit="%(prop:revision)s", + workdir=source_path, + ), + options=StepOptions( + description="Initialize git repository", + descriptionDone="Git repository initialized", + ), + ), + ) -def tarball(): +def git_clone_sq(config: DockerConfig = None, source_path: str = "."): + sequence = BuildSequence() + if config: + sequence.add_step( + git_clone_step( + lambda step: InContainer( + step, + docker_environment=config, + ), + source_path=source_path, + ) + ) + return sequence + sequence.add_step(git_clone_step(source_path=source_path)) + return sequence + + +def tarball(config: DockerConfig): ### INIT sequence = BuildSequence() ### ADD STEPS sequence.add_step(ShellStep(command=PrintEnvironmentDetails())) + sequence.add_step( + git_clone_step(lambda step: InContainer(step, docker_environment=config)) + ) + sequence.add_step( + InContainer( + ShellStep( + command=ArchiveSource( + input_dir=PurePath("."), + output_dir=PurePath("ci_source"), + tarball_name="ci.tar.gz", + generate_sha256=True, + ), + options=StepOptions( + description="Archive source code", + descriptionDone="Source code archived", + ), + ), + docker_environment=config, + ), + ) + + sequence.add_step( + InContainer( + ShellStep( + command=ConfigureMariaDBCMake( + name="Create source package", + cmake_generator=CMakeGenerator( + use_ccache=False, + flags=[ + CMakeOption(OTHER.GIT_BUILD_SRCPKG, True), + CMakeOption(CMAKE.BUILD_TYPE, BuildType.RELWITHDEBUG), + ], + ), + ), + options=StepOptions( + description="Create source package - Configure CMake", + descriptionDone="Create source package - CMake configured", + ), + ), + docker_environment=config, + ), + ) + + sequence.add_step( + InContainer( + ShellStep( + command=SavePackages( + packages=[ + "ci_source/ci.tar.gz", + "*src.tar.gz", + "ci_source/sha256sums.txt", + ], + destination="/packages/%(prop:buildnumber)s", + ), + url=URL( + url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:buildnumber)s", + url_text="Source tarball", + ), + options=StepOptions( + description="Save source packages", + descriptionDone="Source packages saved", + ), + ), + docker_environment=config, + ) + ) + sequence.add_step(trigger.ConC()) return sequence -def deb(): - ### INIT +def get_source_package(config: DockerConfig, source_path: str): sequence = BuildSequence() - - ### ADD STEPS sequence.add_step(ShellStep(command=PrintEnvironmentDetails())) + sequence.add_step( + InContainer( + ShellStep( + command=FetchTarball(workdir=PurePath(source_path)), + options=StepOptions( + description="Fetch tarball", + descriptionDone="Fetch tarball done", + ), + ), + docker_environment=config, + ) + ) + return sequence + + +def bintar( + config: DockerConfig, + test_environments: list[DockerConfig], + jobs: int, + package_platform_suffix: str, + bintar_path: str, + source_path: str, + with_asan_ubsan=False, + with_msan=False, +): + sequence = BuildSequence() + env_vars = None + flags = [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.RELWITHDEBUG), + CMakeOption(OTHER.PACKAGE_PLATFORM_SUFFIX, package_platform_suffix), + CMakeOption(WITH.DOCS, True), + ] + + if with_msan: + flags.append(CMakeOption(WITH.MSAN, True)) + flags.append( + CMakeOption( + CMAKE.EXE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath,${MSAN_LIBDIR}" + ) + ) + flags.append( + CMakeOption( + CMAKE.SHARED_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath,${MSAN_LIBDIR}" + ) + ) + flags.append( + CMakeOption( + CMAKE.MODULE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath,${MSAN_LIBDIR}" + ) + ) + if with_asan_ubsan: + flags.append(CMakeOption(WITH.ASAN, True)) + flags.append(CMakeOption(WITH.UBSAN, True)) + env_vars = [ + ( + "ASAN_OPTIONS", + "detect_stack_use_after_return=1:detect_leaks=1:abort_on_error=1:atexit=0:detect_invalid_pointer_pairs=3:dump_instruction_bytes=1:allocator_may_return_null=1", + ), + ( + "UBSAN_OPTIONS", + f"suppressions=/home/buildbot/{source_path}/UBSAN.supp:print_stacktrace=1:report_error_type=1:halt_on_error=1", + ), + ] + + sequence.add_step( + InContainer( + ShellStep( + command=ConfigureMariaDBCMake( + name="Bintar", + cmake_generator=CMakeGenerator( + source_path=source_path, + builddir=bintar_path, + use_ccache=True, + flags=flags, + ), + ), + env_vars=env_vars, + options=StepOptions( + description="Bintar - Configure CMake", + descriptionDone="Bintar - CMake configured", + ), + ), + docker_environment=config, + ), + ) + + sequence.add_step( + InContainer( + ShellStep( + command=CompileCMakeCommand( + workdir=PurePath(bintar_path), + target=MAKE.PACKAGE, + jobs=jobs, + ), + env_vars=env_vars, + options=StepOptions( + description="Bintar - Compile", + descriptionDone="Bintar - Compile done", + ), + ), + docker_environment=config, + ), + ) + + for test_env in test_environments: + if test_env: + if "almalinux" in test_env.image_tag or "rockylinux" in test_env.image_tag: + sequence.add_step( + InContainer( + ShellStep( + command=InstallRPMPackages( + packages=["epel-release"], + name="Install EPEL repository", + ), + ), + container_commit=True, + docker_environment=test_env, + ) + ) + + sequence.add_step( + InContainer( + ShellStep( + command=InstallRPMPackages( + packages=["cmake", "python3-pyOpenSSL"], + name="Install packages for testing", + ), + ), + container_commit=True, + docker_environment=test_env, + ) + ) + + sequence.add_step( + InContainer( + ShellStep( + command=BashCommand( + name="Test bintar on {}".format(test_env.image_tag), + workdir=PurePath(f"{bintar_path}/unittest/libmariadb"), + cmd="export MYSQL_TEST_HOST=$SIDECAR_HOST && ctest --output-on-failure", + user="root", + ), + env_vars=[ + ("MYSQL_TEST_USER", "root"), + ("MYSQL_TEST_PASSWD", "test"), + ("MYSQL_TEST_PORT", "3306"), + ("MYSQL_TEST_DB", "test"), + ("MYSQL_TEST_VERBOSE", "true"), + ("MARIADB_CC_TEST", 1), + ("MYSQL_TEST_TLS", 0), + ("MYSQL_TEST_SSL_PORT", 0), + ] + + (env_vars if env_vars else []), + options=StepOptions( + description="Bintar - Run C/C ctest", + descriptionDone="Bintar - C/C ctest done", + ), + ), + docker_environment=test_env, + ), + ) return sequence -def rpm(): - ### INIT +def save_packages(config: DockerConfig, packages: list[str], user: str = "buildbot"): sequence = BuildSequence() + sequence.add_step( + InContainer( + ShellStep( + command=SavePackages( + packages=packages, + destination="/packages/%(prop:tarbuildnum)s/%(prop:buildername)s", + user=user, + ), + options=StepOptions( + description="Save packages", + descriptionDone="Save packages done", + ), + url=URL( + url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:tarbuildnum)s/%(prop:buildername)s", + url_text="Packages", + ), + ), + docker_environment=config, + ) + ) + return sequence - ### ADD STEPS - sequence.add_step(ShellStep(command=PrintEnvironmentDetails())) +def windows(jobs: int, target_platform: str): + sequence = BuildSequence() + sequence.add_step(git_clone_step()) + + if target_platform == "32-bit": + cmake_generator = CMakeGenerator( + build_platform=BUILDPLATFORM.WIN32, + build_tool=BUILDTOOLS.WINVS2022, + flags=[], + ) + if target_platform == "64-bit": + cmake_generator = CMakeGenerator(build_tool=BUILDTOOLS.WINVS2022, flags=[]) + + cmake_generator.flags.extend( + [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.RELWITHDEBUG), + CMakeOption(WITH.CURL, True), + CMakeOption(WITH.MSI, True), + ], + ) + + sequence.add_step( + ShellStep( + command=ConfigureMariaDBCMake( + name="RelWithDebugInfo", + cmake_generator=cmake_generator, + ), + options=StepOptions( + description="Configure CMake", + descriptionDone="CMake configured", + ), + ), + ) + + sequence.add_step( + ShellStep( + command=CompileCMakeCommand( + jobs=jobs, + config=BuildType.RELWITHDEBUG, + ), + options=StepOptions( + description="Build package", + descriptionDone="Package built", + ), + ), + ) + + sequence.add_step( + ShellStep( + command=BashCommand( + name="C/C ctest", + cmd="cd unittest/libmariadb && ctest --output-on-failure", + ), + env_vars=[ + ("MYSQL_TEST_USER", "root"), + ("MYSQL_TEST_PASSWD", "test"), + ("MYSQL_TEST_PORT", "3306"), + ("MYSQL_TEST_DB", "test"), + ("MYSQL_TEST_HOST", "127.0.0.1"), + ("MARIADB_CC_TEST", "1"), + ("MYSQL_TEST_TLS", "0"), + ], + options=StepOptions( + description="Run C/C ctest", + descriptionDone="C/C ctest done", + ), + ), + ) + + sequence.add_step( + PropFromShellStep( + command=BashCommand( + name="find MSI", + cmd="find . -maxdepth 1 -type f -name '*.msi' -exec basename {} \\;", + ), + property="packages", + ), + ) + + sequence.add_step( + FileUpload( + workersrc="%(prop:packages)s", + masterdest="/srv/buildbot/connectors/c/%(prop:tarbuildnum)s/%(prop:buildername)s/%(prop:packages)s", + mode=0o755, + url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:tarbuildnum)s/%(prop:buildername)s/", + ) + ) return sequence diff --git a/configuration/schedulers/connectors.py b/configuration/schedulers/connectors.py index 54858f89..486aadf7 100644 --- a/configuration/schedulers/connectors.py +++ b/configuration/schedulers/connectors.py @@ -94,13 +94,18 @@ ), schedulers.Triggerable( name="conc_c_all_scheduler", + # builderNames=[ + # builder.name + # for builder in ( + # builder + # for builders in c_builders.RELEASE_BUILDERS_BY_ARCH.values() + # for builder in builders + # ) + # ] + # + [c_builders.WINDOWS_32_BUILDER.name, c_builders.WINDOWS_64_BUILDER.name], builderNames=[ - builder.name - for builder in ( - builder - for builders in c_builders.RELEASE_BUILDERS_BY_ARCH.values() - for builder in builders - ) + c_builders.WINDOWS_32_BUILDER.name, + c_builders.WINDOWS_64_BUILDER.name, ], ), ] diff --git a/configuration/steps/commands/trigger.py b/configuration/steps/commands/trigger.py index 2d211e8d..4366eaaa 100644 --- a/configuration/steps/commands/trigger.py +++ b/configuration/steps/commands/trigger.py @@ -80,8 +80,14 @@ def __init__(self): self.name = "Trigger Conc-C Builders" self.schedulername = "conc_c_all_scheduler" self.doStepIf = lambda step: True + properties = { + "tarbuildnum": Property( + "buildnumber" + ), # Used by get_tarball.sh to identify the tarball dir on CI + "mariadb_version": "ci", # Used by get_tarball.sh to download ci.tar.gz + } - super().__init__(self.name, self.schedulername, self.doStepIf) + super().__init__(self.name, self.schedulername, self.doStepIf, properties) class Install(Server): diff --git a/configuration/steps/generators/cmake/options.py b/configuration/steps/generators/cmake/options.py index eaee40d7..d3981c67 100644 --- a/configuration/steps/generators/cmake/options.py +++ b/configuration/steps/generators/cmake/options.py @@ -101,11 +101,13 @@ class WITH(StrEnum): ASAN = "ASAN" ASAN_SCOPED = "ASAN_SCOPED" + CURL = "CURL" DBUG_TRACE = "DBUG_TRACE" EMBEDDED_SERVER = "EMBEDDED_SERVER" EXTRA_CHARSETS = "EXTRA_CHARSETS" JEMALLOC = "JEMALLOC" MSAN = "MSAN" + MSI = "MSI" NONE = "NONE" SAFEMALLOC = "SAFEMALLOC" SSL = "SSL" @@ -118,6 +120,7 @@ class WITH(StrEnum): OPENSSL = "OPENSSL" SIGNCODE = "SIGNCODE" EXTERNAL_ZLIB = "EXTERNAL_ZLIB" + DOCS = "DOCS" def __str__(self): return f"WITH_{self.value}" diff --git a/master-migration/master.cfg b/master-migration/master.cfg index aab16a4f..20efbfb0 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -269,6 +269,23 @@ for arch, builders in conc_c_builders.RELEASE_BUILDERS_BY_ARCH.items(): ) ) +# Connector/C Windows builders +c["builders"].append( + conc_c_builders.WINDOWS_64_BUILDER.get_config( + workers=WORKER_POOL.get_workers_for_arch(arch="amd64", names=["bbw5-windows"]), + tags=["connector", "c", "windows"], + jobs=1, + ) +) + +c["builders"].append( + conc_c_builders.WINDOWS_32_BUILDER.get_config( + workers=WORKER_POOL.get_workers_for_arch(arch="amd64", names=["bbw5-windows"]), + tags=["connector", "c", "windows"], + jobs=1, + ) +) + # # Tarball Connector C++ c["builders"].append( conc_cc_builders.TARBALL.get_config(