Merge branch 'main' into 11.8-usr-merge

Author: grooverdan

CMakeLists.txt

@@ -31,7 +31,7 @@ ENDIF()
 # in RPM's:
 
 #set(CPACK_RPM_SPEC_MORE_DEFINE "%define __spec_install_post /bin/true")
-FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0056 CMP0067 CMP0074 CMP0075 CMP0069 CMP0135)
+FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0056 CMP0067 CMP0074 CMP0075 CMP0069 CMP0135 CMP0091)
   IF(POLICY ${p})
     CMAKE_POLICY(SET ${p} NEW)
   ENDIF()
@@ -119,6 +119,18 @@ ENDIF()
 
 FIND_PACKAGE(Git)
 
+IF(WIN32 AND (CMAKE_VERSION VERSION_GREATER "3.21"))
+ # Install runtime dependency by default, when using vcpkg
+ IF(NOT DEFINED INSTALL_RUNTIME_DEPENDENCIES_DEFAULT)
+   IF("${VCPKG_INSTALLED_DIR}")
+     SET(INSTALL_RUNTIME_DEPENDENCIES_DEFAULT OFF)
+   ELSE()
+     SET(INSTALL_RUNTIME_DEPENDENCIES_DEFAULT ON)
+   ENDIF()
+ ENDIF()
+ OPTION(INSTALL_RUNTIME_DEPENDENCIES "Install runtime dependencies"  "${INSTALL_RUNTIME_DEPENDENCIES_DEFAULT}")
+ENDIF()
+
 # Following autotools tradition, add preprocessor definitions
 # specified in environment variable CPPFLAGS
 IF(DEFINED ENV{CPPFLAGS})
@@ -361,6 +373,7 @@ IF(WIN32)
 ELSE()
   SET(DEFAULT_MYSQL_HOME ${CMAKE_INSTALL_PREFIX})
   SET(SHAREDIR ${INSTALL_MYSQLSHAREDIRABS})
+  SET(HOSTNAME "uname -n" CACHE STRING "Command for determining hostname")
 ENDIF()
 
 SET(DEFAULT_BASEDIR "${DEFAULT_MYSQL_HOME}")

appveyor.yml

@@ -1,6 +1,42 @@
 version: build-{build}~branch-{branch}
 
-clone_depth: 1
+clone_depth: 10
+
+skip_branch_with_pr: true
+before_build:
+  - ps: |
+      function Get-Remote-Ref($ref) {
+        try {
+          $result = git ls-remote origin $ref 2>$null
+          if (-not $result) {
+            "Warning: Could not fetch remote ref '$ref'"
+            return $null
+          }
+          return ($result -split "`t")[0]
+        } catch {
+          "Warning: Exception while running git ls-remote for '$ref': $_"
+          return $null
+        }
+      }
+      Get-ChildItem Env: | Where-Object { $_.Name -like 'APPVEYOR*COMMIT' } | ForEach-Object { "$($_.Name)=$($_.Value)" }
+      $commit = $env:APPVEYOR_REPO_COMMIT
+      $commit2 = $env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT
+      $branch = $env:APPVEYOR_REPO_BRANCH
+      $latest = $null
+      $mainBranch = $branch -match '^(main|\d+\.\d+)$'
+      if ($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null) {
+        "Branch build detected"
+        $latest = Get-Remote-Ref "refs/heads/$branch"
+      } else {
+        $pr = $env:APPVEYOR_PULL_REQUEST_NUMBER
+        $latest = Get-Remote-Ref "refs/pull/$pr/head"
+        $mainBranch = $False
+        "Pull Request build detected"
+      }
+      if ($latest -and ($commit -ne $latest) -and ($commit2 -ne $latest) -and (-not $mainBranch)) {
+        "Skipping outdated commit (latest is $latest)"
+        Exit-AppVeyorBuild
+      }
 
 environment:
   OPENSSL_ROOT_DIR: C:/OpenSSL-v32-Win64

client/mysqlbinlog.cc

@@ -160,9 +160,15 @@ static Server_gtid_event_filter *server_id_gtid_filter= NULL;
 
 static char *start_datetime_str, *stop_datetime_str;
 static my_time_t start_datetime= 0, stop_datetime= 0;
-static my_time_t last_processed_datetime= MY_TIME_T_MAX;
 static bool stop_datetime_given= false;
 
+typedef struct _last_processed_ev_t
+{
+  ulonglong position;
+  my_time_t datetime;
+} last_processed_ev_t;
+static last_processed_ev_t last_processed_ev= {0, MY_TIME_T_MAX};
+
 static ulonglong rec_count= 0;
 static MYSQL* mysql = NULL;
 static const char* dirname_for_local_load= 0;
@@ -1369,7 +1375,19 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
 end:
   rec_count++;
 end_skip_count:
-  last_processed_datetime= ev_when;
+  /*
+    Update the last_processed_ev, unless the event is a fake event (i.e. format
+    description (ev pointer is reset to 0) or rotate event (ev->when is 0)), or
+    the event is encrypted (i.e. type is Unknown).
+  */
+  if (ev &&
+      !(ev_type == UNKNOWN_EVENT &&
+        ((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED) &&
+      !(ev_type == ROTATE_EVENT && !ev->when))
+  {
+    last_processed_ev.position= pos + ev->data_written;
+    last_processed_ev.datetime= ev_when;
+  }
 
   DBUG_PRINT("info", ("end event processing"));
   /*
@@ -2661,6 +2679,9 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info,
       if (old_off != BIN_LOG_HEADER_SIZE)
         *len= 1;         // fake event, don't increment old_off
     }
+    DBUG_ASSERT(old_off + ev->data_written == old_off + (*len - 1) ||
+                (*len == 1 &&
+                 (type == ROTATE_EVENT || type == FORMAT_DESCRIPTION_EVENT)));
     Exit_status retval= process_event(print_event_info, ev, old_off, logname);
     if (retval != OK_CONTINUE)
       DBUG_RETURN(retval);
@@ -3035,6 +3056,8 @@ static Exit_status check_header(IO_CACHE* file,
             the new one, so we should not do it ourselves in this
             case.
           */
+          DBUG_ASSERT(tmp_pos + new_description_event->data_written ==
+                      my_b_tell(file));
           Exit_status retval= process_event(print_event_info,
                                             new_description_event, tmp_pos,
                                             logname);
@@ -3188,20 +3211,17 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
       }
       // else read_error == 0 means EOF, that's OK, we break in this case
 
-      /*
-        Emit a warning in the event that we finished processing input
-        before reaching the boundary indicated by --stop-position.
-      */
-      if (((longlong)stop_position != stop_position_default) &&
-          stop_position > my_b_tell(file))
-      {
-          retval = OK_STOP;
-          warning("Did not reach stop position %llu before "
-                  "end of input", stop_position);
-      }
-
       goto end;
     }
+
+    /*
+      The real location that we have read up to in the file should align with
+      the size of the event, unless the event is encrypted.
+    */
+    DBUG_ASSERT(
+        ((ev->get_type_code() == UNKNOWN_EVENT &&
+          ((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED)) ||
+        old_off + ev->data_written == my_b_tell(file));
     if ((retval= process_event(print_event_info, ev, old_off, logname)) !=
         OK_CONTINUE)
       goto end;
@@ -3380,9 +3400,17 @@ int main(int argc, char** argv)
     start_position= BIN_LOG_HEADER_SIZE;
   }
 
-  if (stop_datetime_given && stop_datetime > last_processed_datetime)
+  /*
+    Emit a warning if we finished processing input before reaching the stop
+    boundaries indicated by --stop-datetime or --stop-position.
+  */
+  if (stop_datetime_given && stop_datetime > last_processed_ev.datetime)
     warning("Did not reach stop datetime '%s' before end of input",
             stop_datetime_str);
+  if ((static_cast<longlong>(stop_position) != stop_position_default) &&
+      stop_position > last_processed_ev.position)
+    warning("Did not reach stop position %llu before end of input",
+            stop_position);
 
   /*
     If enable flashback, need to print the events from the end to the

cmake/3rdparty_info.cmake

@@ -0,0 +1,37 @@
+# This file is used for SBOM generation.
+
+# It consists of the list of 3rd party products
+# which can be compiled together with MariaDB server
+# and their licenses, copyright notices, and CPE prefixes
+# this is the vendor:product part of CPE identifier from
+# https://nvd.nist.gov/products/cpe
+
+# We use both git submodules, and CMake external projects
+# dependencies (as well we zlib, which is part of the code)
+# so the information is here for all these types
+
+SET("zlib.license" "Zlib")
+SET("zlib.copyright" "Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler")
+SET("zlib.cpe-prefix" "zlib:zlib")
+SET("minizip.license" "Zlib")
+SET("minizip.copyright" "Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler")
+SET("minizip.cpe-prefix" "zlib:zlib")
+SET("fmt.license" "MIT")
+SET("fmt.copyright" "Copyright (C) 2012 - present, Victor Zverovich")
+SET("fmt.cpe-prefix" "fmt:fmt")
+SET("pcre2.license" "BSD-3-Clause")
+SET("pcre2.cpe-prefix" "pcre:pcre2")
+SET("wolfssl.license" "GPL-2.0")
+SET("wolfssl.copyright" "Copyright (C) 2006-2024 wolfSSL Inc.")
+SET("wolfssl.cpe-prefix" "wolfssl:wolfssl")
+SET("boost.license" "BSL-1.0")
+SET("boost.cpe-prefix" "boost:boost")
+SET("mariadb-connector-c.license" "LGPL-2.1")
+SET("mariadb-connector-c.cpe-prefix" "mariadb:connector\\\\/c")
+SET("rocksdb.license" "GPL-2.0")
+SET("wsrep-lib.license" "GPL-2.0")
+SET("wsrep-api.license" "GPL-2.0")
+SET("mariadb-columnstore-engine.license" "GPL-2.0")
+SET("libmarias3.license" "LGPL-2.1")
+SET("thrift.license" "Apache-2.0")
+SET("thrift.cpe-prefix" "apache:thrift")

cmake/cpack_rpm.cmake

@@ -264,7 +264,7 @@ IF(WITH_WSREP)
     "galera-4" "rsync" "grep" "gawk" "iproute"
     "coreutils" "findutils" "tar")
   SETA(CPACK_RPM_server_PACKAGE_RECOMMENDS "lsof" "socat" "pv")
-  SETA(CPACK_RPM_test_PACKAGE_REQUIRES "socat")
+  SETA(CPACK_RPM_test_PACKAGE_REQUIRES "${CPACK_RPM_PACKAGE_REQUIRES}" "socat")
 ENDIF()
 
 SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-prein.sh)
@@ -311,7 +311,7 @@ ELSEIF(RPM MATCHES "fedora" OR RPM MATCHES "(rhel|centos)7")
   ALTERNATIVE_NAME("server" "mariadb-server")
   ALTERNATIVE_NAME("server" "mysql-compat-server")
   ALTERNATIVE_NAME("test"   "mariadb-test")
-ELSEIF(RPM MATCHES "(rhel|centos|rocky)[89]")
+ELSEIF(RPM MATCHES "(rhel|centos|rocky)")
   SET(epoch 3:)
   ALTERNATIVE_NAME("backup" "mariadb-backup")
   ALTERNATIVE_NAME("client" "mariadb")
@@ -343,53 +343,6 @@ ENDIF()
 
 SET(PYTHON_SHEBANG "/usr/bin/python3" CACHE STRING "python shebang")
 
-# If we want to build build MariaDB-shared-compat,
-# extract compat libraries from MariaDB-shared-5.3 rpm
-FILE(GLOB compat53 RELATIVE ${CMAKE_SOURCE_DIR}
-    "${CMAKE_SOURCE_DIR}/../MariaDB-shared-5.3.*.rpm")
-FILE(GLOB compat101 RELATIVE ${CMAKE_SOURCE_DIR}
-    "${CMAKE_SOURCE_DIR}/../MariaDB-shared-10.1.*.rpm")
-IF(compat53 AND compat101)
-  FOREACH(compat_rpm "${compat53}" "${compat101}")
-    MESSAGE(STATUS "Using ${compat_rpm} to build MariaDB-compat")
-    INSTALL(CODE "EXECUTE_PROCESS(
-                   COMMAND rpm2cpio ${CMAKE_SOURCE_DIR}/${compat_rpm}
-                   COMMAND cpio --extract --make-directories */libmysqlclient*.so.* -
-                   WORKING_DIRECTORY \$ENV{DESTDIR})
-                  EXECUTE_PROCESS(
-                   COMMAND chmod -R a+rX .
-                   WORKING_DIRECTORY \$ENV{DESTDIR})"
-                   COMPONENT Compat)
-  ENDFOREACH()
-
-  EXECUTE_PROCESS(
-    COMMAND rpm -q --provides -p "${CMAKE_SOURCE_DIR}/${compat101}"
-    ERROR_QUIET
-    OUTPUT_VARIABLE compat_provides)
-  EXECUTE_PROCESS(
-    COMMAND rpm -q --obsoletes -p "${CMAKE_SOURCE_DIR}/${compat101}"
-    ERROR_QUIET
-    OUTPUT_VARIABLE compat_obsoletes)
-
-  STRING(REPLACE "\n" " " compat_provides "${compat_provides}")
-  STRING(REPLACE "\n" " " compat_obsoletes "${compat_obsoletes}")
-  STRING(REGEX REPLACE "[^ ]+\\([^ ]+ *" "" compat_obsoletes "${compat_obsoletes}")
-  SETA(CPACK_RPM_compat_PACKAGE_PROVIDES "${compat_provides}")
-  SETA(CPACK_RPM_compat_PACKAGE_OBSOLETES "${compat_obsoletes}")
-
-  SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} Compat)
-
-  # RHEL6/CentOS6 install Postfix by default, and it requires
-  # libmysqlclient.so.16 that pulls in mysql-libs-5.1.x
-  # And the latter conflicts with our rpms.
-  # Make sure that for these distributions all our rpms require
-  # MariaDB-compat, that will replace mysql-libs-5.1
-  IF(RPM MATCHES "(rhel|centos)[67]")
-    SET(CPACK_RPM_common_PACKAGE_REQUIRES "MariaDB-compat")
-    SET(CPACK_RPM_compat_PACKAGE_CONFLICTS "mariadb-libs < 1:10.1.0")
-  ENDIF()
-ENDIF()
-
 ################
 IF(CMAKE_VERSION VERSION_GREATER "3.9.99")
 

cmake/generate_sbom.cmake

@@ -36,30 +36,23 @@ ENDMACRO()
 
 # Get CPE ID ( https://en.wikipedia.org/wiki/Common_Platform_Enumeration )
 # for given project name and version
-# Only "known" CPEs are handled here, e.g currently no CPE for rocksdb
+# CPE prefix are stored with other auxilliary info in the 3rdparty_info.cmake
+# file
 FUNCTION(SBOM_GET_CPE name version var)
-  SET(cpe_prefix_map
-    "zlib" "zlib:zlib"
-    "mariadb-connector-c" "mariadb:connector\\\\/c"
-    "wolfssl" "wolfssl:wolfssl"
-    "minizip" "zlib:zlib"
-    "pcre2" "pcre:pcre2"
-    "fmt" "fmt:fmt"
-    "boost" "boost:boost"
-    "thrift" "apache:thrift"
-  )
-  LIST(FIND cpe_prefix_map "${name}" idx_cpe_mapping)
-  # Version needs to have at least one dot character in it.
-  # Otherwise, we assume it is a git  hash, and do not generate CPE
-  STRING(FIND "${version}" "." idx_version_dot)
-  IF((idx_cpe_mapping GREATER -1) AND (idx_version_dot GREATER -1))
-    MATH(EXPR next_idx "${idx_cpe_mapping}+1")
-    LIST(GET cpe_prefix_map ${next_idx} cpe_name_and_vendor)
-    STRING(REGEX REPLACE "[^0-9\\.]" "" cleaned_version "${version}")
-    SET(${var} "cpe:2.3:a:${cpe_name_and_vendor}:${cleaned_version}:*:*:*:*:*:*:*" PARENT_SCOPE)
-  ELSE()
-    SET(${var} "" PARENT_SCOPE)
+  SET(${var} "" PARENT_SCOPE)
+  STRING(FIND "${version}" "." dot_idx)
+  IF(${dot_idx} EQUAL -1)
+    # Version does not have dot inside.
+    # mostly likely it is just a git hash
+    RETURN()
+  ENDIF()
+  SET(cpe_name_and_vendor "${${repo_name_lower}.cpe-prefix}")
+  IF(NOT cpe_name_and_vendor)
+    RETURN()
   ENDIF()
+
+  STRING(REGEX REPLACE "[^0-9\\.]" "" cleaned_version "${version}")
+  SET(${var} "cpe:2.3:a:${cpe_name_and_vendor}:${cleaned_version}:*:*:*:*:*:*:*" PARENT_SCOPE)
 ENDFUNCTION()
 
 # Add dependency on CMake ExternalProject.
@@ -97,8 +90,8 @@ ENDMACRO()
 # Perhaps it can always be "MariaDB", but security team recommendation is different
 # more towards "author"
 FUNCTION (sbom_get_supplier repo_name repo_user varname)
-  IF("${repo_name_SUPPLIER}")
-    SET(${varname} "${repo_name_SUPPLIER}" PARENT_SCOPE)
+  IF("${${repo_name}_SUPPLIER}")
+    SET(${varname} "${${repo_name}_SUPPLIER}" PARENT_SCOPE)
   ELSEIF (repo_name MATCHES "zlib|minizip")
     # stuff that is checked into out repos
     SET(${varname} "MariaDB" PARENT_SCOPE)
@@ -230,6 +223,7 @@ FUNCTION(GENERATE_SBOM)
       \"ref\": \"${CPACK_PACKAGE_NAME}\",
       \"dependsOn\": [" )
 
+  INCLUDE(3rdparty_info)
   SET(first ON)
   FOREACH(dep ${ALL_THIRD_PARTY})
     # Extract the part after the last "/" from URL
@@ -277,6 +271,14 @@ FUNCTION(GENERATE_SBOM)
     IF(cpe)
       SET(cpe "\n      \"cpe\": \"${cpe}\",")
     ENDIF()
+    SET(license "${${repo_name_lower}.license}")
+    IF(NOT license)
+      MESSAGE(FATAL_ERROR "no license for 3rd party dependency ${repo_name_lower}.")
+    ENDIF()
+    SET(copyright "${${repo_name_lower}.copyright}")
+    IF(NOT copyright)
+      SET(copyright NOASSERTION)
+    ENDIF()
     STRING(APPEND sbom_components "
     {
       \"bom-ref\": \"${bom_ref}\",
@@ -286,7 +288,15 @@ FUNCTION(GENERATE_SBOM)
       \"purl\": \"${purl}\",${cpe}
       \"supplier\": {
           \"name\": \"${supplier}\"
-       }
+       },
+      \"licenses\": [
+          {
+            \"license\": {
+              \"id\": \"${license}\"
+            }
+          }
+        ],
+      \"copyright\": \"${copyright}\"
     }")
     STRING(APPEND sbom_dependencies "
         \"${bom_ref}\"")
@@ -302,5 +312,6 @@ FUNCTION(GENERATE_SBOM)
   IF(NOT DEFINED CPACK_PACKAGE_VERSION)
     SET(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}")
   ENDIF()
+  STRING(TIMESTAMP CURRENT_YEAR "%Y")
   configure_file(${CMAKE_CURRENT_LIST_DIR}/cmake/sbom.json.in ${CMAKE_BINARY_DIR}/sbom.json)
 ENDFUNCTION()