PS-11264: Vector index support in Data Dictionary

include/my_base.h

@@ -105,10 +105,11 @@ enum ha_key_alg {
           SEs default algorithm for keys in mysql_prepare_create_table().
   */
   HA_KEY_ALG_SE_SPECIFIC = 0,
-  HA_KEY_ALG_BTREE = 1,   /* B-tree. */
-  HA_KEY_ALG_RTREE = 2,   /* R-tree, for spatial searches */
-  HA_KEY_ALG_HASH = 3,    /* HASH keys (HEAP, NDB). */
-  HA_KEY_ALG_FULLTEXT = 4 /* FULLTEXT. */
+  HA_KEY_ALG_BTREE = 1,    /* B-tree. */
+  HA_KEY_ALG_RTREE = 2,    /* R-tree, for spatial searches */
+  HA_KEY_ALG_HASH = 3,     /* HASH keys (HEAP, NDB). */
+  HA_KEY_ALG_FULLTEXT = 4, /* FULLTEXT. */
+  HA_KEY_ALG_VECTOR = 5,   /* VECTOR. */
 };
 
 /* Storage media types */
@@ -521,11 +522,14 @@ enum ha_base_keytype {
 #define HA_USES_COMMENT (1 << 12)
 /** Key was automatically created to support Foreign Key constraint. */
 #define HA_GENERATED_KEY (1 << 13)
+/** Vector key (Percona). */
+#define HA_VECTOR (1 << 30)
 
 /* The combination of the above can be used for key type comparison. */
 #define HA_KEYFLAG_MASK                                                       \
   (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \
-   HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY)
+   HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY |      \
+   HA_VECTOR)
 
 /** Fulltext index uses [pre]parser */
 #define HA_USES_PARSER (1 << 14)

share/messages_to_clients.txt

@@ -11112,6 +11112,12 @@ ER_LOG_NAME_NOT_MATCHING_SEC_LOG_PATH_CLIENT
 # Start of Percona Server 8.4/9.7 error messages to be sent to client
 #
 
+ER_VECTOR_INDEX_NEEDS_PK
+  eng "Vector index can only be created in tables with a BIGINT UNSIGNED primary key."
+
+ER_ONLY_SINGLE_VECTOR_INDEX_ALLOWED
+  eng "A table can have at most one vector index."
+
 start-error-number 7100
 
 #

sql/create_field.cc

@@ -23,6 +23,7 @@
 
 #include "sql/create_field.h"
 
+#include "field_types.h"
 #include "m_string.h"
 #include "mysql/strings/dtoa.h"
 #include "sql-common/my_decimal.h"
@@ -200,9 +201,10 @@ bool Create_field::init(
     const LEX_CSTRING *fld_comment, const char *fld_change,
     List<String> *fld_interval_list, const CHARSET_INFO *fld_charset,
     bool has_explicit_collation, uint fld_geom_type,
-    const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info, Value_generator *fld_default_val_expr,
-    LEX_CSTRING fld_masking_policy, std::optional<gis::srid_t> srid,
-    dd::Column::enum_hidden_type hidden, bool is_array_arg) {
+    const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info,
+    Value_generator *fld_default_val_expr, LEX_CSTRING fld_masking_policy,
+    std::optional<gis::srid_t> srid, dd::Column::enum_hidden_type hidden,
+    bool is_array_arg) {
   uint sign_len, allowed_type_modifier = 0;
   ulong max_field_charlength = MAX_FIELD_CHARLENGTH;
 
@@ -780,7 +782,8 @@ size_t Create_field::key_length() const {
     case MYSQL_TYPE_JSON:
     case MYSQL_TYPE_VAR_STRING:
     case MYSQL_TYPE_STRING:
-    case MYSQL_TYPE_VARCHAR: {
+    case MYSQL_TYPE_VARCHAR:
+    case MYSQL_TYPE_VECTOR: {
       return std::min(max_display_width_in_bytes(),
                       static_cast<size_t>(MAX_FIELD_BLOBLENGTH));
     }
@@ -794,10 +797,6 @@ size_t Create_field::key_length() const {
       }
       return pack_length() + (max_display_width_in_bytes() & 7 ? 1 : 0);
     }
-    /* LCOV_EXCL_START */
-    case MYSQL_TYPE_VECTOR:
-      assert(false);  // Key on VECTOR type column is not supported.
-    /* LCOV_EXCL_STOP */
     default: {
       return pack_length(is_array);
     }

sql/dd/dd_table.cc

@@ -735,6 +735,10 @@ bool fill_dd_columns_from_create_fields(THD *thd, dd::Abstract_table *tab_obj,
       col_options->set("is_array", true);
     }
 
+    if (field.sql_type == MYSQL_TYPE_VECTOR) {
+      col_options->set("vector_index", true);
+    }
+
     //
     // Write intervals
     //
@@ -825,6 +829,9 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type(
 
     case HA_KEY_ALG_FULLTEXT:
       return dd::Index::IA_FULLTEXT;
+
+    case HA_KEY_ALG_VECTOR:
+      return dd::Index::IA_SE_SPECIFIC;
   }
 
   /* purecov: begin deadcode */
@@ -836,6 +843,8 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type(
 }
 
 static dd::Index::enum_index_type dd_get_new_index_type(const KEY *key) {
+  if (key->flags & HA_VECTOR) return dd::Index::IT_MULTIPLE;
+
   if (key->flags & HA_FULLTEXT) return dd::Index::IT_FULLTEXT;
 
   if (key->flags & HA_SPATIAL) return dd::Index::IT_SPATIAL;

sql/dd/impl/types/column_impl.cc

@@ -64,11 +64,17 @@ class Sdi_rcontext;
 class Sdi_wcontext;
 
 static const std::set<String_type> default_valid_option_keys = {
-    "column_format",  "geom_type",
-    "interval_count", "not_secondary",
-    "storage",        "treat_bit_as_char", "zip_dict_id",
-    "is_array",       "gipk" /* generated implicit primary key column */,
-    "masking_policy"};
+    "column_format",
+    "geom_type",
+    "interval_count",
+    "not_secondary",
+    "storage",
+    "treat_bit_as_char",
+    "zip_dict_id",
+    "is_array",
+    "gipk" /* generated implicit primary key column */,
+    "masking_policy",
+    "vector_index"};
 
 ///////////////////////////////////////////////////////////////////////////
 // Column_impl implementation.

sql/dd/info_schema/show.cc

@@ -868,6 +868,50 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd,
 
   Select_lex_builder top_query(&pos, thd);
 
+  Item *index_type_item =
+      new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str);
+  if (index_type_item == nullptr) return nullptr;
+
+  Item *se_specific_item = new (thd->mem_root)
+      Item_string(STRING_WITH_LEN("SE_SPECIFIC"), system_charset_info);
+  if (se_specific_item == nullptr) return nullptr;
+
+  Item *is_se_specific =
+      new (thd->mem_root) Item_func_eq(pos, index_type_item, se_specific_item);
+  if (is_se_specific == nullptr) return nullptr;
+
+  Item *sub_part_item =
+      new (thd->mem_root) Item_field(pos, NullS, NullS, alias_sub_part.str);
+  if (sub_part_item == nullptr) return nullptr;
+
+  Item *one_item = new (thd->mem_root)
+      Item_string(STRING_WITH_LEN("1"), system_charset_info);
+  if (one_item == nullptr) return nullptr;
+
+  Item *is_single_sub_part =
+      new (thd->mem_root) Item_func_eq(pos, sub_part_item, one_item);
+  if (is_single_sub_part == nullptr) return nullptr;
+
+  Item *vector_item = new (thd->mem_root)
+      Item_string(STRING_WITH_LEN("VECTOR"), system_charset_info);
+  if (vector_item == nullptr) return nullptr;
+
+  Item *index_type_else_item =
+      new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str);
+  if (index_type_else_item == nullptr) return nullptr;
+
+  Item *index_type_if = new (thd->mem_root)
+      Item_func_if(pos, is_single_sub_part, vector_item, index_type_else_item);
+  if (index_type_if == nullptr) return nullptr;
+
+  Item *index_type_default_item =
+      new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str);
+  if (index_type_default_item == nullptr) return nullptr;
+
+  Item *index_type_expr = new (thd->mem_root)
+      Item_func_if(pos, is_se_specific, index_type_if, index_type_default_item);
+  if (index_type_expr == nullptr) return nullptr;
+
   // SELECT * FROM <sub_query> ...
   if (top_query.add_select_item(alias_table, alias_table) ||
       top_query.add_select_item(alias_non_unique, alias_non_unique) ||
@@ -879,7 +923,7 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd,
       top_query.add_select_item(alias_sub_part, alias_sub_part) ||
       top_query.add_select_item(alias_packed, alias_packed) ||
       top_query.add_select_item(alias_null, alias_null) ||
-      top_query.add_select_item(alias_type, alias_type) ||
+      top_query.add_select_expr(index_type_expr, alias_type) ||
       top_query.add_select_item(alias_comment, alias_comment) ||
       top_query.add_select_item(alias_index_comment, alias_index_comment) ||
       top_query.add_select_item(alias_visible, alias_visible) ||

sql/dd_table_share.cc

@@ -235,6 +235,29 @@ static enum ha_key_alg dd_get_old_index_algorithm_type(
   return HA_KEY_ALG_SE_SPECIFIC;
 }
 
+/**
+  Check whether any visible index element is marked as vector.
+
+  @param[in] idx_obj Index metadata object.
+
+  @return Whether any visible element belongs to a vector column.
+*/
+static bool dd_index_has_vector_column(const dd::Index &idx_obj) {
+  for (const dd::Index_element *idx_elem : idx_obj.elements()) {
+    if (idx_elem->is_hidden()) continue;
+
+    const dd::Properties &col_options = idx_elem->column().options();
+    bool is_vector_column = false;
+    if (col_options.exists("vector_index") &&
+        !col_options.get("vector_index", &is_vector_column) &&
+        is_vector_column) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 /*
   Check if the given key_part is suitable to be promoted as part of
   primary key.
@@ -347,6 +370,8 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share,
              share->key_info[key].algorithm == HA_KEY_ALG_FULLTEXT);
       assert(!(share->key_info[key].flags & HA_SPATIAL) ||
              share->key_info[key].algorithm == HA_KEY_ALG_RTREE);
+      assert(!(share->key_info[key].flags & HA_VECTOR) ||
+             share->key_info[key].algorithm == HA_KEY_ALG_VECTOR);
 
       if (primary_key >= MAX_KEY && (keyinfo->flags & HA_NOSAME)) {
         /*
@@ -1385,6 +1410,8 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share,
   keyinfo->algorithm = dd_get_old_index_algorithm_type(idx_obj->algorithm());
   keyinfo->is_algorithm_explicit = idx_obj->is_algorithm_explicit();
 
+  const bool has_vector_column = dd_index_has_vector_column(*idx_obj);
+
   // Visibility
   keyinfo->is_visible = idx_obj->is_visible();
 
@@ -1395,6 +1422,11 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share,
     if (!idx_ele->is_hidden()) keyinfo->user_defined_key_parts++;
   }
 
+  if (has_vector_column && keyinfo->user_defined_key_parts == 1) {
+    keyinfo->algorithm = HA_KEY_ALG_VECTOR;
+    keyinfo->is_algorithm_explicit = false;
+  }
+
   // flags
   switch (idx_obj->type()) {
     case dd::Index::IT_MULTIPLE:
@@ -1416,6 +1448,10 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share,
       break;
   }
 
+  if (has_vector_column && keyinfo->user_defined_key_parts == 1) {
+    keyinfo->flags |= HA_VECTOR;
+  }
+
   if (idx_obj->is_generated()) keyinfo->flags |= HA_GENERATED_KEY;
 
   /*

sql/field.cc

@@ -34,6 +34,7 @@
 #include <optional>
 
 #include "decimal.h"
+#include "field_types.h"
 #include "my_alloc.h"
 #include "my_byteorder.h"
 #include "my_compare.h"
@@ -75,7 +76,7 @@
 #include "sql/mysqld_cs.h"
 #include "sql/protocol.h"
 #include "sql/psi_memory_key.h"
-#include "sql/spatial.h"                // Geometry
+#include "sql/spatial.h"  // Geometry
 #include "sql/sql_base.h"
 #include "sql/sql_class.h"              // THD
 #include "sql/sql_exception_handler.h"  // handle_std_exception
@@ -1656,6 +1657,7 @@ bool Field::type_can_have_key_part(enum enum_field_types type) {
     case MYSQL_TYPE_VAR_STRING:
     case MYSQL_TYPE_STRING:
     case MYSQL_TYPE_GEOMETRY:
+    case MYSQL_TYPE_VECTOR:
       return true;
     default:
       return false;

sql/handler.h

@@ -535,6 +535,7 @@ enum class SelectExecutedIn : bool { kPrimaryEngine, kSecondaryEngine };
   ANALYZE TABLE on it
 */
 #define HA_ONLINE_ANALYZE (1LL << 56)
+#define HA_CAN_VECTOR (1LL << 57)
 
 /*
   Bits in index_flags(index_number) for what you can do with index.
@@ -3294,7 +3295,6 @@ inline constexpr const decltype(handlerton::flags)
 inline constexpr const decltype(handlerton::flags)
     HTON_SECONDARY_SUPPORTS_TEMPORARY_TABLE(1 << 25);
 
-
 /** Start of Percona specific HTON_* defines */
 
 /**
@@ -3312,7 +3312,6 @@ inline constexpr const decltype(handlerton::flags)
 
 /** End of Percona specific HTON_* defines */
 
-
 /* Whether the handlerton is a secondary engine. */
 inline bool hton_is_secondary_engine(const handlerton *hton) {
   return hton != nullptr && (hton->flags & HTON_IS_SECONDARY_ENGINE) != 0U;
@@ -7055,16 +7054,16 @@ class handler {
     for details.
   */
   [[nodiscard]] int ha_fast_update(THD *thd,
-                                  mem_root_deque<Item *> &update_fields,
-                                  mem_root_deque<Item *> &update_values,
-                                  Item *conds);
+                                   mem_root_deque<Item *> &update_fields,
+                                   mem_root_deque<Item *> &update_values,
+                                   Item *conds);
 
   /**
     @brief Offload an upsert to the storage engine. See handler::upsert()
     for details.
   */
   [[nodiscard]] int ha_upsert(THD *thd, mem_root_deque<Item *> &update_fields,
-                             mem_root_deque<Item *> &update_values);
+                              mem_root_deque<Item *> &update_values);
 
  private:
   /**
@@ -7087,11 +7086,11 @@ class handler {
     handler::ha_update_row(...) does not accept conditions.
   */
   [[nodiscard]] virtual int fast_update(THD *thd [[maybe_unused]],
-                                       mem_root_deque<Item *> &update_fields
-                                       [[maybe_unused]],
-                                       mem_root_deque<Item *> &update_values
-                                       [[maybe_unused]],
-                                       Item *conds [[maybe_unused]]) {
+                                        mem_root_deque<Item *> &update_fields
+                                        [[maybe_unused]],
+                                        mem_root_deque<Item *> &update_values
+                                        [[maybe_unused]],
+                                        Item *conds [[maybe_unused]]) {
     return ENOTSUP;
   }
 
@@ -7112,10 +7111,10 @@ class handler {
     @return an error if the insert should be terminated.
   */
   [[nodiscard]] virtual int upsert(THD *thd [[maybe_unused]],
-                                  mem_root_deque<Item *> &update_fields
-                                  [[maybe_unused]],
-                                  mem_root_deque<Item *> &update_values
-                                  [[maybe_unused]]) {
+                                   mem_root_deque<Item *> &update_fields
+                                   [[maybe_unused]],
+                                   mem_root_deque<Item *> &update_values
+                                   [[maybe_unused]]) {
     return ENOTSUP;
   }
 
@@ -7624,7 +7623,6 @@ class handler {
 
   int get_lock_type() const { return m_lock_type; }
 
-
  public:
   /* Read-free replication interface */
 

sql/key_spec.h

@@ -43,7 +43,8 @@ enum keytype {
   KEYTYPE_MULTIPLE = 2,
   KEYTYPE_FULLTEXT = 4,
   KEYTYPE_SPATIAL = 8,
-  KEYTYPE_FOREIGN = 16
+  KEYTYPE_FOREIGN = 16,
+  KEYTYPE_VECTOR = 32,
 };
 
 enum fk_option {