diff --git a/include/my_base.h b/include/my_base.h index 66a4b1e556b7..df03e746f7c6 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -105,10 +105,11 @@ enum ha_key_alg { SEs default algorithm for keys in mysql_prepare_create_table(). */ HA_KEY_ALG_SE_SPECIFIC = 0, - HA_KEY_ALG_BTREE = 1, /* B-tree. */ - HA_KEY_ALG_RTREE = 2, /* R-tree, for spatial searches */ - HA_KEY_ALG_HASH = 3, /* HASH keys (HEAP, NDB). */ - HA_KEY_ALG_FULLTEXT = 4 /* FULLTEXT. */ + HA_KEY_ALG_BTREE = 1, /* B-tree. */ + HA_KEY_ALG_RTREE = 2, /* R-tree, for spatial searches */ + HA_KEY_ALG_HASH = 3, /* HASH keys (HEAP, NDB). */ + HA_KEY_ALG_FULLTEXT = 4, /* FULLTEXT. */ + HA_KEY_ALG_VECTOR = 5, /* VECTOR. */ }; /* Storage media types */ @@ -521,11 +522,14 @@ enum ha_base_keytype { #define HA_USES_COMMENT (1 << 12) /** Key was automatically created to support Foreign Key constraint. */ #define HA_GENERATED_KEY (1 << 13) +/** Vector key (Percona). */ +#define HA_VECTOR (1 << 30) /* The combination of the above can be used for key type comparison. */ #define HA_KEYFLAG_MASK \ (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \ - HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY) + HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY | \ + HA_VECTOR) /** Fulltext index uses [pre]parser */ #define HA_USES_PARSER (1 << 14) diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index e8a5df1a936d..fba1aff37edc 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -11112,6 +11112,12 @@ ER_LOG_NAME_NOT_MATCHING_SEC_LOG_PATH_CLIENT # Start of Percona Server 8.4/9.7 error messages to be sent to client # +ER_VECTOR_INDEX_NEEDS_PK + eng "Vector index can only be created in tables with a BIGINT UNSIGNED primary key." + +ER_ONLY_SINGLE_VECTOR_INDEX_ALLOWED + eng "A table can have at most one vector index." + start-error-number 7100 # diff --git a/sql/create_field.cc b/sql/create_field.cc index 168d5293cd94..ac61abbaeeca 100644 --- a/sql/create_field.cc +++ b/sql/create_field.cc @@ -23,6 +23,7 @@ #include "sql/create_field.h" +#include "field_types.h" #include "m_string.h" #include "mysql/strings/dtoa.h" #include "sql-common/my_decimal.h" @@ -200,9 +201,10 @@ bool Create_field::init( const LEX_CSTRING *fld_comment, const char *fld_change, List *fld_interval_list, const CHARSET_INFO *fld_charset, bool has_explicit_collation, uint fld_geom_type, - const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info, Value_generator *fld_default_val_expr, - LEX_CSTRING fld_masking_policy, std::optional srid, - dd::Column::enum_hidden_type hidden, bool is_array_arg) { + const LEX_CSTRING *fld_zip_dict_name, Value_generator *fld_gcol_info, + Value_generator *fld_default_val_expr, LEX_CSTRING fld_masking_policy, + std::optional srid, dd::Column::enum_hidden_type hidden, + bool is_array_arg) { uint sign_len, allowed_type_modifier = 0; ulong max_field_charlength = MAX_FIELD_CHARLENGTH; @@ -780,7 +782,8 @@ size_t Create_field::key_length() const { case MYSQL_TYPE_JSON: case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_STRING: - case MYSQL_TYPE_VARCHAR: { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VECTOR: { return std::min(max_display_width_in_bytes(), static_cast(MAX_FIELD_BLOBLENGTH)); } @@ -794,10 +797,6 @@ size_t Create_field::key_length() const { } return pack_length() + (max_display_width_in_bytes() & 7 ? 1 : 0); } - /* LCOV_EXCL_START */ - case MYSQL_TYPE_VECTOR: - assert(false); // Key on VECTOR type column is not supported. - /* LCOV_EXCL_STOP */ default: { return pack_length(is_array); } diff --git a/sql/dd/dd_table.cc b/sql/dd/dd_table.cc index 8a14c56e8599..e3fc670c6512 100644 --- a/sql/dd/dd_table.cc +++ b/sql/dd/dd_table.cc @@ -735,6 +735,10 @@ bool fill_dd_columns_from_create_fields(THD *thd, dd::Abstract_table *tab_obj, col_options->set("is_array", true); } + if (field.sql_type == MYSQL_TYPE_VECTOR) { + col_options->set("vector_index", true); + } + // // Write intervals // @@ -825,6 +829,9 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type( case HA_KEY_ALG_FULLTEXT: return dd::Index::IA_FULLTEXT; + + case HA_KEY_ALG_VECTOR: + return dd::Index::IA_SE_SPECIFIC; } /* purecov: begin deadcode */ @@ -836,6 +843,8 @@ static dd::Index::enum_index_algorithm dd_get_new_index_algorithm_type( } static dd::Index::enum_index_type dd_get_new_index_type(const KEY *key) { + if (key->flags & HA_VECTOR) return dd::Index::IT_MULTIPLE; + if (key->flags & HA_FULLTEXT) return dd::Index::IT_FULLTEXT; if (key->flags & HA_SPATIAL) return dd::Index::IT_SPATIAL; diff --git a/sql/dd/impl/types/column_impl.cc b/sql/dd/impl/types/column_impl.cc index 34cb323def7b..1b6669ed6bc4 100644 --- a/sql/dd/impl/types/column_impl.cc +++ b/sql/dd/impl/types/column_impl.cc @@ -64,11 +64,17 @@ class Sdi_rcontext; class Sdi_wcontext; static const std::set default_valid_option_keys = { - "column_format", "geom_type", - "interval_count", "not_secondary", - "storage", "treat_bit_as_char", "zip_dict_id", - "is_array", "gipk" /* generated implicit primary key column */, - "masking_policy"}; + "column_format", + "geom_type", + "interval_count", + "not_secondary", + "storage", + "treat_bit_as_char", + "zip_dict_id", + "is_array", + "gipk" /* generated implicit primary key column */, + "masking_policy", + "vector_index"}; /////////////////////////////////////////////////////////////////////////// // Column_impl implementation. diff --git a/sql/dd/info_schema/show.cc b/sql/dd/info_schema/show.cc index f37078e4fce8..94cc320d3552 100644 --- a/sql/dd/info_schema/show.cc +++ b/sql/dd/info_schema/show.cc @@ -868,6 +868,50 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd, Select_lex_builder top_query(&pos, thd); + Item *index_type_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_item == nullptr) return nullptr; + + Item *se_specific_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("SE_SPECIFIC"), system_charset_info); + if (se_specific_item == nullptr) return nullptr; + + Item *is_se_specific = + new (thd->mem_root) Item_func_eq(pos, index_type_item, se_specific_item); + if (is_se_specific == nullptr) return nullptr; + + Item *sub_part_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_sub_part.str); + if (sub_part_item == nullptr) return nullptr; + + Item *one_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("1"), system_charset_info); + if (one_item == nullptr) return nullptr; + + Item *is_single_sub_part = + new (thd->mem_root) Item_func_eq(pos, sub_part_item, one_item); + if (is_single_sub_part == nullptr) return nullptr; + + Item *vector_item = new (thd->mem_root) + Item_string(STRING_WITH_LEN("VECTOR"), system_charset_info); + if (vector_item == nullptr) return nullptr; + + Item *index_type_else_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_else_item == nullptr) return nullptr; + + Item *index_type_if = new (thd->mem_root) + Item_func_if(pos, is_single_sub_part, vector_item, index_type_else_item); + if (index_type_if == nullptr) return nullptr; + + Item *index_type_default_item = + new (thd->mem_root) Item_field(pos, NullS, NullS, alias_type.str); + if (index_type_default_item == nullptr) return nullptr; + + Item *index_type_expr = new (thd->mem_root) + Item_func_if(pos, is_se_specific, index_type_if, index_type_default_item); + if (index_type_expr == nullptr) return nullptr; + // SELECT * FROM ... if (top_query.add_select_item(alias_table, alias_table) || top_query.add_select_item(alias_non_unique, alias_non_unique) || @@ -879,7 +923,7 @@ Query_block *build_show_keys_query(const POS &pos, THD *thd, top_query.add_select_item(alias_sub_part, alias_sub_part) || top_query.add_select_item(alias_packed, alias_packed) || top_query.add_select_item(alias_null, alias_null) || - top_query.add_select_item(alias_type, alias_type) || + top_query.add_select_expr(index_type_expr, alias_type) || top_query.add_select_item(alias_comment, alias_comment) || top_query.add_select_item(alias_index_comment, alias_index_comment) || top_query.add_select_item(alias_visible, alias_visible) || diff --git a/sql/dd_table_share.cc b/sql/dd_table_share.cc index c5504726c0a9..56bd874e29f6 100644 --- a/sql/dd_table_share.cc +++ b/sql/dd_table_share.cc @@ -235,6 +235,29 @@ static enum ha_key_alg dd_get_old_index_algorithm_type( return HA_KEY_ALG_SE_SPECIFIC; } +/** + Check whether any visible index element is marked as vector. + + @param[in] idx_obj Index metadata object. + + @return Whether any visible element belongs to a vector column. +*/ +static bool dd_index_has_vector_column(const dd::Index &idx_obj) { + for (const dd::Index_element *idx_elem : idx_obj.elements()) { + if (idx_elem->is_hidden()) continue; + + const dd::Properties &col_options = idx_elem->column().options(); + bool is_vector_column = false; + if (col_options.exists("vector_index") && + !col_options.get("vector_index", &is_vector_column) && + is_vector_column) { + return true; + } + } + + return false; +} + /* Check if the given key_part is suitable to be promoted as part of primary key. @@ -347,6 +370,8 @@ static bool prepare_share(THD *thd, TABLE_SHARE *share, share->key_info[key].algorithm == HA_KEY_ALG_FULLTEXT); assert(!(share->key_info[key].flags & HA_SPATIAL) || share->key_info[key].algorithm == HA_KEY_ALG_RTREE); + assert(!(share->key_info[key].flags & HA_VECTOR) || + share->key_info[key].algorithm == HA_KEY_ALG_VECTOR); if (primary_key >= MAX_KEY && (keyinfo->flags & HA_NOSAME)) { /* @@ -1385,6 +1410,8 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, keyinfo->algorithm = dd_get_old_index_algorithm_type(idx_obj->algorithm()); keyinfo->is_algorithm_explicit = idx_obj->is_algorithm_explicit(); + const bool has_vector_column = dd_index_has_vector_column(*idx_obj); + // Visibility keyinfo->is_visible = idx_obj->is_visible(); @@ -1395,6 +1422,11 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, if (!idx_ele->is_hidden()) keyinfo->user_defined_key_parts++; } + if (has_vector_column && keyinfo->user_defined_key_parts == 1) { + keyinfo->algorithm = HA_KEY_ALG_VECTOR; + keyinfo->is_algorithm_explicit = false; + } + // flags switch (idx_obj->type()) { case dd::Index::IT_MULTIPLE: @@ -1416,6 +1448,10 @@ static bool fill_index_from_dd(THD *thd, TABLE_SHARE *share, break; } + if (has_vector_column && keyinfo->user_defined_key_parts == 1) { + keyinfo->flags |= HA_VECTOR; + } + if (idx_obj->is_generated()) keyinfo->flags |= HA_GENERATED_KEY; /* diff --git a/sql/field.cc b/sql/field.cc index abc98b6f449c..6f969c1c5ddf 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -34,6 +34,7 @@ #include #include "decimal.h" +#include "field_types.h" #include "my_alloc.h" #include "my_byteorder.h" #include "my_compare.h" @@ -75,7 +76,7 @@ #include "sql/mysqld_cs.h" #include "sql/protocol.h" #include "sql/psi_memory_key.h" -#include "sql/spatial.h" // Geometry +#include "sql/spatial.h" // Geometry #include "sql/sql_base.h" #include "sql/sql_class.h" // THD #include "sql/sql_exception_handler.h" // handle_std_exception @@ -1656,6 +1657,7 @@ bool Field::type_can_have_key_part(enum enum_field_types type) { case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_STRING: case MYSQL_TYPE_GEOMETRY: + case MYSQL_TYPE_VECTOR: return true; default: return false; diff --git a/sql/handler.h b/sql/handler.h index cdb1ac488eb4..eebff3109644 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -535,6 +535,7 @@ enum class SelectExecutedIn : bool { kPrimaryEngine, kSecondaryEngine }; ANALYZE TABLE on it */ #define HA_ONLINE_ANALYZE (1LL << 56) +#define HA_CAN_VECTOR (1LL << 57) /* Bits in index_flags(index_number) for what you can do with index. @@ -3294,7 +3295,6 @@ inline constexpr const decltype(handlerton::flags) inline constexpr const decltype(handlerton::flags) HTON_SECONDARY_SUPPORTS_TEMPORARY_TABLE(1 << 25); - /** Start of Percona specific HTON_* defines */ /** @@ -3312,7 +3312,6 @@ inline constexpr const decltype(handlerton::flags) /** End of Percona specific HTON_* defines */ - /* Whether the handlerton is a secondary engine. */ inline bool hton_is_secondary_engine(const handlerton *hton) { return hton != nullptr && (hton->flags & HTON_IS_SECONDARY_ENGINE) != 0U; @@ -7055,16 +7054,16 @@ class handler { for details. */ [[nodiscard]] int ha_fast_update(THD *thd, - mem_root_deque &update_fields, - mem_root_deque &update_values, - Item *conds); + mem_root_deque &update_fields, + mem_root_deque &update_values, + Item *conds); /** @brief Offload an upsert to the storage engine. See handler::upsert() for details. */ [[nodiscard]] int ha_upsert(THD *thd, mem_root_deque &update_fields, - mem_root_deque &update_values); + mem_root_deque &update_values); private: /** @@ -7087,11 +7086,11 @@ class handler { handler::ha_update_row(...) does not accept conditions. */ [[nodiscard]] virtual int fast_update(THD *thd [[maybe_unused]], - mem_root_deque &update_fields - [[maybe_unused]], - mem_root_deque &update_values - [[maybe_unused]], - Item *conds [[maybe_unused]]) { + mem_root_deque &update_fields + [[maybe_unused]], + mem_root_deque &update_values + [[maybe_unused]], + Item *conds [[maybe_unused]]) { return ENOTSUP; } @@ -7112,10 +7111,10 @@ class handler { @return an error if the insert should be terminated. */ [[nodiscard]] virtual int upsert(THD *thd [[maybe_unused]], - mem_root_deque &update_fields - [[maybe_unused]], - mem_root_deque &update_values - [[maybe_unused]]) { + mem_root_deque &update_fields + [[maybe_unused]], + mem_root_deque &update_values + [[maybe_unused]]) { return ENOTSUP; } @@ -7624,7 +7623,6 @@ class handler { int get_lock_type() const { return m_lock_type; } - public: /* Read-free replication interface */ diff --git a/sql/key_spec.h b/sql/key_spec.h index b58e0651b759..47abcff67da2 100644 --- a/sql/key_spec.h +++ b/sql/key_spec.h @@ -43,7 +43,8 @@ enum keytype { KEYTYPE_MULTIPLE = 2, KEYTYPE_FULLTEXT = 4, KEYTYPE_SPATIAL = 8, - KEYTYPE_FOREIGN = 16 + KEYTYPE_FOREIGN = 16, + KEYTYPE_VECTOR = 32, }; enum fk_option { diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 52b03e9f785e..8f699cd6ca77 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2791,6 +2791,8 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, packet->append(STRING_WITH_LEN("FULLTEXT KEY ")); else if (key_info->flags & HA_SPATIAL) packet->append(STRING_WITH_LEN("SPATIAL KEY ")); + else if (key_info->flags & HA_VECTOR) + packet->append(STRING_WITH_LEN("VECTOR KEY ")); else packet->append(STRING_WITH_LEN("KEY ")); @@ -2823,7 +2825,7 @@ bool store_create_info(THD *thd, Table_ref *table_list, String *packet, if (key_part->field && (key_part->length != table->field[key_part->fieldnr - 1]->key_length() && - !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL)))) { + !(key_info->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)))) { packet->append_parenthesized((long)key_part->length / key_part->field->charset()->mbmaxlen); } @@ -5482,6 +5484,23 @@ static int fill_schema_engines(THD *thd, Table_ref *tables, Item *) { #define TMP_TABLE_KEYS_IS_VISIBLE 14 #define TMP_TABLE_KEYS_EXPRESSION 15 +/** + Detect vector indexes for SHOW INDEX output. + + @param[in] key_info index metadata from TABLE_SHARE + + @return Whether this is a vector index. +*/ +static bool is_show_index_vector_type(const KEY *key_info) { + if (key_info->flags & HA_VECTOR) return true; + + if (key_info->user_defined_key_parts != 1) return false; + + const KEY_PART_INFO *key_part = key_info->key_part; + return key_part != nullptr && key_part->field != nullptr && + key_part->field->real_type() == MYSQL_TYPE_VECTOR; +} + static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, TABLE *table, bool res, LEX_CSTRING, LEX_CSTRING table_name) { @@ -5570,6 +5589,8 @@ static int get_schema_tmp_table_keys_record(THD *thd, Table_ref *tables, // INDEX_TYPE if (key_info->flags & HA_SPATIAL) str = "SPATIAL"; + else if (is_show_index_vector_type(key_info)) + str = "VECTOR"; else { const ha_key_alg key_alg = key_info->algorithm; /* If index algorithm is implicit get SE default. */ diff --git a/sql/sql_table.cc b/sql/sql_table.cc index faaf57304faf..b8295fe47d0a 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5175,8 +5175,8 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, return true; } - // VECTOR columns cannot be used as keys - if (sql_field->sql_type == MYSQL_TYPE_VECTOR) { + if (sql_field->sql_type == MYSQL_TYPE_VECTOR && + false /* !(key_info->flags & HA_VECTOR) */) { my_error(ER_NON_SCALAR_USED_AS_KEY, MYF(0), column->get_field_name()); return true; } @@ -5227,6 +5227,13 @@ static bool prepare_key_column(THD *thd, HA_CREATE_INFO *create_info, data prefix, ignoring column->length). */ column_length = is_blob(sql_field->sql_type); + } else if (key->type == KEYTYPE_VECTOR) { + // VECTOR indexes are only allowed on VECTOR columns. + if (sql_field->sql_type != MYSQL_TYPE_VECTOR) { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + column_length = 1; // Dummy value. } else { switch (sql_field->sql_type) { case MYSQL_TYPE_GEOMETRY: @@ -5833,7 +5840,7 @@ static bool prepare_self_ref_fk_parent_key( for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (hton->foreign_keys_flags & HTON_FKS_NEED_DIFFERENT_PARENT_AND_SUPPORTING_KEYS) { @@ -6036,7 +6043,7 @@ static const KEY *find_fk_supporting_key(handlerton *hton, for (const KEY *key = key_info_buffer; key < key_info_buffer + key_count; key++) { // We can't use FULLTEXT or SPATIAL indexes. - if (key->flags & (HA_FULLTEXT | HA_SPATIAL)) continue; + if (key->flags & (HA_FULLTEXT | HA_SPATIAL | HA_VECTOR)) continue; if (key->algorithm == HA_KEY_ALG_HASH) { if (hton->foreign_keys_flags & HTON_FKS_WITH_SUPPORTING_HASH_KEYS) { @@ -7686,6 +7693,17 @@ static bool prepare_key( switch (static_cast(key->type)) { case KEYTYPE_MULTIPLE: break; + case KEYTYPE_VECTOR: + if (!(file->ha_table_flags() & HA_CAN_VECTOR)) { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + if (key->columns.size() != 1) { + my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), 1); + return true; + } + key_info->flags |= HA_VECTOR; + break; case KEYTYPE_FULLTEXT: if (!(file->ha_table_flags() & HA_CAN_FULLTEXT)) { my_error(ER_TABLE_CANT_HANDLE_FT, MYF(0)); @@ -7743,6 +7761,9 @@ static bool prepare_key( } else if (key_info->flags & HA_FULLTEXT) { assert(!key->key_create_info.is_algorithm_explicit); key_info->algorithm = HA_KEY_ALG_FULLTEXT; + } else if (key_info->flags & HA_VECTOR) { + assert(!key->key_create_info.is_algorithm_explicit); + key_info->algorithm = HA_KEY_ALG_VECTOR; } else { if (key->key_create_info.is_algorithm_explicit) { if (key->key_create_info.algorithm != HA_KEY_ALG_RTREE) { @@ -8644,6 +8665,7 @@ bool mysql_prepare_create_table( uint key_number = 0; bool primary_key = false; + uint vector_key_number = 0; // First prepare non-foreign keys so that they are ready when // we prepare foreign keys. @@ -8660,6 +8682,14 @@ bool mysql_prepare_create_table( primary_key = true; } + if (key->type == KEYTYPE_VECTOR) { + if (vector_key_number) { + my_error(ER_ONLY_SINGLE_VECTOR_INDEX_ALLOWED, MYF(0)); + return true; + } + ++vector_key_number; + } + if (key->type != KEYTYPE_FOREIGN) { if (prepare_key(thd, error_schema_name, error_table_name, create_info, &alter_info->create_list, key, key_info_buffer, key_info, @@ -8678,6 +8708,12 @@ bool mysql_prepare_create_table( } } + // We allow VECTOR keys only with tables with PK + if (!primary_key && vector_key_number) { + my_error(ER_VECTOR_INDEX_NEEDS_PK, MYF(0)); + return true; + } + /* At this point all KEY objects are for indexes are fully constructed. So we can check for duplicate indexes for keys for which it was requested. @@ -8724,6 +8760,26 @@ bool mysql_prepare_create_table( /* Sort keys in optimized order */ std::sort(*key_info_buffer, *key_info_buffer + *key_count, sort_keys()); + // We allow VECTOR indexes only on tables with BIGINT UNSIGNED PKs. + if (vector_key_number) { + assert(primary_key); + const KEY &primary_info = *key_info_buffer[0]; + + if (primary_info.actual_key_parts > 1) { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + for (it.rewind(), field_no = 0; (sql_field = it++); field_no++) { + if (field_no >= primary_info.key_part[0].fieldnr) break; + } + assert(sql_field); + if (sql_field->sql_type != MYSQL_TYPE_LONGLONG || !sql_field->is_unsigned) { + my_error(ER_VECTOR_INDEX_NEEDS_PK, MYF(0)); + return true; + } + } + /* Normal keys are done, now prepare foreign keys. @@ -16319,6 +16375,8 @@ bool prepare_fields_and_keys(THD *thd, const dd::Table *src_table, TABLE *table, key_type = KEYTYPE_UNIQUE; } else if (key_info->flags & HA_FULLTEXT) key_type = KEYTYPE_FULLTEXT; + else if (key_info->flags & HA_VECTOR) + key_type = KEYTYPE_VECTOR; else key_type = KEYTYPE_MULTIPLE; diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 3c06ee5adfb1..7a9e0fdf1bd2 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -4655,7 +4655,8 @@ bool btr_validate_index( /* Full Text index are implemented by auxiliary tables, not the B-tree */ - if (dict_index_is_online_ddl(index) || (index->type & DICT_FTS)) { + if (dict_index_is_online_ddl(index) || + ((index->type & DICT_FTS) || dict_index_is_vector(index))) { return (true); } diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 182bf0051a85..eebc751240ef 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -438,7 +438,7 @@ dberr_t dict_create_index_tree_in_mem(dict_index_t *index, trx_t *trx) { DBUG_EXECUTE_IF("ib_dict_create_index_tree_fail", return (DB_OUT_OF_MEMORY);); - if (index->type == DICT_FTS) { + if ((index->type & DICT_FTS) || dict_index_is_vector(index)) { /* FTS index does not need an index tree */ return (DB_SUCCESS); } diff --git a/storage/innobase/dict/dict0dd.cc b/storage/innobase/dict/dict0dd.cc index f77448568965..63798ec908d4 100644 --- a/storage/innobase/dict/dict0dd.cc +++ b/storage/innobase/dict/dict0dd.cc @@ -1979,7 +1979,7 @@ void dd_visit_keys_with_too_long_parts( std::function visitor) { for (uint key_num = 0; key_num < table->s->keys; key_num++) { const KEY &key = table->key_info[key_num]; - if (!(key.flags & (HA_SPATIAL | HA_FULLTEXT))) { + if (!(key.flags & (HA_SPATIAL | HA_FULLTEXT | HA_VECTOR))) { for (unsigned i = 0; i < key.user_defined_key_parts; i++) { const KEY_PART_INFO *key_part = &key.key_part[i]; if (max_part_len < key_part->length) { @@ -2909,7 +2909,7 @@ MY_COMPILER_DIAGNOSTIC_POP() */ static inline uint16_t get_index_prefix_len(const KEY &key, const KEY_PART_INFO *key_part) { - if (key.flags & (HA_SPATIAL | HA_FULLTEXT)) { + if (key.flags & (HA_SPATIAL | HA_FULLTEXT | HA_VECTOR)) { return 0; } @@ -2949,6 +2949,7 @@ template const dict_index_t *dd_find_index( uint key_num) { const KEY &key = form->key_info[key_num]; ulint type = 0; + bool is_vector = false; unsigned n_fields = key.user_defined_key_parts; unsigned n_uniq = n_fields; @@ -2969,6 +2970,10 @@ template const dict_index_t *dd_find_index( ut_ad(!table->is_intrinsic()); type = DICT_FTS; n_uniq = 0; + } else if (key.flags & HA_VECTOR) { + ut_ad(!table->is_intrinsic()); + is_vector = true; + n_uniq = 0; } else if (key_num == form->primary_key) { ut_ad(key.flags & HA_NOSAME); ut_ad(n_uniq > 0); @@ -2977,11 +2982,13 @@ template const dict_index_t *dd_find_index( type = (key.flags & HA_NOSAME) ? DICT_UNIQUE : 0; } - ut_ad(!!(type & DICT_FTS) == (n_uniq == 0)); + ut_ad((!!(type & DICT_FTS) || is_vector) == (n_uniq == 0)); dict_index_t *index = dict_mem_index_create(table->name.m_name, key.name, 0, type, n_fields); + index->is_vector_index = is_vector; + index->n_uniq = n_uniq; DBUG_EXECUTE_IF("ib_create_table_fail_at_create_index", @@ -5206,8 +5213,8 @@ dict_table_t *dd_open_table_one(dd::cache::Dictionary_client *client, } ut_ad(root > 1); - ut_ad(index->type & DICT_FTS || root != FIL_NULL || - dict_table_is_discarded(m_table)); + ut_ad((index->type & DICT_FTS) || dict_index_is_vector(index) || + root != FIL_NULL || dict_table_is_discarded(m_table)); ut_ad(id != 0); index->page = root; index->space = sid; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 6710d5abff14..56344f30c2ee 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -218,6 +218,9 @@ static dict_index_t *dict_index_build_internal_fts( dict_table_t *table, /*!< in: table */ dict_index_t *index); /*!< in: user representation of an FTS index */ +static dict_index_t *dict_index_build_internal_vec(dict_table_t *table, + dict_index_t *index); + /** Removes an index from the dictionary cache. */ static void dict_index_remove_from_cache_low( dict_table_t *table, /*!< in/out: table */ @@ -2217,7 +2220,7 @@ static bool dict_index_too_big_for_tree(const dict_table_t *table, const dict_index_t *new_index) { /* FTS index consists of auxiliary tables, they shall be excluded from index row size check */ - if (new_index->type & DICT_FTS) { + if ((new_index->type & DICT_FTS) || dict_index_is_vector(new_index)) { return (false); } @@ -2446,6 +2449,8 @@ dberr_t dict_index_add_to_cache_w_vcol(dict_table_t *table, dict_index_t *index, if (index->type == DICT_FTS) { new_index = dict_index_build_internal_fts(table, index); + } else if (dict_index_is_vector(index)) { + new_index = dict_index_build_internal_vec(table, index); } else if (index->is_clustered()) { new_index = dict_index_build_internal_clust(table, index); } else { @@ -3284,6 +3289,37 @@ static dict_index_t *dict_index_build_internal_fts( return (new_index); } + +static dict_index_t *dict_index_build_internal_vec( + dict_table_t *table, /*!< in: table */ + dict_index_t *index) /*!< in: user representation of a vector index */ +{ + ut_ad(table && index); + ut_ad(dict_index_is_vector(index)); + ut_ad(!dict_sys_mutex_own()); + ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); + + /* Create a new index */ + auto new_index = + dict_mem_index_create(table->name.m_name, index->name, index->space, + index->type, index->n_fields); + new_index->is_vector_index = index->is_vector_index; + + /* Copy other relevant data from the old index struct to the new + struct: it inherits the values */ + + new_index->n_user_defined_cols = index->n_fields; + + new_index->id = index->id; + + /* Copy fields from index to new_index */ + dict_index_copy(new_index, index, table, 0, index->n_fields); + + new_index->n_uniq = 0; + new_index->cached = true; + + return (new_index); +} /*====================== FOREIGN KEY PROCESSING ========================*/ /** Checks if a table is referenced by foreign keys. @@ -3371,7 +3407,8 @@ NOT NULL */ while (index != nullptr) { if (types_idx != index && !(index->type & DICT_FTS) && - !dict_index_is_spatial(index) && !index->to_be_dropped && + !dict_index_is_vector(index) && !dict_index_is_spatial(index) && + !index->to_be_dropped && (!(index->uncommitted && ((index->online_status == ONLINE_INDEX_ABORTED_DROPPED) || (index->online_status == ONLINE_INDEX_ABORTED)))) && @@ -3628,6 +3665,7 @@ bool dict_index_check_search_tuple( ut_ad(index->page >= FSP_FIRST_INODE_PAGE_NO); ut_ad(dtuple_check_typed(tuple)); ut_ad(!(index->type & DICT_FTS)); + ut_ad(!dict_index_is_vector(index)); return true; } #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 69a2c99df38e..a32e70bdfd5e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3231,7 +3231,7 @@ ha_innobase::ha_innobase(handlerton *hton, TABLE_SHARE *table_arg) HA_ATTACHABLE_TRX_COMPATIBLE | HA_CAN_INDEX_VIRTUAL_GENERATED_COLUMN | HA_DESCENDING_INDEX | HA_MULTI_VALUED_KEY_SUPPORT | HA_BLOB_PARTIAL_UPDATE | HA_SUPPORTS_GEOGRAPHIC_GEOMETRY_COLUMN | - HA_SUPPORTS_DEFAULT_EXPRESSION | HA_ONLINE_ANALYZE), + HA_SUPPORTS_DEFAULT_EXPRESSION | HA_ONLINE_ANALYZE | HA_CAN_VECTOR), m_start_of_scan(), m_stored_select_lock_type(LOCK_NONE_UNSET), m_mysql_has_locked() {} @@ -5750,13 +5750,13 @@ static int innodb_init(void *p) { innobase_hton->lock_hton_log = innobase_lock_hton_log; innobase_hton->unlock_hton_log = innobase_unlock_hton_log; innobase_hton->collect_hton_log_info = innobase_collect_hton_log_info; - innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS | - HTON_SUPPORTS_FOREIGN_KEYS | HTON_SUPPORTS_ATOMIC_DDL | - HTON_CAN_RECREATE | HTON_SUPPORTS_SECONDARY_ENGINE | - HTON_SUPPORTS_TABLE_ENCRYPTION | - HTON_SUPPORTS_GENERATED_INVISIBLE_PK | - HTON_SUPPORTS_BULK_LOAD | HTON_SUPPORTS_SQL_FK | - HTON_SUPPORTS_ONLINE_BACKUPS | HTON_SUPPORTS_COMPRESSED_COLUMNS; + innobase_hton->flags = + HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS | + HTON_SUPPORTS_ATOMIC_DDL | HTON_CAN_RECREATE | + HTON_SUPPORTS_SECONDARY_ENGINE | HTON_SUPPORTS_TABLE_ENCRYPTION | + HTON_SUPPORTS_GENERATED_INVISIBLE_PK | HTON_SUPPORTS_BULK_LOAD | + HTON_SUPPORTS_SQL_FK | HTON_SUPPORTS_ONLINE_BACKUPS | + HTON_SUPPORTS_COMPRESSED_COLUMNS; // TODO(WL9440): to be enabled when distance scan is implemented in innodb. //| HTON_SUPPORTS_DISTANCE_SCAN; @@ -8303,7 +8303,7 @@ int ha_innobase::open(const char *name, int, uint open_flags, dict_table_autoinc_unlock(ib_table); } - /* Set plugin parser for fulltext index */ + /* Set plugin parser for fulltext index / handle vector index. */ for (uint i = 0; i < table->s->keys; i++) { if (table->key_info[i].flags & HA_USES_PARSER) { dict_index_t *index = innobase_get_index(i); @@ -11019,7 +11019,7 @@ int ha_innobase::index_read( : HA_ERR_TABLE_DEF_CHANGED; } - if (index->type & DICT_FTS) { + if ((index->type & DICT_FTS) || dict_index_is_vector(index)) { return HA_ERR_KEY_NOT_FOUND; } @@ -12871,6 +12871,7 @@ inline int create_index( index = dict_mem_index_create(table_name, key->name, 0, ind_type, key->user_defined_key_parts); + index->is_vector_index = (key->flags & HA_VECTOR); innodb_session_t *&priv = thd_to_innodb_session(trx->mysql_thd); dict_table_t *handler = priv->lookup_table_handler(table_name); @@ -12964,6 +12965,7 @@ inline int create_index( } ut_ad(key->flags & HA_FULLTEXT || !(index->type & DICT_FTS)); + ut_ad((key->flags & HA_VECTOR) || !dict_index_is_vector(index)); multi_val_idx = ((index->type & DICT_MULTI_VALUE) == DICT_MULTI_VALUE); @@ -14059,7 +14061,7 @@ bool create_table_info_t::innobase_table_flags() { if (fts_doc_id_index_bad) { goto index_bad; } - } else if (key->flags & HA_SPATIAL) { + } else if (key->flags & (HA_SPATIAL | HA_VECTOR)) { assert(~m_create_info->options & (HA_LEX_CREATE_TMP_TABLE | HA_LEX_CREATE_INTERNAL_TMP_TABLE)); } @@ -15656,16 +15658,51 @@ int innobase_truncate::exec() { template int innobase_truncate::exec(); template int innobase_truncate::exec(); -/** Check if a column is the only column in an index. -@param[in] index data dictionary index -@param[in] column the column to look for -@return whether the column is the only column in the index */ +/** + Check if a column is the only column in an index. + + @param[in] index data dictionary index + @param[in] column the column to look for + + @return Whether the column is the only column in the index. +*/ static bool dd_is_only_column(const dd::Index *index, const dd::Column *column) { return (index->elements().size() == 1 && &(*index->elements().begin())->column() == column); } +/** + Check if an index uses marker-based vector metadata. + + @param[in] index data dictionary index + + @return Whether the index is a vector index. +*/ +static bool dd_is_vector_index(const dd::Index *index) { + if (index->algorithm() != dd::Index::IA_SE_SPECIFIC || + index->type() != dd::Index::IT_MULTIPLE) { + return false; + } + + uint visible_elements = 0; + for (const dd::Index_element *elem : index->elements()) { + if (elem->is_hidden()) continue; + + visible_elements++; + + const dd::Properties &col_options = elem->column().options(); + bool is_vector_column = false; + if (col_options.exists("vector_index") && + !col_options.get("vector_index", &is_vector_column) && + is_vector_column) { + return visible_elements == 1; + } + } + + return false; +} + /** Add hidden columns and indexes to an InnoDB table definition. @param[in,out] dd_table data dictionary cache object @return error number @@ -15693,6 +15730,10 @@ int ha_innobase::get_extra_columns_and_keys(const HA_CREATE_INFO *, fts_doc_id_index = i; } + if (dd_is_vector_index(i)) { + continue; + } + switch (i->algorithm()) { case dd::Index::IA_SE_SPECIFIC: ut_d(ut_error); @@ -18334,7 +18375,8 @@ void ha_innobase::info_low_key(uint flag, const dict_table_t *ib_table) { /* We do not maintain stats for fulltext or spatial indexes. Thus, we can't calculate pct_cached below because we need dict_index_t::stat_n_leaf_pages for that. See dict_stats_should_ignore_index(). */ - if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL)) { + if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL) || + (key->flags & HA_VECTOR)) { pct_cached = IN_MEMORY_ESTIMATE_UNKNOWN; } else { pct_cached = index_pct_cached(index); @@ -18352,7 +18394,8 @@ void ha_innobase::info_low_key(uint flag, const dict_table_t *ib_table) { } for (ulong j = 0; j < key->actual_key_parts; j++) { - if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL)) { + if ((key->flags & HA_FULLTEXT) || (key->flags & HA_SPATIAL) || + (key->flags & HA_VECTOR)) { /* The record per key does not apply to FTS or Spatial indexes. */ key->set_records_per_key(j, 1.0f); continue; @@ -18678,7 +18721,8 @@ static bool innobase_get_index_column_cardinality( } DEBUG_SYNC(thd, "innodb.after_init_check"); - if (index->type & (DICT_FTS | DICT_SPATIAL)) { + if ((index->type & (DICT_FTS | DICT_SPATIAL)) || + dict_index_is_vector(index)) { /* For these indexes innodb_rec_per_key is fixed as 1.0 */ *cardinality = ib_table->stat_n_rows; diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index abb3b0a8d35d..b7e415c43c31 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -134,6 +134,22 @@ static inline ulint dict_index_is_spatial( return (index->type & DICT_SPATIAL); } +/** + Check whether the index is a vector index. + + @param[in] index Index. + + @return Nonzero for vector index, zero for other indexes +*/ +static inline ulint dict_index_is_vector( + const dict_index_t *index) /*!< in: index */ +{ + ut_ad(index); + ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); + + return (index->is_vector()); +} + /** Check whether the index contains a virtual column @param[in] index index @return nonzero for the index has virtual column, zero for other indexes */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 9a083c0c1c74..a0a6640848df 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -113,7 +113,7 @@ constexpr uint32_t DICT_SDI = 256; constexpr uint32_t DICT_MULTI_VALUE = 512; /** number of bits used for SYS_INDEXES.TYPE */ -constexpr uint32_t DICT_IT_BITS = 10; +constexpr uint32_t DICT_IT_BITS = 11; /** @} */ #if 0 /* not implemented, retained for history */ @@ -1206,6 +1206,9 @@ struct dict_index_t { /** if the index is an hidden index */ bool hidden; + + /** true if this is a vector index according to DD metadata */ + bool is_vector_index; #endif /* !UNIV_HOTBACKUP */ /** list of indexes of the table */ @@ -1337,6 +1340,17 @@ struct dict_index_t { return (type & DICT_MULTI_VALUE); } + /** + Check whether the index is a vector index. + + @return true if the index is a vector index, false otherwise + */ + [[nodiscard]] bool is_vector() const { + ut_ad(magic_n == DICT_INDEX_MAGIC_N); + + return (is_vector_index); + } + /** Returns the minimum data size of an index record. @return minimum data size in bytes */ ulint get_min_size() const { diff --git a/storage/innobase/include/dict0mem.ic b/storage/innobase/include/dict0mem.ic index 65993ac857a0..7b3bb7dbf788 100644 --- a/storage/innobase/include/dict0mem.ic +++ b/storage/innobase/include/dict0mem.ic @@ -76,6 +76,7 @@ static inline void dict_mem_fill_index_struct( index->allow_duplicates = false; index->nulls_equal = false; index->disable_ahi = false; + index->is_vector_index = false; index->last_ins_cur = nullptr; index->last_sel_cur = nullptr; #ifndef UNIV_HOTBACKUP diff --git a/storage/temptable/src/handler.cc b/storage/temptable/src/handler.cc index 7c3e8e0058bd..929e811a6972 100644 --- a/storage/temptable/src/handler.cc +++ b/storage/temptable/src/handler.cc @@ -877,6 +877,7 @@ ulong Handler::index_flags(uint index_no, uint, bool) const { case HA_KEY_ALG_SE_SPECIFIC: case HA_KEY_ALG_RTREE: case HA_KEY_ALG_FULLTEXT: + case HA_KEY_ALG_VECTOR: flags = 0; break; } diff --git a/storage/temptable/src/table.cc b/storage/temptable/src/table.cc index 3c87e1e7728c..ef3504aee96f 100644 --- a/storage/temptable/src/table.cc +++ b/storage/temptable/src/table.cc @@ -290,6 +290,7 @@ void Table::indexes_create() { case HA_KEY_ALG_SE_SPECIFIC: case HA_KEY_ALG_RTREE: case HA_KEY_ALG_FULLTEXT: + case HA_KEY_ALG_VECTOR: DBUG_ABORT(); } }