diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 034850a..f302f42 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -152,8 +152,55 @@ export default defineConfig( link: "/extensions/", collapsed: false, items: [ - { text: "Bundled", link: "/extensions/bundled/" }, - { text: "Additional", link: "/extensions/additional/" } + { + text: "Bundled", + link: "/extensions/bundled/", + collapsed: true, + items: [ + { text: "auto_explain", link: "/extensions/bundled/auto-explain" }, + { text: "btree_gin", link: "/extensions/bundled/btree_gin" }, + { text: "citext", link: "/extensions/bundled/citext" }, + { text: "dblink", link: "/extensions/bundled/dblink" }, + { text: "fuzzystrmatch", link: "/extensions/bundled/fuzzystrmatch" }, + { text: "gp_array_agg", link: "/extensions/bundled/gp_array_agg" }, + { text: "gp_check_functions", link: "/extensions/bundled/gp_check_functions" }, + { text: "gp_exttable_fdw", link: "/extensions/bundled/gp_exttable_fdw" }, + { text: "gp_legacy_string_agg", link: "/extensions/bundled/gp_legacy_string_agg" }, + { text: "gp_parallel_retrieve_cursor", link: "/extensions/bundled/gp_parallel_retrieve_cursor" }, + { text: "gp_percentile_agg", link: "/extensions/bundled/gp_percentile_agg" }, + { text: "gp_pitr", link: "/extensions/bundled/gp_pitr" }, + { text: "gp_sparse_vector", link: "/extensions/bundled/gp_sparse_vector" }, + { text: "gp_subtransaction_overflow", link: "/extensions/bundled/gp_subtransaction_overflow" }, + { text: "greenplum_fdw", link: "/extensions/bundled/greenplum_fdw" }, + { text: "hstore", link: "/extensions/bundled/hstore" }, + { text: "ip4r", link: "/extensions/bundled/ip4r" }, + { text: "isn", link: "/extensions/bundled/isn" }, + { text: "ltree", link: "/extensions/bundled/ltree" }, + { text: "orafce", link: "/extensions/bundled/orafce_ref" }, + { text: "pageinspect", link: "/extensions/bundled/pageinspect" }, + { text: "pg_buffercache", link: "/extensions/bundled/pg_buffercache" }, + { text: "pg_cron", link: "/extensions/bundled/pg_cron" }, + { text: "pg_hint_plan", link: "/extensions/bundled/pg_hint_plan" }, + { text: "pg_stat_statements", link: "/extensions/bundled/pg_stat_statements" }, + { text: "pg_trgm", link: "/extensions/bundled/pg_trgm" }, + { text: "pgcrypto", link: "/extensions/bundled/pgcrypto" }, + { text: "postgres_fdw", link: "/extensions/bundled/postgres_fdw" }, + { text: "sslinfo", link: "/extensions/bundled/sslinfo" }, + { text: "tablefunc", link: "/extensions/bundled/tablefunc" }, + { text: "timestamp9", link: "/extensions/bundled/timestamp9" }, + { text: "tsm_system_rows", link: "/extensions/bundled/tsm_system_rows" }, + { text: "tsm_system_time", link: "/extensions/bundled/tsm_system_time" }, + { text: "uuid-ossp", link: "/extensions/bundled/uuid-ossp" } + ], + }, + { + text: "Additional", + link: "/extensions/additional/", + collapsed: true, + items: [ + { text: "whpg-vector", link: "/extensions/additional/pgvector/" } + ], + } ], } ] diff --git a/docs/extensions/additional/index.md b/docs/extensions/additional/index.md index eeed74e..a8678e4 100644 --- a/docs/extensions/additional/index.md +++ b/docs/extensions/additional/index.md @@ -3,6 +3,8 @@ title: Additional extensions description: Open source extensions for WarehousePG that require a separate package install on each host. --- -These extensions require installing a package on each host in your cluster before you can enable them. See [Installing Extensions](/docs/7x/install_guide/additional_modules/index.md) for installation steps. +These extensions require a separate package install on each host in your WarehousePG cluster before you can enable them, and availability varies by version. -No additional extensions are available yet. +| Extension | Description | 7.x | +| --- | --- | :---: | +| [whpg-vector](pgvector/index.md) | Provides vector similarity search capabilities for WarehousePG that enable searching, storing, and querying machine-learning-generated embeddings at scale. | ✓ | diff --git a/docs/extensions/additional/pgvector/index.md b/docs/extensions/additional/pgvector/index.md new file mode 100644 index 0000000..0303d38 --- /dev/null +++ b/docs/extensions/additional/pgvector/index.md @@ -0,0 +1,25 @@ +--- +title: whpg-vector +description: Learn how to store, query, and index embeddings in WarehousePG using the whpg-vector extension. +redirects: + - /docs/7x/ref_guide/modules/extensions/pgvector +navigation: + - release_notes + - installing + - using +--- + +::: info Version availability + +whpg-vector is available starting with WarehousePG 7.3.0. +::: + +The whpg-vector extension provides high-performance vector similarity search capabilities for WarehousePG (WHPG). By representing complex data (like text, images, or audio) as machine-learning-generated embeddings, you can perform semantic and syntactic searches at scale. + +The WarehousePG whpg-vector extension is equivalent to version 0.7.4 of the pgvector extension used with PostgreSQL. See [Limitations](using.md#limitations) to understand the limitations of the WarehousePG version of the extension. + +To get started with whpg-vector, see: + +- [Release notes](release_notes.md) +- [Installing whpg-vector](installing.md) +- [Using whpg-vector](using.md) diff --git a/docs/extensions/additional/pgvector/installing.md b/docs/extensions/additional/pgvector/installing.md new file mode 100644 index 0000000..2920f10 --- /dev/null +++ b/docs/extensions/additional/pgvector/installing.md @@ -0,0 +1,52 @@ +--- +title: Installing whpg-vector +description: Install the whpg-vector extension for WarehousePG from its repository. +--- + +Install whpg-vector on each host in your WarehousePG cluster, as it doesn't ship as part of the WarehousePG server package. + +## Supported platforms + + + +::: info Version availability +whpg-vector is available starting with WarehousePG 7.3.0. +::: + +## Installing the extension + +Download and install whpg-vector from the [whpg-vector repository](https://github.com/warehouse-pg/whpg-vector). + + + +## Registering whpg-vector + +After you install whpg-vector on every host, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION vector; +``` + +## Verifying the installation + +Confirm that `vector` is registered in the database: + +```sql +\dx vector +``` + +## Upgrading whpg-vector + +After you install the new whpg-vector package on every host, update the extension in each database where you registered it: + +```sql +ALTER EXTENSION vector UPDATE; +``` + +## Uninstalling whpg-vector + + + +```sql +DROP EXTENSION vector; +``` diff --git a/docs/extensions/additional/pgvector/release_notes.md b/docs/extensions/additional/pgvector/release_notes.md new file mode 100644 index 0000000..87aaa2e --- /dev/null +++ b/docs/extensions/additional/pgvector/release_notes.md @@ -0,0 +1,221 @@ +--- +title: whpg-vector release notes +navTitle: Release notes +description: Release notes for the whpg-vector extension for WarehousePG. +--- + +The whpg-vector documentation describes the latest version, including minor releases and patches. These release notes cover what's new in each release. + +## 0.7.4 + +Release date: September 20, 2026 + +- Fixed locking for parallel HNSW index builds. +- Fixed compilation error with GCC 14 on i386 when SSE2 is not enabled. + +## 0.7.3 + +- Fixed `failed to add index item` error with `sparsevec`. +- Fixed compilation error with FreeBSD ARM. +- Fixed compilation warning with MSVC and Postgres 16. + +## 0.7.2 + +- Fixed initialization fork for indexes on unlogged tables. + +## 0.7.1 + +- Improved performance of on-disk HNSW index builds. +- Fixed `undefined symbol` error with GCC 8. +- Fixed compilation error with universal binaries on Mac. +- Fixed compilation warning with Clang < 14. + +## 0.7.0 + +- Added `halfvec` type. +- Added `sparsevec` type. +- Added support for indexing `bit` type. +- Added support for indexing L1 distance with HNSW. +- Added `binary_quantize` function. +- Added `hamming_distance` function. +- Added `jaccard_distance` function. +- Added `l2_normalize` function. +- Added `subvector` function. +- Added concatenate operator for vectors. +- Added CPU dispatching for distance functions on Linux x86-64. +- Updated comparison operators to support vectors with different dimensions. + +## 0.6.2 + +- Reduced lock contention with parallel HNSW index builds. + +## 0.6.1 + +- Fixed error with `ANALYZE` and vectors with different dimensions. +- Fixed segmentation fault with `shared_preload_libraries`. +- Fixed vector subtraction being marked as commutative. + +## 0.6.0 + +If upgrading with Postgres 12 or Docker, see the upgrade notes in the pgvector repository. + +- Added support for parallel index builds for HNSW. +- Added validation for GUC parameters. +- Changed storage for `vector` from `extended` to `external`. +- Improved performance of HNSW. +- Reduced memory usage for HNSW index builds. +- Reduced WAL generation for HNSW index builds. +- Fixed error with logical replication. +- Fixed `invalid memory alloc request size` error with HNSW index builds. +- Moved Docker image to pgvector org. +- Added Docker tags for each supported version of Postgres. +- Dropped support for Postgres 11. + +## 0.5.1 + +- Improved performance of HNSW index builds. +- Added check for MVCC-compliant snapshot for index scans. + +## 0.5.0 + +- Added HNSW index type. +- Added support for parallel index builds for IVFFlat. +- Added `l1_distance` function. +- Added element-wise multiplication for vectors. +- Added `sum` aggregate. +- Improved performance of distance functions. +- Fixed out of range results for cosine distance. +- Fixed results for `NULL` and `NaN` distances for IVFFlat. + +## 0.4.4 + +- Improved error message for malformed vector literal. +- Fixed segmentation fault with text input. +- Fixed consecutive delimiters with text input. + +## 0.4.3 + +- Improved cost estimation. +- Improved support for spaces with text input. +- Fixed infinite and `NaN` values with binary input. +- Fixed infinite values with vector addition and subtraction. +- Fixed infinite values with list centers. +- Fixed compilation error when `float8` is pass by reference. +- Fixed compilation error on PowerPC. +- Fixed segmentation fault with index creation on i386. + +## 0.4.2 + +- Added notice when index created with little data. +- Fixed dimensions check for some direct function calls. +- Fixed installation error with Postgres 12.0-12.2. + +## 0.4.1 + +- Improved performance of cosine distance. +- Fixed index scan count. + +## 0.4.0 + +If upgrading with Postgres < 13, see the upgrade note in the pgvector repository. + +- Changed text representation for `vector` elements to match `real`. +- Changed storage for `vector` from `plain` to `extended`. +- Increased max dimensions for `vector` from 1024 to 16000. +- Increased max dimensions for index from 1024 to 2000. +- Improved accuracy of text parsing for certain inputs. +- Added `avg` aggregate for `vector`. +- Added experimental support for Windows. +- Dropped support for Postgres 10. + +## 0.3.2 + +- Fixed `invalid memory alloc request size` error. + +## 0.3.1 + +If upgrading from 0.2.7 or 0.3.0, recreate all `ivfflat` indexes after upgrading to ensure all data is indexed. + +- Fixed issue with inserts silently corrupting `ivfflat` indexes (introduced in 0.2.7). +- Fixed segmentation fault with index creation when `lists` > 6500. + +## 0.3.0 + +- Added support for Postgres 15. +- Dropped support for Postgres 9.6. + +## 0.2.7 + +- Fixed `unexpected data beyond EOF` error. + +## 0.2.6 + +- Improved performance of index creation for Postgres < 12. + +## 0.2.5 + +- Reduced memory usage during index creation. +- Fixed index creation exceeding `maintenance_work_mem`. +- Fixed error with index creation when `lists` > 1600. + +## 0.2.4 + +- Added support for parallel vacuum. +- Fixed issue with index not reusing space. + +## 0.2.3 + +- Added indexing progress for Postgres 12+. +- Improved interrupt handling during index creation. + +## 0.2.2 + +- Fixed compilation error on Mac ARM. + +## 0.2.1 + +- Fixed `operator is not unique` error. + +## 0.2.0 + +- Added support for Postgres 14. + +## 0.1.8 + +- Added cast for `vector` to `real[]`. + +## 0.1.7 + +- Added cast for `numeric[]` to `vector`. + +## 0.1.6 + +- Fixed segmentation fault with `COUNT`. + +## 0.1.5 + +- Reduced memory usage during index creation. + +## 0.1.4 + +- Fixed kmeans for inner product. +- Fixed `multiple definition` error with GCC 10. + +## 0.1.3 + +- Added Dockerfile. +- Fixed version. + +## 0.1.2 + +- Vectorized distance calculations. +- Improved cost estimation. + +## 0.1.1 + +- Added binary representation for `COPY`. +- Marked functions as `PARALLEL SAFE`. + +## 0.1.0 + +- First release. diff --git a/docs/extensions/additional/pgvector/using.md b/docs/extensions/additional/pgvector/using.md new file mode 100644 index 0000000..e068293 --- /dev/null +++ b/docs/extensions/additional/pgvector/using.md @@ -0,0 +1,322 @@ +--- +title: Using whpg-vector +navTitle: Using +description: Store, query, index, and tune embeddings in WarehousePG with whpg-vector. +--- + +You can use whpg-vector to search, store, and query embeddings in WarehousePG. + +## About the vector types, operators, and functions + +whpg-vector introduces the `vector` data type along with specialized index access methods `ivfflat` and `hnsw`. These tools enable both exact and approximate nearest neighbor (ANN) searches using various distance metrics. + +### vector data type + +The `vector` data type represents an n-dimensional coordinate. Each `vector` takes `4 * dimensions + 8` bytes of storage. Each element is a single precision floating-point number (similar to the `real` type in WarehousePG), and all of the elements must be finite (no `NaN`, `Infinity`, or `-Infinity`). Vectors can have up to 16,000 dimensions. + +### vector operators + +whpg-vector provides the following operators for the `vector` data type: + +| Operator | Description | +| -------- | --------------------------- | +| + | Element-wise addition | +| - | Element-wise subtraction | +| \* | Element-wise multiplication | +| <-> | Euclidean distance | +| <#> | Negative inner product | +| <=> | Cosine distance | + +::: info Note + + Because WarehousePG supports only `ASC` (ascending) order, the `<#>` returns the negative inner product to ensure the smallest value appears first. +::: + +### vector functions + +whpg-vector provides the following functions for the `vector` data type: + +| Function name | Return type | Description | +| --------------------------------- | ------------------ | -------------------------------- | +| `cosine_distance(vector, vector)` | `double precision` | Computes the cosine distance | +| `inner_product(vector, vector)` | `double precision` | Computes the inner product | +| `l2_distance(vector, vector)` | `double precision` | Computes the Euclidean distance | +| `l1_distance(vector, vector)` | `double precision` | Computes the taxicab distance | +| `vector_dims(vector)` | `integer` | Returns the number of dimensions | +| `vector_norm(vector)` | `double precision` | Computes the Euclidean norm | + +### vector aggregate functions + +whpg-vector provides the following aggregate functions for the `vector` data type: + +| Function | Return type | Description | +| ------------- | ----------- | --------------------------------------- | +| `avg(vector)` | `vector` | Computes the arithmetic mean | +| `sum(vector)` | `vector` | Computes the sum of the vector elements | + +## Storing embeddings + +You can define `vector` columns with a specific dimension count to ensure data integrity. + +- Create a table with a `vector` column with three dimensions: + + ```sql + CREATE TABLE items (id bigserial PRIMARY KEY); + ``` + +- Add a column to an existing table: + + ```sql + ALTER TABLE items ADD COLUMN embedding vector(3); + ``` + +## Performing common DML operations + +- Insert data: + + ```sql + INSERT INTO items (embedding) VALUES ('[1,2,3]'), ('[4,5,6]'); + ``` + +- Upsert: + + ```sql + INSERT INTO items (id, embedding) VALUES (1, '[1,2,3]') + ON CONFLICT (id) DO UPDATE SET embedding = EXCLUDED.embedding; + ``` + +- Update and delete data: + + ```sql + UPDATE items SET embedding = '[1,2,3]' WHERE id = 1; + DELETE FROM items WHERE id = 1; + ``` + +- Insert and update data: + + ```sql + INSERT INTO items (embedding) VALUES ('[1.1, 2.2, 3.3]'); + UPDATE items SET embedding = '[4.4, 5.5, 6.6]' WHERE id = 1; + ``` + +## Querying embeddings + +- Get the nearest neighbors to a `vector` by L2 distance: + + ```sql + SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; + ``` + +- Get the nearest neighbors to a row: + + ```sql + SELECT * FROM items WHERE id != 1 + ORDER BY embedding <-> (SELECT embedding FROM items WHERE id = 1) LIMIT 5; + ``` + +- Filter by a certain distance: + + ```sql + SELECT * FROM items WHERE embedding <-> '[3,1,2]' < 5; + ``` + +## Evaluating embedding distance + +- Inner product (multiply by -1): + + ```sql + SELECT (embedding <#> '[3,1,2]') * -1 AS inner_product FROM items; + ``` + +- Cosine similarity (1 - distance): + + ```sql + SELECT 1 - (embedding <=> '[3,1,2]') AS cosine_similarity FROM items; + ``` + +## Aggregating embeddings + +- Average the vectors in a table: + + ```sql + SELECT AVG(embedding) FROM items; + ``` + +- Average a group of vectors in a table: + + ```sql + SELECT category_id, AVG(embedding) FROM items GROUP BY category_id; + ``` + +## Indexing embeddings + +By default, whpg-vector performs an exact nearest neighbor search, which guarantees perfect recall. To trade some recall for performance, you can use an Approximate Nearest Neighbor (ANN) index. + +::: info Note +Unlike typical indexes, adding an approximate index may change query results. +::: + +When you create an index for an embedding, you use the `lists` parameter to specify the number of clusters created during index creation. Each cluster is a partition of the data set. + +For optimal search accuracy: + +- Create the index after the table has data. +- Choose an appropriate `lists` value. Use `rows / 1000` for up to 1M rows and `sqrt(rows)` for over 1M rows. +- whpg-vector provides a `probes` parameter that you can set at query time to specify the number of regions to search during a query. When querying, specify an appropriate number of `probes`. A high number is better for recall, a low number is better for speed. A reasonable initial value is `sqrt(lists)`. + +**Inverted File Flat (IVFFlat) indexes** + +IVFFlat indexes are best for large datasets where memory is limited. It partitions vectors into clusters (lists). You can index a `vector` that has up to 2,000 dimensions. + +- Create an index on the L2 distance: + + ```sql + CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100); + ``` + +- Create an index on the inner product: + + ```sql + CREATE INDEX ON items USING ivfflat (embedding vector_ip_ops) WITH (lists = 100); + ``` + +- Create an index on the cosine distance: + + ```sql + CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100); + ``` + +- Specify the number of `probes` (1 by default) to specify how many regions to search. + + ```sql + SET ivfflat.probes = 10; + ``` + +- Use `SET LOCAL` inside a transaction block to set `probes` for a single query: + + ```sql + BEGIN; + SET LOCAL ivfflat.probes = 10; + SELECT ... --- your queries + COMMIT; + ``` + +**Hierarchical navigable small world (HNSW) indexes** + +HNSW indexes build a graph for faster search and higher recall than IVFFlat, supporting up to 2,000 dimensions. + +- Create an index on the L2 distance: + + ```sql + CREATE INDEX ON items USING hnsw (embedding vector_l2_ops); + ``` + +- Create an index on the inner product: + + ```sql + CREATE INDEX ON items USING hnsw (embedding vector_ip_ops); + ``` + +- Create an index on the cosine distance: + + ```sql + CREATE INDEX ON items USING hnsw (embedding vector_cosine_ops); + ``` + +HNSW indexes support the following parameters: + +- `m`: Specifies the maximum number of connections per layer (16 by default). +- `ef_construction`: Specifies the size of the dynamic candidate list for constructing the graph (64 by default). + +For example: + +```sql +CREATE INDEX ON items USING hnsw (embedding vector_l2_ops) WITH (m = 16, ef_construction = 64); +``` + +You can specify a custom size for the dynamic candidate list for a search: + +```sql +SET hnsw.ef_search = 100; +``` + +Higher value provides better recall at the cost of speed. The default size of the candidate list is 40. For example: + +```sql +BEGIN; +SET LOCAL hnsw.ef_search = 100; +SELECT ... --- your queries +COMMIT; +``` + +## Advanced usage + +**Parallel index creation** + +WarehousePG version 7.3.1 and later supports parallel index creation for HNSW and IVFFlat indexes through the `max_parallel_maintenance_workers` configuration parameter. By default, `max_parallel_maintenance_workers` is set to `2`, which allows two parallel workers to run per segment during index creation. To disable parallel workers, set `max_parallel_maintenance_workers` to `0`. + +**Viewing indexing progress** + +You can check index creation progress in WarehousePG as described in [CREATE INDEX Progress Reporting](/docs/7x/admin_guide/managing/progress_reporting.md#create_index_progress): + +```sql +SELECT phase, tuples_done, tuples_total FROM gp_stat_progress_create_index; +``` + +**Filtering** + +Index nearest neighbor queries with a `WHERE` clause: + +```sql +SELECT * FROM items WHERE category_id = 123 ORDER BY embedding <-> '[3,1,2]' LIMIT 5; +``` + +For approximate search, create a partial index on the `vector` column: + +```sql +CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 100) + WHERE (category_id = 123); +``` + +Use [partitioning](/docs/7x/admin_guide/ddl/ddl-partition/index.md) to perform an approximate search on many different values of the `WHERE` columns: + +```sql +CREATE TABLE items (embedding vector(3), category_id int) PARTITION BY LIST(category_id); +``` + +**Performing hybrid search** + +Use whpg-vector together with [full-text search](/docs/7x/admin_guide/query/textsearch/index.md) for a hybrid search: + +```sql +SELECT id, content FROM items, to_tsquery('hello & search') query + WHERE textsearch @@ query ORDER BY ts_rank_cd(textsearch, query) DESC LIMIT 5; +``` + +**Performance tuning and debugging** + +Use `EXPLAIN ANALYZE` to debug performance: + +```sql +EXPLAIN ANALYZE SELECT * FROM items ORDER BY embedding <-> '[3,1,2]' LIMIT 5; +``` + +If vectors are normalized to length 1, use inner product for the best performance: + +```sql +SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5; +``` + +To speed up queries with an index, increase the number of inverted `lists` (at the expense of recall): + +```sql +CREATE INDEX ON items USING ivfflat (embedding vector_l2_ops) WITH (lists = 1000); +``` + + +## Limitations + +whpg-vector for WarehousePG has the following limitations: + +- The WarehousePG query optimizer (ORCA) doesn't support `ivfflat` and `hnsw` indexes. Queries using these indexes will fall back to the Postgres-based planner. +- The size of a vector index can be larger than the size of the base table.