flechasdb下载flechasdb源代码下载

flechasdb

其他源码

1.0.0

下载

Flechasdb

Flechasdb品牌

您手中无服务器友好的矢量数据库。

flechasdb软件包是用Rust编写的FlechasDB系统的核心库。

Flechasdb
- FlechasDB系统
  - 核心特征和进度
- 安装FlechasDB
- 使用FlechasDB
  - 构建矢量数据库
  - 加载和查询矢量数据库
  - 加载和查询矢量数据库Async
- 基准
- API文档
- 算法和结构
  - indexivfpq
  - K-均值++
  - 数据库结构
- 发展
  - 建立图书馆
  - 生成文档
- 类似的项目
- 执照

FlechasDB系统

FlechasDB系统的目标是成为一个完美拟合无服务器环境的矢量数据库。 FlechasDB系统的信条很简单。它不需要持续运行的专用服务器。

核心特征和进度

*：由另一个软件包flechasdb-s3提供。

安装FlechasDB

尚未出版板条箱。请在您的货物中添加以下行Cargo.toml文件：

[ dependencies ]
flechasdb = { git = " https://github.com/codemonger-io/flechasdb.git " }

使用FlechasDB

构建矢量数据库

这是从随机生成的向量构建向量数据库的Exmple。

 use rand :: Rng ;

use flechasdb :: db :: build :: {
    DatabaseBuilder ,
    proto :: serialize_database ,
} ;
use flechasdb :: io :: LocalFileSystem ;
use flechasdb :: vector :: BlockVectorSet ;

fn main ( ) {
    const M : usize = 100000 ; // number of vectors
    const N : usize = 1536 ; // vector size
    const D : usize = 12 ; // number of subvector divisions
    const P : usize = 100 ; // number of partitions
    const C : usize = 256 ; // number of clusters for product quantization
    let time = std :: time :: Instant :: now ( ) ;
    let mut data : Vec < f32 > = Vec :: with_capacity ( M * N ) ;
    unsafe { data . set_len ( M * N ) ; }
    let mut rng = rand :: thread_rng ( ) ;
    rng . fill ( & mut data [ .. ] ) ;
    let vs = BlockVectorSet :: chunk ( data , N . try_into ( ) . unwrap ( ) ) . unwrap ( ) ;
    println ! ( "prepared data in {} s" , time . elapsed ( ) . as_secs_f32 ( ) ) ;
    let time = std :: time :: Instant :: now ( ) ;
    let mut db = DatabaseBuilder :: new ( vs )
        . with_partitions ( P . try_into ( ) . unwrap ( ) )
        . with_divisions ( D . try_into ( ) . unwrap ( ) )
        . with_clusters ( C . try_into ( ) . unwrap ( ) )
        . build ( )
        . unwrap ( ) ;
    println ! ( "built database in {} s" , time . elapsed ( ) . as_secs_f32 ( ) ) ;
    for i in 0 .. M {
        db . set_attribute_at ( i , ( "datum_id" , i as u64 ) ) . unwrap ( ) ;
    }
    let time = std :: time :: Instant :: now ( ) ;
    serialize_database ( & db , & mut LocalFileSystem :: new ( "testdb" ) ) . unwrap ( ) ;
    println ! ( "serialized database in {} s" , time . elapsed ( ) . as_secs_f32 ( ) ) ;
}

您可以在examples/build-random文件夹中找到完整的示例。

仅供参考：在我的机器上花了一段时间（Apple M1 Pro，32GB RAM，1TB SSD）。

 prepared data in 0.9123601 s
built database in 906.51526 s
serialized database in 0.14329213 s

加载和查询矢量数据库

这是加载向量数据库并查询K-Nearest邻居（K-NN）随机生成的向量的示例。

 use rand :: Rng ;
use std :: env :: args ;
use std :: path :: Path ;

use flechasdb :: db :: stored :: { Database , LoadDatabase } ;
use flechasdb :: io :: LocalFileSystem ;

fn main ( ) {
    const K : usize = 10 ; // k-nearest neighbors
    const NPROBE : usize = 5 ; // number of partitions to query
    let time = std :: time :: Instant :: now ( ) ;
    let db_path = args ( ) . nth ( 1 ) . expect ( "no db path given" ) ;
    let db_path = Path :: new ( & db_path ) ;
    let db = Database :: < f32 , _ > :: load_database (
        LocalFileSystem :: new ( db_path . parent ( ) . unwrap ( ) ) ,
        db_path . file_name ( ) . unwrap ( ) . to_str ( ) . unwrap ( ) ,
    ) . unwrap ( ) ;
    println ! ( "loaded database in {} s" , time . elapsed ( ) . as_secs_f32 ( ) ) ;
    let mut qv : Vec < f32 > = Vec :: with_capacity ( db . vector_size ( ) ) ;
    unsafe { qv . set_len ( db . vector_size ( ) ) ; }
    let mut rng = rand :: thread_rng ( ) ;
    rng . fill ( & mut qv [ .. ] ) ;
    for r in 0 .. 2 { // second round should run faster
        let time = std :: time :: Instant :: now ( ) ;
        let results = db . query (
            & qv ,
            K . try_into ( ) . unwrap ( ) ,
            NPROBE . try_into ( ) . unwrap ( ) ,
        ) . unwrap ( ) ;
        println ! ( "[{}] queried k-NN in {} s" , r , time . elapsed ( ) . as_secs_f32 ( ) ) ;
        let time = std :: time :: Instant :: now ( ) ;
        for ( i , result ) in results . into_iter ( ) . enumerate ( ) {
            // getting attributes will incur additional disk reads
            let attr = result . get_attribute ( "datum_id" ) . unwrap ( ) ;
            println ! (
                " t {}: partition={}, approx. distance²={}, datum_id={:?}" ,
                i ,
                result . partition_index ,
                result . squared_distance ,
                attr ,
            ) ;
        }
        println ! (
            "[{}] printed results in {} s" ,
            r ,
            time . elapsed ( ) . as_secs_f32 ( ) ,
        ) ;
    }
}

您可以在examples/query-sync文件夹中找到完整的示例。

仅供参考：我的机器上输出（Apple M1 Pro，32GB RAM，1TB SSD）：

 loaded database in 0.000142083 s
[0] queried k-NN in 0.0078015 s
	0: partition=95, approx. distance²=126.23533, datum_id=Some(Uint64(90884))
	1: partition=29, approx. distance²=127.76597, datum_id=Some(Uint64(30864))
	2: partition=95, approx. distance²=127.80611, datum_id=Some(Uint64(75236))
	3: partition=56, approx. distance²=127.808174, datum_id=Some(Uint64(27890))
	4: partition=25, approx. distance²=127.85459, datum_id=Some(Uint64(16417))
	5: partition=95, approx. distance²=127.977425, datum_id=Some(Uint64(70910))
	6: partition=25, approx. distance²=128.06209, datum_id=Some(Uint64(3237))
	7: partition=95, approx. distance²=128.22603, datum_id=Some(Uint64(41942))
	8: partition=79, approx. distance²=128.26906, datum_id=Some(Uint64(89799))
	9: partition=25, approx. distance²=128.27995, datum_id=Some(Uint64(6593))
[0] printed results in 0.003392833 s
[1] queried k-NN in 0.001475625 s
	0: partition=95, approx. distance²=126.23533, datum_id=Some(Uint64(90884))
	1: partition=29, approx. distance²=127.76597, datum_id=Some(Uint64(30864))
	2: partition=95, approx. distance²=127.80611, datum_id=Some(Uint64(75236))
	3: partition=56, approx. distance²=127.808174, datum_id=Some(Uint64(27890))
	4: partition=25, approx. distance²=127.85459, datum_id=Some(Uint64(16417))
	5: partition=95, approx. distance²=127.977425, datum_id=Some(Uint64(70910))
	6: partition=25, approx. distance²=128.06209, datum_id=Some(Uint64(3237))
	7: partition=95, approx. distance²=128.22603, datum_id=Some(Uint64(41942))
	8: partition=79, approx. distance²=128.26906, datum_id=Some(Uint64(89799))
	9: partition=25, approx. distance²=128.27995, datum_id=Some(Uint64(6593))
[1] printed results in 0.0000215 s

加载和查询矢量数据库（async）

这是一个异步加载向量数据库并查询K-NN随机生成的向量的一个示例。

 use rand :: Rng ;
use std :: env :: args ;
use std :: path :: Path ;

use flechasdb :: asyncdb :: io :: LocalFileSystem ;
use flechasdb :: asyncdb :: stored :: { Database , LoadDatabase } ;

# [ tokio :: main ]
async fn main ( ) {
    const K : usize = 10 ; // k-nearest neighbors
    const NPROBE : usize = 5 ; // number of partitions to search
    let time = std :: time :: Instant :: now ( ) ;
    let db_path = args ( ) . nth ( 1 ) . expect ( "missing db path" ) ;
    let db_path = Path :: new ( & db_path ) ;
    let db = Database :: < f32 , _ > :: load_database (
        LocalFileSystem :: new ( db_path . parent ( ) . unwrap ( ) ) ,
        db_path . file_name ( ) . unwrap ( ) . to_str ( ) . unwrap ( ) ,
    ) . await . unwrap ( ) ;
    println ! ( "loaded database in {} s" , time . elapsed ( ) . as_secs_f32 ( ) ) ;
    let mut qv = Vec :: with_capacity ( db . vector_size ( ) ) ;
    unsafe { qv . set_len ( db . vector_size ( ) ) ; }
    let mut rng = rand :: thread_rng ( ) ;
    rng . fill ( & mut qv [ .. ] ) ;
    for r in 0 .. 2 { // second round should run faster
        let time = std :: time :: Instant :: now ( ) ;
        let results = db . query (
            & qv ,
            K . try_into ( ) . unwrap ( ) ,
            NPROBE . try_into ( ) . unwrap ( ) ,
        ) . await . unwrap ( ) ;
        println ! ( "[{}] queried k-NN in {} s" , r , time . elapsed ( ) . as_secs_f32 ( ) ) ;
        let time = std :: time :: Instant :: now ( ) ;
        for ( i , result ) in results . into_iter ( ) . enumerate ( ) {
            // getting attributes will incur additional disk reads
            let attr = result . get_attribute ( "datum_id" ) . await . unwrap ( ) ;
            println ! (
                " t {}: partition={}, approx. distance²={}, datum_id={:?}" ,
                i ,
                result . partition_index ,
                result . squared_distance ,
                attr ,
            ) ;
        }
        println ! (
            "[{}] printed results in {} s" ,
            r ,
            time . elapsed ( ) . as_secs_f32 ( ) ,
        ) ;
    }
}

完整的示例是examples/query-async文件夹。

仅供参考：我的机器上输出（Apple M1 Pro，32GB RAM，1TB SSD）：

 loaded database in 0.000170959 s
[0] queried k-NN in 0.008041208 s
	0: partition=67, approx. distance²=128.50703, datum_id=Some(Uint64(69632))
	1: partition=9, approx. distance²=129.98079, datum_id=Some(Uint64(73093))
	2: partition=9, approx. distance²=130.10867, datum_id=Some(Uint64(7536))
	3: partition=20, approx. distance²=130.29523, datum_id=Some(Uint64(67750))
	4: partition=67, approx. distance²=130.71976, datum_id=Some(Uint64(77054))
	5: partition=9, approx. distance²=130.80556, datum_id=Some(Uint64(93180))
	6: partition=9, approx. distance²=130.90681, datum_id=Some(Uint64(22473))
	7: partition=9, approx. distance²=130.94006, datum_id=Some(Uint64(40167))
	8: partition=67, approx. distance²=130.9795, datum_id=Some(Uint64(8590))
	9: partition=9, approx. distance²=131.03018, datum_id=Some(Uint64(53138))
[0] printed results in 0.00194175 s
[1] queried k-NN in 0.000789417 s
	0: partition=67, approx. distance²=128.50703, datum_id=Some(Uint64(69632))
	1: partition=9, approx. distance²=129.98079, datum_id=Some(Uint64(73093))
	2: partition=9, approx. distance²=130.10867, datum_id=Some(Uint64(7536))
	3: partition=20, approx. distance²=130.29523, datum_id=Some(Uint64(67750))
	4: partition=67, approx. distance²=130.71976, datum_id=Some(Uint64(77054))
	5: partition=9, approx. distance²=130.80556, datum_id=Some(Uint64(93180))
	6: partition=9, approx. distance²=130.90681, datum_id=Some(Uint64(22473))
	7: partition=9, approx. distance²=130.94006, datum_id=Some(Uint64(40167))
	8: partition=67, approx. distance²=130.9795, datum_id=Some(Uint64(8590))
	9: partition=9, approx. distance²=131.03018, datum_id=Some(Uint64(53138))
[1] printed results in 0.000011084 s

基准

更现实的数据有一个基准。

API文档

https://codemonger-io.github.io/flechasdb/api/flechasdb/

算法和结构

indexivfpq

flechasdb实现了本文中描述的索引。

K-均值++

flechasdb实现K-均值++，以初始化NäiveK-均值聚类的质心。

数据库结构

TBD

发展

建立图书馆

cargo build

生成文档

cargo doc --lib --no-deps --release

类似的项目

松果
完全管理的矢量数据库。
米尔维斯
具有许多功能的开源矢量数据库。
Lancedb
他们的功能之一也是无服务器，它们的核心用Rust编写！

执照

麻省理工学院

Codemonger的以下材料已根据CC BY-SA 4.0许可：

Flechasdb品牌

展开

附加信息

版本 1.0.0
类型其他源码
更新时间 2025-05-30
大小 1.88MB
来自于 Github

flechasdb

Flechasdb

FlechasDB系统

核心特征和进度

安装FlechasDB

使用FlechasDB

构建矢量数据库

加载和查询矢量数据库

加载和查询矢量数据库（async）

基准

API文档

算法和结构

indexivfpq

K-均值++

数据库结构

发展

建立图书馆

生成文档

类似的项目

执照

Google Dorks

shepherd

mongo express

hidusbf

Free Algorithms Books

markdownpedia

chat.petals.dev

GPT Prompt Templates

GPTyped

Google Dorks

shepherd

mongo express

Google Dorks

shepherd

mongo express