Features/vector storage

LICENSE

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
 Copyright (c) 2014-2022 Mauricio David
+Copyright (c) 2024�2025 Matthew Hurley
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -19,3 +20,8 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+
+This is a fork of the original LiteDB project by Mauricio David.
+This fork adds vector and graph-based memory capabilities and cognitive agent support.
+Original LiteDB license terms apply.

LiteDB.Benchmarks/Benchmarks/Queries/QueryWithVectorSimilarity.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using LiteDB.Benchmarks.Models;
+using LiteDB.Benchmarks.Models.Generators;
+
+namespace LiteDB.Benchmarks.Benchmarks.Queries
+{
+    [BenchmarkCategory(Constants.Categories.QUERIES)]
+    public class QueryWithVectorSimilarity : BenchmarkBase
+    {
+        private ILiteCollection<FileMetaBase> _fileMetaCollection;
+        private float[] _queryVector;
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            File.Delete(DatabasePath);
+
+            DatabaseInstance = new LiteDatabase(ConnectionString());
+            _fileMetaCollection = DatabaseInstance.GetCollection<FileMetaBase>();
+            _fileMetaCollection.EnsureIndex(fileMeta => fileMeta.ShouldBeShown);
+
+            var rnd = new Random();
+
+            _fileMetaCollection.Insert(FileMetaGenerator<FileMetaBase>.GenerateList(DatasetSize)); // executed once per each N value
+
+            _queryVector = Enumerable.Range(0, 128).Select(_ => (float)rnd.NextDouble()).ToArray();
+
+            DatabaseInstance.Checkpoint();
+        }
+
+        [Benchmark]
+        public List<FileMetaBase> WhereNear_Filter()
+        {
+            return _fileMetaCollection.Query()
+                .WhereNear(x => x.Vectors, _queryVector, maxDistance: 0.5)
+                .ToList();
+        }
+
+        [Benchmark]
+        public List<FileMetaBase> TopKNear_OrderLimit()
+        {
+            return _fileMetaCollection.Query()
+                .TopKNear(x => x.Vectors, _queryVector, k: 10)
+                .ToList();
+        }
+    }
+}

LiteDB.Benchmarks/Models/FileMetaBase.cs

@@ -28,6 +28,8 @@ public class FileMetaBase
 
         public bool ShouldBeShown { get; set; }
 
+        public float[] Vectors { get; set; }
+
         public virtual bool IsValid => ValidFrom == null || ValidFrom <= DateTimeOffset.UtcNow && ValidTo == null || ValidTo > DateTimeOffset.UtcNow;
     }
 }

LiteDB.Benchmarks/Models/Generators/FileMetaGenerator.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace LiteDB.Benchmarks.Models.Generators
 {
@@ -18,7 +19,8 @@ private static T Generate()
                 Title = $"Document-{docGuid}",
                 MimeType = "application/pdf",
                 IsFavorite = _random.Next(10) >= 9,
-                ShouldBeShown = _random.Next(10) >= 7
+                ShouldBeShown = _random.Next(10) >= 7,
+                Vectors =  Enumerable.Range(0, 128).Select(_ => (float)_random.NextDouble()).ToArray()
             };
 
             if (_random.Next(10) >= 5)

LiteDB.Tests/BsonValue/BsonVector_Tests.cs

@@ -0,0 +1,169 @@
+using System.Collections.Generic;
+using System.Linq;
+using FluentAssertions;
+using Xunit;
+
+namespace LiteDB.Tests.BsonValue_Types;
+
+public class BsonVector_Tests
+{
+
+    private static readonly Collation _collation = Collation.Binary;
+    private static readonly BsonDocument _root = new BsonDocument();
+
+    [Fact]
+    public void BsonVector_RoundTrip_Success()
+    {
+        var original = new BsonDocument
+        {
+            ["vec"] = new BsonVector(new float[] { 1.0f, 2.5f, -3.75f })
+        };
+
+        var bytes = BsonSerializer.Serialize(original);
+        var deserialized = BsonSerializer.Deserialize(bytes);
+
+        var vec = deserialized["vec"].AsVector;
+        Assert.Equal(3, vec.Length);
+        Assert.Equal(1.0f, vec[0]);
+        Assert.Equal(2.5f, vec[1]);
+        Assert.Equal(-3.75f, vec[2]);
+    }
+
+    private class VectorDoc
+    {
+        public int Id { get; set; }
+        public float[] Embedding { get; set; }
+    }
+
+    [Fact]
+    public void VectorSim_Query_ReturnsExpectedNearest()
+    {
+        using var db = new LiteDatabase(":memory:");
+        var col = db.GetCollection<VectorDoc>("vectors");
+
+        // Insert vectorized documents
+        col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } });
+        col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } });
+        col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } });
+
+        // Create index on the embedding field (if applicable to your implementation)
+        col.EnsureIndex("Embedding", "Embedding");
+
+        // Query: Find vectors nearest to [1, 0]
+        var target = new float[] { 1.0f, 0.0f };
+        var results = col.Query()
+            .WhereNear(r => r.Embedding, [1.0f, 0.0f], maxDistance:.28)
+            .ToList();
+
+        results.Should().NotBeEmpty();
+        results.Select(x => x.Id).Should().Contain(1);
+        results.Select(x => x.Id).Should().NotContain(2);
+        results.Select(x => x.Id).Should().NotContain(3); // too far away
+    }
+
+    [Fact]
+    public void VectorSim_ExpressionQuery_WorksViaSQL()
+    {
+        using var db = new LiteDatabase(":memory:");
+        var col = db.GetCollection("vectors");
+
+        col.Insert(new BsonDocument
+        {
+            ["_id"] = 1,
+            ["Embedding"] = new BsonVector(new float[] { 1.0f, 0.0f })
+        });
+        col.Insert(new BsonDocument
+        {
+            ["_id"] = 2,
+            ["Embedding"] = new BsonVector(new float[] { 0.0f, 1.0f })
+        });
+        col.Insert(new BsonDocument
+        {
+            ["_id"] = 3,
+            ["Embedding"] = new BsonVector(new float[] { 1.0f, 1.0f })
+        });
+
+        //var query = "SELECT * FROM vectors WHERE vector_sim([1.0, 0.0], $.Embedding) < 0.3";
+        //var results = db.Execute(query).ToList();
+        var expr = BsonExpression.Create("VECTOR_SIM($.Embedding, [1.0, 0.0]) < 0.25");
+
+        var results = db
+            .GetCollection("vectors")
+            .Find(expr)
+            .ToList();
+
+        results.Select(r => r["_id"].AsInt32).Should().Contain(1);
+        results.Select(r => r["_id"].AsInt32).Should().NotContain(2);
+        results.Select(r => r["_id"].AsInt32).Should().NotContain(3); // cosine ~ 0.293
+    }
+
+    [Fact]
+    public void VectorSim_ReturnsZero_ForIdenticalVectors()
+    {
+        var left = new BsonArray { 1.0, 0.0 };
+        var right = new BsonVector(new float[] { 1.0f, 0.0f });
+
+        var result = BsonExpressionMethods.VECTOR_SIM(left, right);
+
+        Assert.NotNull(result);
+        Assert.True(result.IsDouble);
+        Assert.Equal(0.0, result.AsDouble, 6); // Cosine distance = 0.0
+    }
+
+    [Fact]
+    public void VectorSim_ReturnsOne_ForOrthogonalVectors()
+    {
+        var left = new BsonArray { 1.0, 0.0 };
+        var right = new BsonVector(new float[] { 0.0f, 1.0f });
+
+        var result = BsonExpressionMethods.VECTOR_SIM(left, right);
+
+        Assert.NotNull(result);
+        Assert.True(result.IsDouble);
+        Assert.Equal(1.0, result.AsDouble, 6); // Cosine distance = 1.0
+    }
+
+    [Fact]
+    public void VectorSim_ReturnsNull_ForInvalidInput()
+    {
+        var left = new BsonArray { "a", "b" };
+        var right = new BsonVector(new float[] { 1.0f, 0.0f });
+
+        var result = BsonExpressionMethods.VECTOR_SIM(left, right);
+
+        Assert.True(result.IsNull);
+    }
+
+    [Fact]
+    public void VectorSim_ReturnsNull_ForMismatchedLengths()
+    {
+        var left = new BsonArray { 1.0, 2.0, 3.0 };
+        var right = new BsonVector(new float[] { 1.0f, 2.0f });
+
+        var result = BsonExpressionMethods.VECTOR_SIM(left, right);
+
+        Assert.True(result.IsNull);
+    }
+
+
+    [Fact]
+    public void VectorSim_TopK_ReturnsCorrectOrder()
+    {
+        using var db = new LiteDatabase(":memory:");
+        var col = db.GetCollection<VectorDoc>("vectors");
+
+        col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } }); // sim = 0.0
+        col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } }); // sim = 1.0
+        col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } }); // sim ≈ 0.293
+
+        var target = new float[] { 1.0f, 0.0f };
+
+        var results = col.Query()
+            .TopKNear(x => x.Embedding, target, 2)
+            .ToList();
+
+        var ids = results.Select(r => r.Id).ToList();
+        ids.Should().BeEquivalentTo(new[] { 1, 3 }, options => options.WithStrictOrdering());
+    }
+
+}

LiteDB.Tests/Document/Decimal_Tests.cs

@@ -10,8 +10,8 @@ public void BsonValue_New_Decimal_Type()
         {
             var d0 = 0m;
             var d1 = 1m;
-            var dmin = new BsonValue(decimal.MinValue);
-            var dmax = new BsonValue(decimal.MaxValue);
+            var dmin = new LiteDB.BsonValue(decimal.MinValue);
+            var dmax = new LiteDB.BsonValue(decimal.MaxValue);
 
             JsonSerializer.Serialize(d0).Should().Be("{\"$numberDecimal\":\"0\"}");
             JsonSerializer.Serialize(d1).Should().Be("{\"$numberDecimal\":\"1\"}");

LiteDB.Tests/Document/Implicit_Tests.cs

@@ -13,9 +13,9 @@ public void BsonValue_Implicit_Convert()
             long l = long.MaxValue;
             ulong u = ulong.MaxValue;
 
-            BsonValue bi = i;
-            BsonValue bl = l;
-            BsonValue bu = u;
+            LiteDB.BsonValue bi = i;
+            LiteDB.BsonValue bl = l;
+            LiteDB.BsonValue bu = u;
 
             bi.IsInt32.Should().BeTrue();
             bl.IsInt64.Should().BeTrue();
@@ -35,7 +35,7 @@ public void BsonDocument_Inner()
             customer["CreateDate"] = DateTime.Now;
             customer["Phones"] = new BsonArray { "8000-0000", "9000-000" };
             customer["IsActive"] = true;
-            customer["IsAdmin"] = new BsonValue(true);
+            customer["IsAdmin"] = new LiteDB.BsonValue(true);
             customer["Address"] = new BsonDocument
             {
                 ["Street"] = "Av. Protasio Alves"

LiteDB.sln

@@ -13,6 +13,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteDB.Benchmarks", "LiteDB
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteDB.Stress", "LiteDB.Stress\LiteDB.Stress.csproj", "{FFBC5669-DA32-4907-8793-7B414279DA3B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmarks", "benchmarks", "{33DF8B19-4DE5-4076-8A36-CE05AB8FDA78}"
+	ProjectSection(SolutionItems) = preProject
+		LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.csv = LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.csv
+		LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.html = LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.html
+		LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.txt = LiteDB.Benchmarks\bin\Release\net472\BenchmarkDotNet.Artifacts\results\LiteDB.Benchmarks.Benchmarks.Queries.QueryWithVectorSimilarity-report.txt
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -43,6 +52,9 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{33DF8B19-4DE5-4076-8A36-CE05AB8FDA78} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {45099B85-2AE1-434B-913A-6AD14FD3AF4A}
 	EndGlobalSection

LiteDB/Client/Database/ILiteQueryable.cs

@@ -26,6 +26,22 @@ public interface ILiteQueryable<T> : ILiteQueryableResult<T>
 
         ILiteQueryableResult<BsonDocument> Select(BsonExpression selector);
         ILiteQueryableResult<K> Select<K>(Expression<Func<T, K>> selector);
+
+        /// <summary>
+        /// Filters documents where the given vector field is within cosine distance from the target vector.
+        /// </summary>
+        ILiteQueryable<T> WhereNear(string vectorField, float[] target, double maxDistance);
+
+        /// <summary>
+        /// Immediately returns documents nearest to the target vector based on cosine distance.
+        /// </summary>
+        IEnumerable<T> FindNearest(string vectorField, float[] target, double maxDistance);
+
+        ILiteQueryable<T> WhereNear<K>(Expression<Func<T, K>> field, float[] target, double maxDistance);
+        ILiteQueryableResult<T> TopKNear<K>(Expression<Func<T, K>> field, float[] target, int k);
+        ILiteQueryableResult<T> TopKNear(string field, float[] target, int k);
+        ILiteQueryableResult<T> TopKNear(BsonExpression fieldExpr, float[] target, int k);
+
     }
 
     public interface ILiteQueryableResult<T>