Merge pull request #11 from JKamsker/codex/add-vector-aware-index-support

Integrate vector indexes into planner and tests

Author: JKamsker

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
 
     steps:
       - name: Check out repository
@@ -22,18 +25,40 @@ jobs:
         run: dotnet restore LiteDB.sln
 
       - name: Build
-        run: dotnet build LiteDB.sln --configuration Release --no-restore
+        run: dotnet build LiteDB.sln --configuration Release --no-restore /p:DefineConstants=TESTING
 
-      - name: Test
-        timeout-minutes: 5
-        continue-on-error: true
-        run: dotnet test LiteDB.sln --configuration Release --no-build --verbosity normal --settings tests.runsettings --logger "trx;LogFileName=TestResults.trx" --logger "console;verbosity=detailed"
+      - name: Test (with hang diagnostics)
+        timeout-minutes: 8
+        env:
+          VSTEST_TESTHOST_HANG_DUMP_TYPE: full
+          VSTEST_TESTHOST_HANG_DUMP_FOLDER: ${{ github.workspace }}/hangdumps
+          DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+          DOTNET_NOLOGO: 1
+        run: >-
+          dotnet test LiteDB.sln
+          --configuration Release
+          --no-build
+          --verbosity normal
+          --settings tests.ci.runsettings
+          --logger "trx;LogFileName=TestResults.trx"
+          --logger "console;verbosity=detailed"
+          --blame-hang --blame-hang-timeout 00:02:00
+          --diag "TestHost.diag.log;tracelevel=verbose"
+          /p:DefineConstants=TESTING
 
       - name: Upload test results
-        if: always()
         uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: test-results
+          path: "**/*TestResults*.trx"
+
+      - name: Upload hang dumps (if any)
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: hangdumps
           path: |
-            **/TestResults.trx
-            **/TestResults/**/*
+            hangdumps
+            **/TestResults/**/*.dmp
+          if-no-files-found: ignore

.github/workflows/publish-prerelease.yml

@@ -10,6 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
 
     steps:
       - name: Check out repository
@@ -22,30 +25,74 @@ jobs:
         with:
           dotnet-version: 8.0.x
 
+      - name: Set prerelease version
+        id: version
+        run: |
+          # Use SemVer-compliant prerelease without leading zeros
+          echo "package_version=6.0.0-prerelease.${{ github.run_number }}" >> "$GITHUB_OUTPUT"
+          echo "Version set to 6.0.0-prerelease.${{ github.run_number }}"
+
       - name: Restore
         run: dotnet restore LiteDB.sln
 
-      - name: Build
-        run: dotnet build LiteDB.sln --configuration Release --no-restore
+      - name: Test (with hang diagnostics)
+        timeout-minutes: 8
+        env:
+          VSTEST_TESTHOST_HANG_DUMP_TYPE: full
+          VSTEST_TESTHOST_HANG_DUMP_FOLDER: ${{ github.workspace }}/hangdumps
+          DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+          DOTNET_NOLOGO: 1
+        run: >-
+          dotnet test LiteDB.sln
+          --configuration Release
+          --verbosity normal
+          --settings tests.ci.runsettings
+          --logger "trx;LogFileName=TestResults.trx"
+          --logger "console;verbosity=detailed"
+          --blame-hang --blame-hang-timeout 00:02:00
+          --diag "TestHost.diag.log;tracelevel=verbose"
+          /p:DefineConstants=TESTING
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-results
+          path: "**/*TestResults*.trx"
+      
+      - name: Upload hang dumps (if any)
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: hangdumps
+          path: |
+            hangdumps
+            **/TestResults/**/*.dmp
+          if-no-files-found: ignore
 
-      - name: Test
-        timeout-minutes: 5
-        run: dotnet test LiteDB.sln --configuration Release --no-build --verbosity normal --settings tests.runsettings --logger "trx;LogFileName=TestResults.trx" --logger "console;verbosity=detailed"
+      - name: Build
+        if: success()
+        run: dotnet build LiteDB/LiteDB.csproj --configuration Release --no-restore -p:Version=${{ steps.version.outputs.package_version }} -p:PackageVersion=${{ steps.version.outputs.package_version }} -p:MinVerSkip=true
 
       - name: Pack
-        run: |
-          dotnet pack LiteDB/LiteDB.csproj --configuration Release --no-build -o artifacts
+        if: success()
+        run: dotnet pack LiteDB/LiteDB.csproj --configuration Release --no-build -o artifacts -p:PackageVersion=${{ steps.version.outputs.package_version }} -p:Version=${{ steps.version.outputs.package_version }} -p:MinVerSkip=true
 
-      - name: Capture package version
-        id: version
-        run: |
-          PACKAGE_PATH=$(ls artifacts/LiteDB.*.nupkg | head -n 1)
-          PACKAGE_FILENAME=$(basename "$PACKAGE_PATH")
-          PACKAGE_VERSION=${PACKAGE_FILENAME#LiteDB.}
-          PACKAGE_VERSION=${PACKAGE_VERSION%.nupkg}
-          echo "package_version=${PACKAGE_VERSION}" >> "$GITHUB_OUTPUT"
+      - name: Publish GitHub prerelease
+        if: success()
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.version.outputs.package_version }}
+          name: LiteDB ${{ steps.version.outputs.package_version }}
+          generate_release_notes: true
+          prerelease: true
+          files: artifacts/*.nupkg
+          target_commitish: ${{ github.sha }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Retrieve secrets from Bitwarden
+        if: success()
         uses: bitwarden/sm-action@v2
         with:
           access_token: ${{ secrets.BW_ACCESS_TOKEN }}
@@ -54,16 +101,7 @@ jobs:
             265b2fb6-2cf0-4859-9bc8-b24c00ab4378 > NUGET_API_KEY
 
       - name: Push package to NuGet
+        if: success()
         run: |
           dotnet nuget push "artifacts/*.nupkg" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json --skip-duplicate
 
-      - name: Publish GitHub prerelease
-        uses: softprops/action-gh-release@v2
-        with:
-          tag_name: v${{ steps.version.outputs.package_version }}
-          name: LiteDB ${{ steps.version.outputs.package_version }}
-          generate_release_notes: true
-          prerelease: true
-          files: artifacts/*.nupkg
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

ConsoleApp1/Program.cs

@@ -27,7 +27,7 @@
 {
     using (var db = new LiteEngine(settings))
     {
-#if DEBUG
+#if DEBUG || TESTING
         db.SimulateDiskWriteFail = (page) =>
         {
             var p = new BasePage(page);

LiteDB.Benchmarks/Benchmarks/Queries/QueryWithVectorSimilarity.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using BenchmarkDotNet.Attributes;
+using LiteDB;
 using LiteDB.Benchmarks.Models;
 using LiteDB.Benchmarks.Models.Generators;
 
@@ -12,6 +13,7 @@ namespace LiteDB.Benchmarks.Benchmarks.Queries
     public class QueryWithVectorSimilarity : BenchmarkBase
     {
         private ILiteCollection<FileMetaBase> _fileMetaCollection;
+        private ILiteCollection<FileMetaBase> _unindexedCollection;
         private float[] _queryVector;
 
         [GlobalSetup]
@@ -20,12 +22,18 @@ public void GlobalSetup()
             File.Delete(DatabasePath);
 
             DatabaseInstance = new LiteDatabase(ConnectionString());
-            _fileMetaCollection = DatabaseInstance.GetCollection<FileMetaBase>();
+            _fileMetaCollection = DatabaseInstance.GetCollection<FileMetaBase>("withIndex");
+            _unindexedCollection = DatabaseInstance.GetCollection<FileMetaBase>("withoutIndex");
+
             _fileMetaCollection.EnsureIndex(fileMeta => fileMeta.ShouldBeShown);
+            _unindexedCollection.EnsureIndex(fileMeta => fileMeta.ShouldBeShown);
+            _fileMetaCollection.EnsureIndex(fileMeta => fileMeta.Vectors, new VectorIndexOptions(128));
 
             var rnd = new Random();
+            var data = FileMetaGenerator<FileMetaBase>.GenerateList(DatasetSize);
 
-            _fileMetaCollection.Insert(FileMetaGenerator<FileMetaBase>.GenerateList(DatasetSize)); // executed once per each N value
+            _fileMetaCollection.Insert(data); // executed once per each N value
+            _unindexedCollection.Insert(data);
 
             _queryVector = Enumerable.Range(0, 128).Select(_ => (float)rnd.NextDouble()).ToArray();
 
@@ -34,6 +42,14 @@ public void GlobalSetup()
 
         [Benchmark]
         public List<FileMetaBase> WhereNear_Filter()
+        {
+            return _unindexedCollection.Query()
+                .WhereNear(x => x.Vectors, _queryVector, maxDistance: 0.5)
+                .ToList();
+        }
+
+        [Benchmark]
+        public List<FileMetaBase> WhereNear_Filter_Indexed()
         {
             return _fileMetaCollection.Query()
                 .WhereNear(x => x.Vectors, _queryVector, maxDistance: 0.5)
@@ -42,6 +58,14 @@ public List<FileMetaBase> WhereNear_Filter()
 
         [Benchmark]
         public List<FileMetaBase> TopKNear_OrderLimit()
+        {
+            return _unindexedCollection.Query()
+                .TopKNear(x => x.Vectors, _queryVector, k: 10)
+                .ToList();
+        }
+
+        [Benchmark]
+        public List<FileMetaBase> TopKNear_OrderLimit_Indexed()
         {
             return _fileMetaCollection.Query()
                 .TopKNear(x => x.Vectors, _queryVector, k: 10)