Skip to content

Commit 96e6372

Browse files
SommerEngineeringSommerEngineering
authored
Added a common augmentation interface & default implementation (#288)
1 parent f01cf49 commit 96e6372

File tree

19 files changed

+249
-62
lines changed

19 files changed

+249
-62
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ Things we are currently working on:
2020
- [ ] Runtime: Integration of the vector database [LanceDB](https://github.com/lancedb/lancedb)
2121
- [ ] App: Implement the continuous process of vectorizing data
2222
- [x] ~~App: Define a common retrieval context interface for the integration of RAG processes in chats (PR [#281](https://github.com/MindWorkAI/AI-Studio/pull/281), [#284](https://github.com/MindWorkAI/AI-Studio/pull/284), [#286](https://github.com/MindWorkAI/AI-Studio/pull/286), [#287](https://github.com/MindWorkAI/AI-Studio/pull/287))~~
23-
- [ ] App: Define a common augmentation interface for the integration of RAG processes in chats
23+
- [x] ~~App: Define a common augmentation interface for the integration of RAG processes in chats (PR [#288](https://github.com/MindWorkAI/AI-Studio/pull/288))~~
2424
- [x] ~~App: Integrate data sources in chats (PR [#282](https://github.com/MindWorkAI/AI-Studio/pull/282))~~
2525

2626

‎app/MindWork AI Studio.sln.DotSettings‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MSG/@EntryIndexedValue">MSG</s:String>
88
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RAG/@EntryIndexedValue">RAG</s:String>
99
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UI/@EntryIndexedValue">UI</s:String>
10+
<s:Boolean x:Key="/Default/UserDictionary/Words/=agentic/@EntryIndexedValue">True</s:Boolean>
1011
<s:Boolean x:Key="/Default/UserDictionary/Words/=groq/@EntryIndexedValue">True</s:Boolean>
1112
<s:Boolean x:Key="/Default/UserDictionary/Words/=ollama/@EntryIndexedValue">True</s:Boolean>
1213
<s:Boolean x:Key="/Default/UserDictionary/Words/=tauri_0027s/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

‎app/MindWork AI Studio/Chat/ChatRole.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public enum ChatRole
1212
USER,
1313
AI,
1414
AGENT,
15+
RAG,
1516
}
1617

1718
/// <summary>

‎app/MindWork AI Studio/Chat/ContentImage.cs‎

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ namespace AIStudio.Chat;
77
/// <summary>
88
/// Represents an image inside the chat.
99
/// </summary>
10-
public sealed class ContentImage : IContent
10+
public sealed class ContentImage : IContent, IImageSource
1111
{
1212
#region Implementation of IContent
1313

@@ -47,62 +47,4 @@ public Task CreateFromProviderAsync(IProvider provider, Model chatModel, IConten
4747
/// The image source.
4848
/// </summary>
4949
public required string Source { get; set; }
50-
51-
/// <summary>
52-
/// Read the image content as a base64 string.
53-
/// </summary>
54-
/// <remarks>
55-
/// The images are directly converted to base64 strings. The maximum
56-
/// size of the image is around 10 MB. If the image is larger, the method
57-
/// returns an empty string.
58-
///
59-
/// As of now, this method does no sort of image processing. LLMs usually
60-
/// do not work with arbitrary image sizes. In the future, we might have
61-
/// to resize the images before sending them to the model.
62-
/// </remarks>
63-
/// <param name="token">The cancellation token.</param>
64-
/// <returns>The image content as a base64 string; might be empty.</returns>
65-
public async Task<string> AsBase64(CancellationToken token = default)
66-
{
67-
switch (this.SourceType)
68-
{
69-
case ContentImageSource.BASE64:
70-
return this.Source;
71-
72-
case ContentImageSource.URL:
73-
{
74-
using var httpClient = new HttpClient();
75-
using var response = await httpClient.GetAsync(this.Source, HttpCompletionOption.ResponseHeadersRead, token);
76-
if(response.IsSuccessStatusCode)
77-
{
78-
// Read the length of the content:
79-
var lengthBytes = response.Content.Headers.ContentLength;
80-
if(lengthBytes > 10_000_000)
81-
return string.Empty;
82-
83-
var bytes = await response.Content.ReadAsByteArrayAsync(token);
84-
return Convert.ToBase64String(bytes);
85-
}
86-
87-
return string.Empty;
88-
}
89-
90-
case ContentImageSource.LOCAL_PATH:
91-
if(File.Exists(this.Source))
92-
{
93-
// Read the content length:
94-
var length = new FileInfo(this.Source).Length;
95-
if(length > 10_000_000)
96-
return string.Empty;
97-
98-
var bytes = await File.ReadAllBytesAsync(this.Source, token);
99-
return Convert.ToBase64String(bytes);
100-
}
101-
102-
return string.Empty;
103-
104-
default:
105-
return string.Empty;
106-
}
107-
}
10850
}

‎app/MindWork AI Studio/Chat/IImageSource.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace AIStudio.Chat;
2+
3+
public interface IImageSource
4+
{
5+
/// <summary>
6+
/// The type of the image source.
7+
/// </summary>
8+
/// <remarks>
9+
/// Is the image source a URL, a local file path, a base64 string, etc.?
10+
/// </remarks>
11+
public ContentImageSource SourceType { get; init; }
12+
13+
/// <summary>
14+
/// The image source.
15+
/// </summary>
16+
public string Source { get; set; }
17+
}

‎app/MindWork AI Studio/Chat/IImageSourceExtensions.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
namespace AIStudio.Chat;
2+
3+
public static class IImageSourceExtensions
4+
{
5+
/// <summary>
6+
/// Read the image content as a base64 string.
7+
/// </summary>
8+
/// <remarks>
9+
/// The images are directly converted to base64 strings. The maximum
10+
/// size of the image is around 10 MB. If the image is larger, the method
11+
/// returns an empty string.
12+
///
13+
/// As of now, this method does no sort of image processing. LLMs usually
14+
/// do not work with arbitrary image sizes. In the future, we might have
15+
/// to resize the images before sending them to the model.
16+
/// </remarks>
17+
/// <param name="image">The image source.</param>
18+
/// <param name="token">The cancellation token.</param>
19+
/// <returns>The image content as a base64 string; might be empty.</returns>
20+
public static async Task<string> AsBase64(this IImageSource image, CancellationToken token = default)
21+
{
22+
switch (image.SourceType)
23+
{
24+
case ContentImageSource.BASE64:
25+
return image.Source;
26+
27+
case ContentImageSource.URL:
28+
{
29+
using var httpClient = new HttpClient();
30+
using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, token);
31+
if(response.IsSuccessStatusCode)
32+
{
33+
// Read the length of the content:
34+
var lengthBytes = response.Content.Headers.ContentLength;
35+
if(lengthBytes > 10_000_000)
36+
return string.Empty;
37+
38+
var bytes = await response.Content.ReadAsByteArrayAsync(token);
39+
return Convert.ToBase64String(bytes);
40+
}
41+
42+
return string.Empty;
43+
}
44+
45+
case ContentImageSource.LOCAL_PATH:
46+
if(File.Exists(image.Source))
47+
{
48+
// Read the content length:
49+
var length = new FileInfo(image.Source).Length;
50+
if(length > 10_000_000)
51+
return string.Empty;
52+
53+
var bytes = await File.ReadAllBytesAsync(image.Source, token);
54+
return Convert.ToBase64String(bytes);
55+
}
56+
57+
return string.Empty;
58+
59+
default:
60+
return string.Empty;
61+
}
62+
}
63+
}

‎app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
3737
ChatRole.USER => "user",
3838
ChatRole.AI => "assistant",
3939
ChatRole.AGENT => "assistant",
40+
ChatRole.RAG => "assistant",
4041

4142
_ => "user",
4243
},

‎app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
4949
ChatRole.AI => "assistant",
5050
ChatRole.AGENT => "assistant",
5151
ChatRole.SYSTEM => "system",
52+
ChatRole.RAG => "assistant",
5253

5354
_ => "user",
5455
},

‎app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Provider.Mod
5050
ChatRole.AI => "assistant",
5151
ChatRole.AGENT => "assistant",
5252
ChatRole.SYSTEM => "system",
53+
ChatRole.RAG => "assistant",
5354

5455
_ => "user",
5556
},

‎app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public override async IAsyncEnumerable<string> StreamChatCompletion(Model chatMo
5050
ChatRole.AI => "assistant",
5151
ChatRole.AGENT => "assistant",
5252
ChatRole.SYSTEM => "system",
53+
ChatRole.RAG => "assistant",
5354

5455
_ => "user",
5556
},

0 commit comments

Comments
 (0)