Skip to content

Paste from Explorer/ Add color instructions to OCR #193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Feb 13, 2025
Merged

Paste from Explorer/ Add color instructions to OCR #193

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
81f0a7f
Paste images from Explorer Copy
brunosonnino Feb 11, 2025
ea445ef
Merge with current version
brunosonnino Feb 11, 2025
f1adc2f
Fixes for merge
brunosonnino Feb 11, 2025
e525be2
Make sure image doesn't take all the space
nmetulev Feb 12, 2025
18d7cf3
Merge branch 'alzollin/wcrapis' into bsonnino/paste_from_explorer
nmetulev Feb 12, 2025
d3abf8e
Remove IsImageFile function from Utils.cs
brunosonnino Feb 13, 2025
0433f21
Merge branch 'bsonnino/paste_from_explorer' of https://github.com/mic…
brunosonnino Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions AIDevGallery/Samples/SharedCode/Utils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
using Microsoft.UI;
using Microsoft.UI.Xaml;
using Microsoft.UI.Xaml.Media;
using System.IO;
using System.Linq;
using Windows.UI;

namespace AIDevGallery.Samples.SharedCode;
Expand All @@ -25,4 +27,10 @@ public static Visibility BoolToVisibleInversed(bool value)
{
return value ? Visibility.Collapsed : Visibility.Visible;
}

public static bool IsImageFile(string fileName)
{
string[] imageExtensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif"];
return imageExtensions.Contains(Path.GetExtension(fileName)?.ToLowerInvariant());
}
}
2 changes: 1 addition & 1 deletion AIDevGallery/Samples/WCRAPIs/BackgroundRemover.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<RowDefinition Height="*" />
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="*" />
</Grid.ColumnDefinitions>
<StackPanel
Expand Down
18 changes: 18 additions & 0 deletions AIDevGallery/Samples/WCRAPIs/BackgroundRemover.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Windows.ApplicationModel.DataTransfer;
using Windows.Graphics;
using Windows.Graphics.Imaging;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;

Expand Down Expand Up @@ -88,6 +89,23 @@ private async void PasteImage_Click(object sender, RoutedEventArgs e)
using IRandomAccessStream stream = await streamRef.OpenReadAsync();
await SetImage(stream);
}
else if (package.Contains(StandardDataFormats.StorageItems))
{
var storageItems = await package.GetStorageItemsAsync();
if (SharedCode.Utils.IsImageFile(storageItems[0].Path))
{
try
{
var storageFile = await StorageFile.GetFileFromPathAsync(storageItems[0].Path);
using var stream = await storageFile.OpenReadAsync();
await SetImage(stream);
}
catch
{
Console.WriteLine("Invalid Image File");
}
}
}
}

private async Task SetImage(IRandomAccessStream stream)
Expand Down
62 changes: 44 additions & 18 deletions AIDevGallery/Samples/WCRAPIs/ImageDescription.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
using Microsoft.Windows.AI.Generative;
using Microsoft.Windows.Management.Deployment;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Windows.ApplicationModel.DataTransfer;
using Windows.Graphics.Imaging;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;

Expand Down Expand Up @@ -79,9 +82,26 @@ private async void PasteImage_Click(object sender, RoutedEventArgs e)
{
var streamRef = await package.GetBitmapAsync();

IRandomAccessStream stream = await streamRef.OpenReadAsync();
using var stream = await streamRef.OpenReadAsync();
await SetImage(stream);
}
else if (package.Contains(StandardDataFormats.StorageItems))
{
var storageItems = await package.GetStorageItemsAsync();
if (SharedCode.Utils.IsImageFile(storageItems[0].Path))
{
try
{
var storageFile = await StorageFile.GetFileFromPathAsync(storageItems[0].Path);
using var stream = await storageFile.OpenReadAsync();
await SetImage(stream);
}
catch
{
Console.WriteLine("Invalid Image File");
}
}
}
}

private async Task SetImage(IRandomAccessStream stream)
Expand Down Expand Up @@ -113,27 +133,33 @@ private async void DescribeImage(SoftwareBitmap bitmap)
});

var isFirstWord = true;
using var bitmapBuffer = ImageBuffer.CreateCopyFromBitmap(bitmap);
var describeTask = _imageDescriptor?.DescribeAsync(bitmapBuffer);
if (describeTask != null)
try
{
describeTask.Progress += (asyncInfo, delta) =>
using var bitmapBuffer = ImageBuffer.CreateCopyFromBitmap(bitmap);
var describeTask = _imageDescriptor?.DescribeAsync(bitmapBuffer);
if (describeTask != null)
{
var result = asyncInfo.GetResults().Response;

DispatcherQueue?.TryEnqueue(() =>
describeTask.Progress += (asyncInfo, delta) =>
{
if (isFirstWord)
DispatcherQueue?.TryEnqueue(() =>
{
Loader.Visibility = Visibility.Collapsed;
OutputTxt.Visibility = Visibility.Visible;
isFirstWord = false;
}

ResponseTxt.Text = result;
});
};
await describeTask;
if (isFirstWord)
{
Loader.Visibility = Visibility.Collapsed;
OutputTxt.Visibility = Visibility.Visible;
isFirstWord = false;
}

ResponseTxt.Text = delta;
});
};

await describeTask;
}
}
catch (Exception ex)
{
ResponseTxt.Text = ex.Message;
}

Loader.Visibility = Visibility.Collapsed;
Expand Down
4 changes: 2 additions & 2 deletions AIDevGallery/Samples/WCRAPIs/IncreaseFidelity.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
<RowDefinition Height="Auto" />
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="*" />
</Grid.ColumnDefinitions>
<StackPanel
Grid.ColumnSpan="2"
Expand Down
46 changes: 35 additions & 11 deletions AIDevGallery/Samples/WCRAPIs/IncreaseFidelity.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using System.Threading.Tasks;
using Windows.ApplicationModel.DataTransfer;
using Windows.Graphics.Imaging;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;

Expand Down Expand Up @@ -72,7 +73,7 @@ private async void LoadImage_Click(object sender, RoutedEventArgs e)
if (file != null)
{
using var stream = await file.OpenReadAsync();
SetImage(stream);
await SetImage(stream);
}
}

Expand All @@ -82,20 +83,43 @@ private async void PasteImage_Click(object sender, RoutedEventArgs e)
if (package.Contains(StandardDataFormats.Bitmap))
{
var streamRef = await package.GetBitmapAsync();

IRandomAccessStream stream = await streamRef.OpenReadAsync();
SetImage(stream);
using IRandomAccessStream stream = await streamRef.OpenReadAsync();
await SetImage(stream);
}
else if (package.Contains(StandardDataFormats.StorageItems))
{
var storageItems = await package.GetStorageItemsAsync();
if (SharedCode.Utils.IsImageFile(storageItems[0].Path))
{
try
{
var storageFile = await StorageFile.GetFileFromPathAsync(storageItems[0].Path);
using var stream = await storageFile.OpenReadAsync();
await SetImage(stream);
}
catch
{
Console.WriteLine("Invalid Image File");
}
}
}
}

private async void SetImage(IRandomAccessStream stream)
private async Task SetImage(IRandomAccessStream stream)
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
_originalImage = await decoder.GetSoftwareBitmapAsync();
OptionsPanel.Visibility = Visibility.Visible;
OriginalPanel.Visibility = Visibility.Visible;
await SetImageSource(OriginalImage, _originalImage, OriginalDimensionsTxt);
ScaleImage();
try
{
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
_originalImage = await decoder.GetSoftwareBitmapAsync();
OptionsPanel.Visibility = Visibility.Visible;
OriginalPanel.Visibility = Visibility.Visible;
await SetImageSource(OriginalImage, _originalImage, OriginalDimensionsTxt);
ScaleImage();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}

private async void ScaleImage()
Expand Down
26 changes: 26 additions & 0 deletions AIDevGallery/Samples/WCRAPIs/OCRLineSample.xaml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using Windows.ApplicationModel.DataTransfer;
using Windows.Foundation;
using Windows.Graphics.Imaging;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;
using Windows.UI.Text;
Expand Down Expand Up @@ -92,6 +93,23 @@ private async void PasteImage_Click(object sender, RoutedEventArgs e)
IRandomAccessStream stream = await streamRef.OpenReadAsync();
await SetImage(stream);
}
else if (package.Contains(StandardDataFormats.StorageItems))
{
var storageItems = await package.GetStorageItemsAsync();
if (SharedCode.Utils.IsImageFile(storageItems[0].Path))
{
try
{
var storageFile = await StorageFile.GetFileFromPathAsync(storageItems[0].Path);
using var stream = await storageFile.OpenReadAsync();
await SetImage(stream);
}
catch
{
Console.WriteLine("Invalid Image File");
}
}
}
}

private async Task SetImage(IRandomAccessStream stream)
Expand Down Expand Up @@ -121,10 +139,12 @@ private async Task RecognizeAndAddTextAsync(SoftwareBitmap bitmap)
{
CopyTextButton.Visibility = Visibility.Collapsed;
Loader.Visibility = Visibility.Visible;
RectCanvas.Visibility = Visibility.Collapsed;
using var imageBuffer = ImageBuffer.CreateBufferAttachedToBitmap(bitmap);
RecognizedText? result = _textRecognizer?.RecognizeTextFromImage(imageBuffer, new TextRecognizerOptions());
if (result == null)
{
Loader.Visibility = Visibility.Collapsed;
return;
}

Expand All @@ -135,6 +155,12 @@ private async Task RecognizeAndAddTextAsync(SoftwareBitmap bitmap)
RectCanvas.Children.Clear();
TextPanel.Children.Clear();

if (result.Lines == null || result.Lines.Length == 0)
{
Loader.Visibility = Visibility.Collapsed;
return;
}

foreach (var line in result.Lines)
{
var boundingBox = line.BoundingBox;
Expand Down
49 changes: 30 additions & 19 deletions AIDevGallery/Samples/WCRAPIs/OCRSample.xaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,16 @@
xmlns:muxc="using:Microsoft.UI.Xaml.Controls"
xmlns:samples="using:AIDevGallery.Samples"
mc:Ignorable="d">
<Grid ColumnSpacing="24" RowSpacing="48">
<Grid ColumnSpacing="24" RowSpacing="24">
<Grid.RowDefinitions>
<RowDefinition Height="Auto" />
<RowDefinition Height="Auto" />
<RowDefinition Height="*" />
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="Auto" />
<ColumnDefinition Width="*" />
<ColumnDefinition Width="*"/>
</Grid.ColumnDefinitions>

<DropDownButton AutomationProperties.Name="Select image">
<StackPanel Orientation="Horizontal" Spacing="8">
<FontIcon FontSize="16" Glyph="&#xEE71;" />
Expand All @@ -31,37 +31,48 @@
</MenuFlyout>
</DropDownButton.Flyout>
</DropDownButton>

<TextBlock
x:Name="InstructionTxt"
Grid.Row="1"
Grid.ColumnSpan="2"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Style="{StaticResource CaptionTextBlockStyle}"
Text="The colors of the recognized text show the confidence of recognition: Green - good recognition Brown - medium recognition Red - bad recognition"
TextWrapping="Wrap"
Visibility="Collapsed" />
<Image
x:Name="ImageSrc"
Grid.Row="1"
Grid.Row="2"
MaxHeight="360"
HorizontalAlignment="Stretch"
VerticalAlignment="Top"
Stretch="Uniform" />

<ScrollViewer
x:Name="OutputPanel"
Grid.Row="1"
<Grid x:Name="OutputPanel" Grid.Row="2"
Grid.Column="1"
HorizontalAlignment="Stretch"
VerticalAlignment="Stretch"
HorizontalScrollBarVisibility="Disabled"
VerticalScrollBarVisibility="Auto"
Visibility="Collapsed">
<StackPanel Orientation="Vertical">
<TextBlock FontWeight="SemiBold" Text="Recognized text" />
HorizontalAlignment="Stretch" RowSpacing="10"
VerticalAlignment="Stretch" Visibility="Collapsed">
<Grid.RowDefinitions>
<RowDefinition Height="Auto" />
<RowDefinition Height="*" />
</Grid.RowDefinitions>
<TextBlock FontWeight="SemiBold" Text="Recognized text" />
<ScrollViewer
Grid.Row="1"
HorizontalAlignment="Stretch"
VerticalAlignment="Stretch"
HorizontalScrollBarVisibility="Disabled"
VerticalScrollBarVisibility="Auto" >
<TextBlock
x:Name="OcrTextBlock"
Grid.Row="1"
IsTextSelectionEnabled="True"
TextWrapping="WrapWholeWords" />
</StackPanel>

</ScrollViewer>
</ScrollViewer>
</Grid>
<ProgressRing
x:Name="Loader"
Grid.Row="1"
Grid.RowSpan="2"
Grid.Column="1"
Width="48"
Height="48"
Expand Down
Loading