1  // Copyright (c) Microsoft Corporation
  2  // The Microsoft Corporation licenses this file to you under the MIT license.
  3  // See the LICENSE file in the project root for more information.
  4  
  5  using System;
  6  using System.Collections.Generic;
  7  using System.Globalization;
  8  using System.Linq;
  9  using System.Runtime.InteropServices;
 10  using System.Windows;
 11  using System.Windows.Controls;
 12  using System.Windows.Controls.Primitives;
 13  using System.Windows.Input;
 14  using System.Windows.Interop;
 15  using System.Windows.Media;
 16  
 17  using Common.UI;
 18  using ManagedCommon;
 19  using Microsoft.PowerToys.Telemetry;
 20  using PowerOCR.Helpers;
 21  using PowerOCR.Settings;
 22  using PowerOCR.Utilities;
 23  using Windows.Globalization;
 24  using Windows.Media.Ocr;
 25  
 26  namespace PowerOCR;
 27  
 28  /// <summary>
 29  /// Interaction logic for MainWindow.xaml
 30  /// </summary>
 31  public partial class OCROverlay : Window
 32  {
 33      private bool isShiftDown;
 34      private Point clickedPoint;
 35      private Point shiftPoint;
 36      private Border selectBorder = new();
 37      private Language? selectedLanguage;
 38  
 39      private bool IsSelecting { get; set; }
 40  
 41      private double selectLeft;
 42      private double selectTop;
 43  
 44      private double xShiftDelta;
 45      private double yShiftDelta;
 46      private bool isComboBoxReady;
 47      private const double ActiveOpacity = 0.4;
 48      private readonly UserSettings userSettings = new(new ThrottledActionInvoker());
 49      private System.Drawing.Rectangle screenRectangle;
 50      private DpiScale dpiScale;
 51  
 52      [DllImport("user32.dll", SetLastError = true)]
 53      private static extern bool MoveWindow(IntPtr hWnd, int x, int y, int nWidth, int nHeight, bool bRepaint);
 54  
 55      public OCROverlay(System.Drawing.Rectangle screenRectangleParam, DpiScale dpiScaleParam)
 56      {
 57          screenRectangle = screenRectangleParam;
 58          dpiScale = dpiScaleParam;
 59  
 60          Left = screenRectangle.Left;
 61          Top = screenRectangle.Top;
 62          Width = screenRectangle.Width / dpiScale.DpiScaleX;
 63          Height = screenRectangle.Height / dpiScale.DpiScaleY;
 64  
 65          InitializeComponent();
 66  
 67          Wpf.Ui.Appearance.SystemThemeWatcher.Watch(this, Wpf.Ui.Controls.WindowBackdropType.None);
 68  
 69          PopulateLanguageMenu();
 70      }
 71  
 72      private void PopulateLanguageMenu()
 73      {
 74          string? selectedLanguageName = userSettings.PreferredLanguage.Value;
 75  
 76          // build context menu
 77          if (string.IsNullOrEmpty(selectedLanguageName))
 78          {
 79              selectedLanguage = ImageMethods.GetOCRLanguage();
 80              selectedLanguageName = selectedLanguage?.NativeName;
 81          }
 82  
 83          List<Language> possibleOcrLanguages = OcrEngine.AvailableRecognizerLanguages.ToList();
 84  
 85          int count = 0;
 86  
 87          foreach (Language language in possibleOcrLanguages)
 88          {
 89              MenuItem menuItem = new() { Header = EnsureStartUpper(language.NativeName), Tag = language, IsCheckable = true };
 90              menuItem.IsChecked = language.NativeName.Equals(selectedLanguageName, StringComparison.OrdinalIgnoreCase);
 91              LanguagesComboBox.Items.Add(new ComboBoxItem { Content = EnsureStartUpper(language.NativeName), Tag = language });
 92              if (language.NativeName.Equals(selectedLanguageName, StringComparison.OrdinalIgnoreCase))
 93              {
 94                  selectedLanguage = language;
 95                  LanguagesComboBox.SelectedIndex = count;
 96              }
 97  
 98              menuItem.Click += LanguageMenuItem_Click;
 99              CanvasContextMenu.Items.Add(menuItem);
100              count++;
101          }
102  
103          isComboBoxReady = true;
104      }
105  
106      private void LanguageMenuItem_Click(object sender, RoutedEventArgs e)
107      {
108          MenuItem menuItem = (MenuItem)sender;
109          foreach (var item in CanvasContextMenu.Items)
110          {
111              if (item is MenuItem menuItemLoop)
112              {
113                  menuItemLoop.IsChecked = item.Equals(menuItem);
114              }
115          }
116  
117          selectedLanguage = menuItem.Tag as Language;
118          LanguagesComboBox.SelectedItem = selectedLanguage;
119      }
120  
121      private void Window_Loaded(object sender, RoutedEventArgs e)
122      {
123          FullWindow.Rect = new Rect(0, 0, Width, Height);
124          KeyDown += MainWindow_KeyDown;
125          KeyUp += MainWindow_KeyUp;
126  
127          BackgroundImage.Source = ImageMethods.GetWindowBoundsImage(this);
128          BackgroundBrush.Opacity = ActiveOpacity;
129  
130          TopButtonsStackPanel.Visibility = Visibility.Visible;
131  
132  #if DEBUG
133          Topmost = false;
134  #endif
135          IntPtr hwnd = new WindowInteropHelper(this).Handle;
136  
137          // The first move puts it on the correct monitor, which triggers WM_DPICHANGED
138          // The +1/-1 coerces WPF to update Window.Top/Left/Width/Height in the second move
139          MoveWindow(hwnd, (int)(screenRectangle.Left + 1), (int)screenRectangle.Top, (int)(screenRectangle.Width - 1), (int)screenRectangle.Height, false);
140          MoveWindow(hwnd, (int)screenRectangle.Left, (int)screenRectangle.Top, (int)screenRectangle.Width, (int)screenRectangle.Height, true);
141      }
142  
143      private void Window_Unloaded(object sender, RoutedEventArgs e)
144      {
145          BackgroundImage.Source = null;
146          BackgroundImage.UpdateLayout();
147  
148          KeyDown -= MainWindow_KeyDown;
149          KeyUp -= MainWindow_KeyUp;
150  
151          Loaded -= Window_Loaded;
152          Unloaded -= Window_Unloaded;
153  
154          RegionClickCanvas.MouseDown -= RegionClickCanvas_MouseDown;
155          RegionClickCanvas.MouseUp -= RegionClickCanvas_MouseUp;
156          RegionClickCanvas.MouseMove -= RegionClickCanvas_MouseMove;
157      }
158  
159      private void MainWindow_KeyUp(object sender, KeyEventArgs e)
160      {
161          switch (e.Key)
162          {
163              case Key.LeftShift:
164              case Key.RightShift:
165                  isShiftDown = false;
166                  clickedPoint = new Point(clickedPoint.X + xShiftDelta, clickedPoint.Y + yShiftDelta);
167                  break;
168              default:
169                  break;
170          }
171      }
172  
173      private void MainWindow_KeyDown(object sender, KeyEventArgs e)
174      {
175          WindowUtilities.OcrOverlayKeyDown(e.Key);
176      }
177  
178      private void RegionClickCanvas_MouseDown(object sender, MouseButtonEventArgs e)
179      {
180          if (e.LeftButton != MouseButtonState.Pressed)
181          {
182              return;
183          }
184  
185          TopButtonsStackPanel.Visibility = Visibility.Collapsed;
186          RegionClickCanvas.CaptureMouse();
187  
188          CursorClipper.ClipCursor(this);
189          clickedPoint = e.GetPosition(this);
190          selectBorder.Height = 1;
191          selectBorder.Width = 1;
192  
193          try
194          {
195              RegionClickCanvas.Children.Remove(selectBorder);
196          }
197          catch (Exception)
198          {
199          }
200  
201          selectBorder.BorderThickness = new Thickness(2);
202          Color borderColor = Color.FromArgb(255, 40, 118, 126);
203          selectBorder.BorderBrush = new SolidColorBrush(borderColor);
204          _ = RegionClickCanvas.Children.Add(selectBorder);
205          Canvas.SetLeft(selectBorder, clickedPoint.X);
206          Canvas.SetTop(selectBorder, clickedPoint.Y);
207  
208          IsSelecting = true;
209      }
210  
211      private void RegionClickCanvas_MouseMove(object sender, MouseEventArgs e)
212      {
213          if (!IsSelecting)
214          {
215              return;
216          }
217  
218          Point movingPoint = e.GetPosition(this);
219  
220          if (System.Windows.Input.Keyboard.Modifiers == ModifierKeys.Shift)
221          {
222              if (!isShiftDown)
223              {
224                  shiftPoint = movingPoint;
225                  selectLeft = Canvas.GetLeft(selectBorder);
226                  selectTop = Canvas.GetTop(selectBorder);
227              }
228  
229              isShiftDown = true;
230              xShiftDelta = movingPoint.X - shiftPoint.X;
231              yShiftDelta = movingPoint.Y - shiftPoint.Y;
232  
233              double leftValue = selectLeft + xShiftDelta;
234              double topValue = selectTop + yShiftDelta;
235  
236              clippingGeometry.Rect = new Rect(
237                  new Point(leftValue, topValue),
238                  new Size(selectBorder.Width, selectBorder.Height));
239              Canvas.SetLeft(selectBorder, leftValue - 1);
240              Canvas.SetTop(selectBorder, topValue - 1);
241              return;
242          }
243  
244          isShiftDown = false;
245  
246          double left = Math.Min(clickedPoint.X, movingPoint.X);
247          double top = Math.Min(clickedPoint.Y, movingPoint.Y);
248  
249          selectBorder.Height = Math.Max(clickedPoint.Y, movingPoint.Y) - top;
250          selectBorder.Width = Math.Max(clickedPoint.X, movingPoint.X) - left;
251          selectBorder.Height += 2;
252          selectBorder.Width += 2;
253  
254          clippingGeometry.Rect = new Rect(
255              new Point(left, top),
256              new Size(selectBorder.Width - 2, selectBorder.Height - 2));
257          Canvas.SetLeft(selectBorder, left - 1);
258          Canvas.SetTop(selectBorder, top - 1);
259      }
260  
261      private async void RegionClickCanvas_MouseUp(object sender, MouseButtonEventArgs e)
262      {
263          if (IsSelecting == false)
264          {
265              return;
266          }
267  
268          TopButtonsStackPanel.Visibility = Visibility.Visible;
269          IsSelecting = false;
270  
271          CursorClipper.UnClipCursor();
272          RegionClickCanvas.ReleaseMouseCapture();
273          Matrix m = PresentationSource.FromVisual(this).CompositionTarget.TransformToDevice;
274  
275          Point movingPoint = e.GetPosition(this);
276          movingPoint.X *= m.M11;
277          movingPoint.Y *= m.M22;
278  
279          movingPoint.X = Math.Round(movingPoint.X);
280          movingPoint.Y = Math.Round(movingPoint.Y);
281  
282          double xDimScaled = Canvas.GetLeft(selectBorder) * m.M11;
283          double yDimScaled = Canvas.GetTop(selectBorder) * m.M22;
284  
285          System.Drawing.Rectangle regionScaled = new(
286              (int)xDimScaled,
287              (int)yDimScaled,
288              (int)(selectBorder.Width * m.M11),
289              (int)(selectBorder.Height * m.M22));
290  
291          string grabbedText;
292  
293          try
294          {
295              RegionClickCanvas.Children.Remove(selectBorder);
296              clippingGeometry.Rect = new Rect(0, 0, 0, 0);
297          }
298          catch
299          {
300          }
301  
302          if (regionScaled.Width < 3 || regionScaled.Height < 3)
303          {
304              BackgroundBrush.Opacity = 0;
305              Logger.LogInfo($"Getting clicked word, {selectedLanguage?.LanguageTag}");
306              grabbedText = await ImageMethods.GetClickedWord(this, new Point(xDimScaled, yDimScaled), selectedLanguage);
307          }
308          else
309          {
310              if (TableMenuItem.IsChecked)
311              {
312                  Logger.LogInfo($"Getting region as table, {selectedLanguage?.LanguageTag}");
313                  grabbedText = await OcrExtensions.GetRegionsTextAsTableAsync(this, regionScaled, selectedLanguage);
314              }
315              else
316              {
317                  Logger.LogInfo($"Standard region capture, {selectedLanguage?.LanguageTag}");
318                  grabbedText = await ImageMethods.GetRegionsText(this, regionScaled, selectedLanguage);
319  
320                  if (SingleLineMenuItem.IsChecked)
321                  {
322                      Logger.LogInfo($"Making grabbed text single line");
323                      grabbedText = grabbedText.MakeStringSingleLine();
324                  }
325              }
326          }
327  
328          if (string.IsNullOrWhiteSpace(grabbedText))
329          {
330              BackgroundBrush.Opacity = ActiveOpacity;
331              return;
332          }
333  
334          try
335          {
336              Clipboard.SetText(grabbedText);
337          }
338          catch (Exception ex)
339          {
340              Logger.LogError($"Clipboard.SetText exception: {ex}");
341          }
342  
343          WindowUtilities.CloseAllOCROverlays();
344          PowerToysTelemetry.Log.WriteEvent(new PowerOCR.Telemetry.PowerOCRCaptureEvent());
345      }
346  
347      private void CancelMenuItem_Click(object sender, RoutedEventArgs e)
348      {
349          WindowUtilities.CloseAllOCROverlays();
350          PowerToysTelemetry.Log.WriteEvent(new PowerOCR.Telemetry.PowerOCRCancelledEvent());
351      }
352  
353      private void LanguagesComboBox_SelectionChanged(object sender, SelectionChangedEventArgs e)
354      {
355          if (sender is not ComboBox languageComboBox || !isComboBoxReady)
356          {
357              return;
358          }
359  
360          // TODO: Set the preferred language based upon what was chosen here
361          int selection = languageComboBox.SelectedIndex;
362          selectedLanguage = (languageComboBox.SelectedItem as ComboBoxItem)?.Tag as Language;
363  
364          if (selectedLanguage == null)
365          {
366              return;
367          }
368  
369          Logger.LogError($"Changed language to {selectedLanguage?.LanguageTag}");
370  
371          // Set the language in the context menu
372          foreach (var item in CanvasContextMenu.Items)
373          {
374              if (item is MenuItem menuItemLoop)
375              {
376                  menuItemLoop.IsChecked = menuItemLoop.Tag as Language == selectedLanguage;
377              }
378          }
379  
380          switch (selection)
381          {
382              case 0:
383                  WindowUtilities.OcrOverlayKeyDown(Key.D1);
384                  break;
385              case 1:
386                  WindowUtilities.OcrOverlayKeyDown(Key.D2);
387                  break;
388              case 2:
389                  WindowUtilities.OcrOverlayKeyDown(Key.D3);
390                  break;
391              case 3:
392                  WindowUtilities.OcrOverlayKeyDown(Key.D4);
393                  break;
394              case 4:
395                  WindowUtilities.OcrOverlayKeyDown(Key.D5);
396                  break;
397              case 5:
398                  WindowUtilities.OcrOverlayKeyDown(Key.D6);
399                  break;
400              case 6:
401                  WindowUtilities.OcrOverlayKeyDown(Key.D7);
402                  break;
403              case 7:
404                  WindowUtilities.OcrOverlayKeyDown(Key.D8);
405                  break;
406              case 8:
407                  WindowUtilities.OcrOverlayKeyDown(Key.D9);
408                  break;
409              default:
410                  break;
411          }
412      }
413  
414      private void SingleLineMenuItem_Click(object sender, RoutedEventArgs e)
415      {
416          bool isActive = CheckIfCheckingOrUnchecking(sender);
417          WindowUtilities.OcrOverlayKeyDown(Key.S, isActive);
418      }
419  
420      private void TableToggleButton_Click(object sender, RoutedEventArgs e)
421      {
422          bool isActive = CheckIfCheckingOrUnchecking(sender);
423          WindowUtilities.OcrOverlayKeyDown(Key.T, isActive);
424      }
425  
426      private void SettingsMenuItem_Click(object sender, RoutedEventArgs e)
427      {
428          WindowUtilities.CloseAllOCROverlays();
429          SettingsDeepLink.OpenSettings(SettingsDeepLink.SettingsWindow.PowerOCR);
430      }
431  
432      private static bool CheckIfCheckingOrUnchecking(object? sender)
433      {
434          if (sender is ToggleButton tb && tb.IsChecked is not null)
435          {
436              return tb.IsChecked.Value;
437          }
438  
439          if (sender is MenuItem mi)
440          {
441              return mi.IsChecked;
442          }
443  
444          return false;
445      }
446  
447      internal void KeyPressed(Key key, bool? isActive)
448      {
449          switch (key)
450          {
451              // This case is handled in the WindowUtilities.OcrOverlayKeyDown
452              // case Key.Escape:
453              //     WindowUtilities.CloseAllFullscreenGrabs();
454              //     break;
455              case Key.S:
456                  if (isActive is null)
457                  {
458                      SingleLineMenuItem.IsChecked = !SingleLineMenuItem.IsChecked;
459                  }
460                  else
461                  {
462                      SingleLineMenuItem.IsChecked = isActive.Value;
463                  }
464  
465                  // Possibly save this in settings later and remember this preference
466                  break;
467              case Key.T:
468                  if (isActive is null)
469                  {
470                      TableToggleButton.IsChecked = !TableToggleButton.IsChecked;
471                  }
472                  else
473                  {
474                      TableToggleButton.IsChecked = isActive.Value;
475                  }
476  
477                  break;
478              case Key.D1:
479              case Key.D2:
480              case Key.D3:
481              case Key.D4:
482              case Key.D5:
483              case Key.D6:
484              case Key.D7:
485              case Key.D8:
486              case Key.D9:
487                  int numberPressed = (int)key - 34; // D1 casts to 35, D2 to 36, etc.
488                  int numberOfLanguages = LanguagesComboBox.Items.Count;
489  
490                  if (numberPressed <= numberOfLanguages
491                      && numberPressed - 1 >= 0
492                      && numberPressed - 1 != LanguagesComboBox.SelectedIndex
493                      && isComboBoxReady)
494                  {
495                      LanguagesComboBox.SelectedIndex = numberPressed - 1;
496                  }
497  
498                  break;
499              default:
500                  break;
501          }
502      }
503  
504      public System.Drawing.Rectangle GetScreenRectangle()
505      {
506          return screenRectangle;
507      }
508  
509      private string EnsureStartUpper(string input)
510      {
511          if (string.IsNullOrEmpty(input))
512          {
513              return input;
514          }
515  
516          var inputArray = input.ToCharArray();
517          inputArray[0] = char.ToUpper(inputArray[0], CultureInfo.CurrentCulture);
518          return new string(inputArray);
519      }
520  }