Skip to content

Commit

Permalink
Merge pull request #74 from jasonjmcghee/fix-external-monitor-bugs
Browse files Browse the repository at this point in the history
fix external monitor bugs, mainly by calling CGDisplayCreateImage without passing the display frame
  • Loading branch information
jasonjmcghee authored Jan 18, 2024
2 parents 6f1d981 + e0b3afc commit 8260313
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 12 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,13 @@ Also, that means there is no tracking / analytics of any kind, which means I don
- [x] Search everything you've viewed with keyword search (and filter by application)
- [x] Easily grab recent context for use with LLMs
- [x] First [Intel build](https://github.com/jasonjmcghee/rem/releases/download/v0.1.11/rem-0.1.11-intel.dmg) (please help test!)
- [x] It "works" with external / multiple monitors connected
- [ ] Natural language search / agent interaction via updating local vector embedding
- [I've also been exploring novel approaches to vector dbs](https://github.com/jasonjmcghee/portable-hnsw)
- [ ] Novel search experiences like spatial / similar images
- [ ] More search filters (by time, etc.)
- [ ] Fine-grained purging / trimming / selecting recording
- [ ] Multi-monitor support
- [ ] Better / First-class multi-monitor support

## Getting Started

Expand Down
4 changes: 4 additions & 0 deletions rem.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

/* Begin PBXBuildFile section */
102CA4C82B3E240C00C3DA2E /* SQLite.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 96E66BC32B2F5745006E1E97 /* SQLite.framework */; };
960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 960AC0232B58D0590050C62A /* ImageResizer.swift */; };
961C95DA2B2E19B30093F228 /* remApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95D92B2E19B30093F228 /* remApp.swift */; };
961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95DB2B2E19B30093F228 /* ContentView.swift */; };
961C95E12B2E19B40093F228 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 961C95E02B2E19B40093F228 /* Preview Assets.xcassets */; };
Expand Down Expand Up @@ -193,6 +194,7 @@
/* End PBXCopyFilesBuildPhase section */

/* Begin PBXFileReference section */
960AC0232B58D0590050C62A /* ImageResizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageResizer.swift; sourceTree = "<group>"; };
961C95D62B2E19B30093F228 /* rem.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = rem.app; sourceTree = BUILT_PRODUCTS_DIR; };
961C95D92B2E19B30093F228 /* remApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = remApp.swift; sourceTree = "<group>"; };
961C95DB2B2E19B30093F228 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -289,6 +291,7 @@
969F3F092B3B7F760085787B /* Info.plist */,
961C96122B2EB7DB0093F228 /* TimelineView.swift */,
BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */,
960AC0232B58D0590050C62A /* ImageResizer.swift */,
961C95D92B2E19B30093F228 /* remApp.swift */,
961C95DB2B2E19B30093F228 /* ContentView.swift */,
961C95DD2B2E19B40093F228 /* Assets.xcassets */,
Expand Down Expand Up @@ -656,6 +659,7 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */,
961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */,
961C96152B2EBEE50093F228 /* DB.swift in Sources */,
96B0DA3A2B3A08280030E8AE /* TextMerger.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,73 @@
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>6</integer>
<integer>7</integer>
</dict>
<key>SQLite (Playground) 2.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>7</integer>
<integer>8</integer>
</dict>
<key>SQLite (Playground) 3.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>9</integer>
</dict>
<key>SQLite (Playground) 4.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>10</integer>
</dict>
<key>SQLite (Playground) 5.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>11</integer>
</dict>
<key>SQLite (Playground) 6.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>12</integer>
</dict>
<key>SQLite (Playground) 7.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>13</integer>
</dict>
<key>SQLite (Playground) 8.xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>14</integer>
</dict>
<key>SQLite (Playground).xcscheme</key>
<dict>
<key>isShown</key>
<false/>
<key>orderHint</key>
<integer>5</integer>
<integer>6</integer>
</dict>
<key>ffmpegX.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>5</integer>
<integer>4</integer>
</dict>
<key>rem.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>4</integer>
<integer>5</integer>
</dict>
</dict>
</dict>
Expand Down
42 changes: 42 additions & 0 deletions rem/ImageResizer.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
//
// ImageResizer.swift
// rem
//
// Created by Jason McGhee on 1/17/24.
//

import Foundation
import Cocoa

class ImageResizer {
private var context: CGContext
private let targetWidth: CGFloat
private let targetHeight: CGFloat

init(targetWidth: Int, targetHeight: Int) {
self.targetWidth = CGFloat(targetWidth)
self.targetHeight = CGFloat(targetHeight)

let colorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue
context = CGContext(data: nil, width: targetWidth, height: targetHeight, bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace, bitmapInfo: bitmapInfo)!
}

func resizeAndPad(image: CGImage) -> CGImage? {
let widthScaleRatio = targetWidth / CGFloat(image.width)
let heightScaleRatio = targetHeight / CGFloat(image.height)
let scaleFactor = min(widthScaleRatio, heightScaleRatio)

let scaledWidth = CGFloat(image.width) * scaleFactor
let scaledHeight = CGFloat(image.height) * scaleFactor
let imageRect = CGRect(x: (targetWidth - scaledWidth) / 2, y: (targetHeight - scaledHeight) / 2, width: scaledWidth, height: scaledHeight)

context.clear(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
context.setFillColor(NSColor.black.cgColor)
context.fill(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
context.interpolationQuality = .high
context.draw(image, in: imageRect)

return context.makeImage()
}
}
2 changes: 1 addition & 1 deletion rem/TimelineView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class CustomHostingView: NSHostingView<AnyView> {
private func configureImageView(with image: NSImage, in frame: NSRect) {
imageView.image = image

imageView.imageScaling = .scaleAxesIndependently
imageView.imageScaling = .scaleProportionallyUpOrDown

// Configuring frame to account for the offset and scaling
imageView.frame = CGRect(x: 0, y: 0, width: frame.width, height: frame.height)
Expand Down
23 changes: 18 additions & 5 deletions rem/remApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ class AppDelegate: NSObject, NSApplicationDelegate {

private var lastImageData: Data? = nil
private var lastActiveApplication: String? = nil

private var imageResizer = ImageResizer(targetWidth: Int(NSScreen.main!.frame.width), targetHeight: Int(NSScreen.main!.frame.height))

func applicationDidFinishLaunching(_ notification: Notification) {
let _ = DatabaseManager.shared
Expand Down Expand Up @@ -349,6 +351,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
var displayID: CGDirectDisplayID? = nil
if let screenID = NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber {
displayID = CGDirectDisplayID(screenID.uint32Value)
logger.debug("Display ID: \(displayID ?? 999)")
}
guard displayID != nil else { return }

Expand All @@ -358,10 +361,20 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
logger.debug("Active Application: \(activeApplicationName ?? "<undefined>")")

// Do we want to record the timeline being searched?
guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return }
guard let image = CGDisplayCreateImage(display.displayID) else {
logger.error("Failed to create a screenshot for the display!")
return
}
guard let resizedImage = imageResizer.resizeAndPad(image: image) else {
logger.error("Failed to resize the image!")
return
}

let bitmapRep = NSBitmapImageRep(cgImage: image)
guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else { return }
let bitmapRep = NSBitmapImageRep(cgImage: resizedImage)
guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else {
logger.error("Failed to create a PNG from the screenshot!")
return
}

// Might as well only check if the applications are the same, otherwise obviously different
if activeApplicationName != lastActiveApplication || displayImageChangedFromLast(imageData: imageData) {
Expand All @@ -371,7 +384,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName)

if settingsManager.settings.onlyOCRFrontmostWindow {
// User wants to perform OCR on only active window.
// default: User wants to perform OCR on only active window.

// We need to determine the scale factor for cropping. CGImage is
// measured in pixels, display sizes are measured in points.
Expand All @@ -384,7 +397,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
self.performOCR(frameId: frameId, on: cropped)
}
} else {
// default: User wants to perform OCR on full display.
// User wants to perform OCR on full display.
self.performOCR(frameId: frameId, on: image)
}

Expand Down

0 comments on commit 8260313

Please sign in to comment.