diff --git a/README.md b/README.md index f2f41dc..76d3d50 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,13 @@ Also, that means there is no tracking / analytics of any kind, which means I don - [x] Search everything you've viewed with keyword search (and filter by application) - [x] Easily grab recent context for use with LLMs - [x] First [Intel build](https://github.com/jasonjmcghee/rem/releases/download/v0.1.11/rem-0.1.11-intel.dmg) (please help test!) +- [x] It "works" with external / multiple monitors connected - [ ] Natural language search / agent interaction via updating local vector embedding - [I've also been exploring novel approaches to vector dbs](https://github.com/jasonjmcghee/portable-hnsw) - [ ] Novel search experiences like spatial / similar images - [ ] More search filters (by time, etc.) - [ ] Fine-grained purging / trimming / selecting recording -- [ ] Multi-monitor support +- [ ] Better / First-class multi-monitor support ## Getting Started diff --git a/rem.xcodeproj/project.pbxproj b/rem.xcodeproj/project.pbxproj index 4ad89c0..904b2f4 100644 --- a/rem.xcodeproj/project.pbxproj +++ b/rem.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ 102CA4C82B3E240C00C3DA2E /* SQLite.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 96E66BC32B2F5745006E1E97 /* SQLite.framework */; }; + 960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 960AC0232B58D0590050C62A /* ImageResizer.swift */; }; 961C95DA2B2E19B30093F228 /* remApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95D92B2E19B30093F228 /* remApp.swift */; }; 961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95DB2B2E19B30093F228 /* ContentView.swift */; }; 961C95E12B2E19B40093F228 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 961C95E02B2E19B40093F228 /* Preview Assets.xcassets */; }; @@ -193,6 +194,7 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 960AC0232B58D0590050C62A /* ImageResizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageResizer.swift; sourceTree = ""; }; 961C95D62B2E19B30093F228 /* rem.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = rem.app; sourceTree = BUILT_PRODUCTS_DIR; }; 961C95D92B2E19B30093F228 /* remApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = remApp.swift; sourceTree = ""; }; 961C95DB2B2E19B30093F228 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; @@ -289,6 +291,7 @@ 969F3F092B3B7F760085787B /* Info.plist */, 961C96122B2EB7DB0093F228 /* TimelineView.swift */, BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */, + 960AC0232B58D0590050C62A /* ImageResizer.swift */, 961C95D92B2E19B30093F228 /* remApp.swift */, 961C95DB2B2E19B30093F228 /* ContentView.swift */, 961C95DD2B2E19B40093F228 /* Assets.xcassets */, @@ -656,6 +659,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */, 961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */, 961C96152B2EBEE50093F228 /* DB.swift in Sources */, 96B0DA3A2B3A08280030E8AE /* TextMerger.swift in Sources */, diff --git a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist index 9ac48dd..b9d8542 100644 --- a/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist @@ -9,31 +9,73 @@ isShown orderHint - 6 + 7 SQLite (Playground) 2.xcscheme isShown orderHint - 7 + 8 + + SQLite (Playground) 3.xcscheme + + isShown + + orderHint + 9 + + SQLite (Playground) 4.xcscheme + + isShown + + orderHint + 10 + + SQLite (Playground) 5.xcscheme + + isShown + + orderHint + 11 + + SQLite (Playground) 6.xcscheme + + isShown + + orderHint + 12 + + SQLite (Playground) 7.xcscheme + + isShown + + orderHint + 13 + + SQLite (Playground) 8.xcscheme + + isShown + + orderHint + 14 SQLite (Playground).xcscheme isShown orderHint - 5 + 6 ffmpegX.xcscheme_^#shared#^_ orderHint - 5 + 4 rem.xcscheme_^#shared#^_ orderHint - 4 + 5 diff --git a/rem/ImageResizer.swift b/rem/ImageResizer.swift new file mode 100644 index 0000000..828c40a --- /dev/null +++ b/rem/ImageResizer.swift @@ -0,0 +1,42 @@ +// +// ImageResizer.swift +// rem +// +// Created by Jason McGhee on 1/17/24. +// + +import Foundation +import Cocoa + +class ImageResizer { + private var context: CGContext + private let targetWidth: CGFloat + private let targetHeight: CGFloat + + init(targetWidth: Int, targetHeight: Int) { + self.targetWidth = CGFloat(targetWidth) + self.targetHeight = CGFloat(targetHeight) + + let colorSpace = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue + context = CGContext(data: nil, width: targetWidth, height: targetHeight, bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace, bitmapInfo: bitmapInfo)! + } + + func resizeAndPad(image: CGImage) -> CGImage? { + let widthScaleRatio = targetWidth / CGFloat(image.width) + let heightScaleRatio = targetHeight / CGFloat(image.height) + let scaleFactor = min(widthScaleRatio, heightScaleRatio) + + let scaledWidth = CGFloat(image.width) * scaleFactor + let scaledHeight = CGFloat(image.height) * scaleFactor + let imageRect = CGRect(x: (targetWidth - scaledWidth) / 2, y: (targetHeight - scaledHeight) / 2, width: scaledWidth, height: scaledHeight) + + context.clear(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight)) + context.setFillColor(NSColor.black.cgColor) + context.fill(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight)) + context.interpolationQuality = .high + context.draw(image, in: imageRect) + + return context.makeImage() + } +} diff --git a/rem/TimelineView.swift b/rem/TimelineView.swift index 28f38a4..d0a3be8 100644 --- a/rem/TimelineView.swift +++ b/rem/TimelineView.swift @@ -118,7 +118,7 @@ class CustomHostingView: NSHostingView { private func configureImageView(with image: NSImage, in frame: NSRect) { imageView.image = image - imageView.imageScaling = .scaleAxesIndependently + imageView.imageScaling = .scaleProportionallyUpOrDown // Configuring frame to account for the offset and scaling imageView.frame = CGRect(x: 0, y: 0, width: frame.width, height: frame.height) diff --git a/rem/remApp.swift b/rem/remApp.swift index c2e9216..79d3661 100644 --- a/rem/remApp.swift +++ b/rem/remApp.swift @@ -89,6 +89,8 @@ class AppDelegate: NSObject, NSApplicationDelegate { private var lastImageData: Data? = nil private var lastActiveApplication: String? = nil + + private var imageResizer = ImageResizer(targetWidth: Int(NSScreen.main!.frame.width), targetHeight: Int(NSScreen.main!.frame.height)) func applicationDidFinishLaunching(_ notification: Notification) { let _ = DatabaseManager.shared @@ -349,6 +351,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool { var displayID: CGDirectDisplayID? = nil if let screenID = NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber { displayID = CGDirectDisplayID(screenID.uint32Value) + logger.debug("Display ID: \(displayID ?? 999)") } guard displayID != nil else { return } @@ -358,10 +361,20 @@ func drawStatusBarIcon(rect: CGRect) -> Bool { logger.debug("Active Application: \(activeApplicationName ?? "")") // Do we want to record the timeline being searched? - guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return } + guard let image = CGDisplayCreateImage(display.displayID) else { + logger.error("Failed to create a screenshot for the display!") + return + } + guard let resizedImage = imageResizer.resizeAndPad(image: image) else { + logger.error("Failed to resize the image!") + return + } - let bitmapRep = NSBitmapImageRep(cgImage: image) - guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else { return } + let bitmapRep = NSBitmapImageRep(cgImage: resizedImage) + guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else { + logger.error("Failed to create a PNG from the screenshot!") + return + } // Might as well only check if the applications are the same, otherwise obviously different if activeApplicationName != lastActiveApplication || displayImageChangedFromLast(imageData: imageData) { @@ -371,7 +384,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool { let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName) if settingsManager.settings.onlyOCRFrontmostWindow { - // User wants to perform OCR on only active window. + // default: User wants to perform OCR on only active window. // We need to determine the scale factor for cropping. CGImage is // measured in pixels, display sizes are measured in points. @@ -384,7 +397,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool { self.performOCR(frameId: frameId, on: cropped) } } else { - // default: User wants to perform OCR on full display. + // User wants to perform OCR on full display. self.performOCR(frameId: frameId, on: image) }