diff --git a/SampleApp/App/Constants.swift b/SampleApp/App/Constants.swift index 06103b58..32e20214 100644 --- a/SampleApp/App/Constants.swift +++ b/SampleApp/App/Constants.swift @@ -9,6 +9,12 @@ enum Constants { static let fovy = Angle(degrees: 65) #endif static let modelCenterZ: Float = -8 + /// Fly speed, as a fraction of the current camera distance per second. + static let movementSpeed: Float = 0.9 + /// Multiplier while the speed modifier (shift) is held. + static let movementBoost: Float = 3 + /// Keyboard look rate, in mouse-pixel equivalents per second. + static let lookSpeed: Float = 260 // Procedural splat geometry static let proceduralCubeSize: Float = 1.0 diff --git a/SampleApp/MetalSplatter_SampleApp.xcodeproj/project.pbxproj b/SampleApp/MetalSplatter_SampleApp.xcodeproj/project.pbxproj index 894a0f05..ede75ff5 100644 --- a/SampleApp/MetalSplatter_SampleApp.xcodeproj/project.pbxproj +++ b/SampleApp/MetalSplatter_SampleApp.xcodeproj/project.pbxproj @@ -250,6 +250,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.metalsplatter.sampleapp; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; @@ -287,6 +288,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.metalsplatter.sampleapp; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator macosx xros xrsimulator"; diff --git a/SampleApp/Model/ProceduralSplatController.swift b/SampleApp/Model/ProceduralSplatController.swift index 5968706b..eccb5646 100644 --- a/SampleApp/Model/ProceduralSplatController.swift +++ b/SampleApp/Model/ProceduralSplatController.swift @@ -28,7 +28,8 @@ final class ProceduralSplatController: @unchecked Sendable { depthFormat: MTLPixelFormat, sampleCount: Int, maxViewCount: Int, - maxSimultaneousRenders: Int) async throws { + maxSimultaneousRenders: Int, + isolation: isolated (any Actor)? = #isolation) async throws { splatRenderer = try SplatRenderer(device: device, colorFormat: colorFormat, depthFormat: depthFormat, diff --git a/SampleApp/Scene/MetalKitSceneRenderer.swift b/SampleApp/Scene/MetalKitSceneRenderer.swift index f6e1727b..9b832cb2 100644 --- a/SampleApp/Scene/MetalKitSceneRenderer.swift +++ b/SampleApp/Scene/MetalKitSceneRenderer.swift @@ -12,7 +12,7 @@ import SwiftUI @MainActor class MetalKitSceneRenderer: NSObject, MTKViewDelegate { private static let log = - Logger(subsystem: Bundle.main.bundleIdentifier!, + Logger(subsystem: Bundle.main.bundleIdentifier ?? "com.metalsplatter.sampleapp", category: "MetalKitSceneRenderer") let metalKitView: MTKView @@ -28,6 +28,23 @@ class MetalKitSceneRenderer: NSObject, MTKViewDelegate { var lastRotationUpdateTimestamp: Date? = nil var rotation: Angle = .zero + // Orbit camera. `rotation` above is the idle auto-spin, which folds into + // cameraYaw and stops as soon as the user takes control. + var autoRotate = true + var cameraYaw: Float = 0 + var cameraPitch: Float = 0 + var cameraDistance: Float = -Constants.modelCenterZ + var cameraTarget: SIMD3 = .zero + /// Whether to apply the built-in 180-degrees-about-Z up-axis calibration. + /// Toggle at runtime rather than guessing the convention of a given file. + var applyUpCalibration = true + /// Camera-relative fly input: x = right, y = up, z = forward. Unit length. + var moveInput: SIMD3 = .zero + /// Speed multiplier while the modifier key is held. + var moveBoost: Float = 1 + /// Keyboard look input: x = right, y = down (matching mouse delta signs). + var lookInput: SIMD2 = .zero + var drawableSize: CGSize = .zero init?(_ metalKitView: MTKView) { @@ -85,32 +102,143 @@ class MetalKitSceneRenderer: NSObject, MTKViewDelegate { private var viewport: ModelRendererViewportDescriptor { let projectionMatrix = matrix_perspective_right_hand(fovyRadians: Float(Constants.fovy.radians), aspectRatio: Float(drawableSize.width / drawableSize.height), - nearZ: 0.1, - farZ: 100.0) + nearZ: max(0.01, cameraDistance * 0.002), + farZ: max(100, cameraDistance * 20)) - let rotationMatrix = matrix4x4_rotation(radians: Float(rotation.radians), - axis: Constants.rotationAxis) - let translationMatrix = matrix4x4_translation(0.0, 0.0, Constants.modelCenterZ) // Turn common 3D GS PLY files rightside-up. This isn't generally meaningful, it just // happens to be a useful default for the most common datasets at the moment. - let commonUpCalibration = matrix4x4_rotation(radians: .pi, axis: SIMD3(0, 0, 1)) + let commonUpCalibration = applyUpCalibration + ? matrix4x4_rotation(radians: .pi, axis: SIMD3(0, 0, 1)) + : matrix_identity_float4x4 + + // view = inverse(target * yaw * pitch * dolly) + let viewMatrix = matrix4x4_translation(0, 0, -cameraDistance) + * matrix4x4_rotation(radians: -cameraPitch, axis: SIMD3(1, 0, 0)) + * matrix4x4_rotation(radians: -(cameraYaw + Float(rotation.radians)), axis: Constants.rotationAxis) + * matrix4x4_translation(-cameraTarget.x, -cameraTarget.y, -cameraTarget.z) + * commonUpCalibration let viewport = MTLViewport(originX: 0, originY: 0, width: drawableSize.width, height: drawableSize.height, znear: 0, zfar: 1) return ModelRendererViewportDescriptor(viewport: viewport, projectionMatrix: projectionMatrix, - viewMatrix: translationMatrix * rotationMatrix * commonUpCalibration, + viewMatrix: viewMatrix, screenSize: SIMD2(x: Int(drawableSize.width), y: Int(drawableSize.height))) } private func updateRotation() { let now = Date() - defer { - lastRotationUpdateTimestamp = now + let deltaTime = lastRotationUpdateTimestamp.map { now.timeIntervalSince($0) } ?? 0 + lastRotationUpdateTimestamp = now + + if autoRotate { + rotation += Constants.rotationPerSecond * deltaTime } + // Clamp dt so a stall (window drag, model load) doesn't teleport the camera. + let dt = Float(min(deltaTime, 0.1)) + applyLook(deltaTime: dt) + applyMovement(deltaTime: dt) + } + + private func applyLook(deltaTime: Float) { + guard lookInput != .zero, deltaTime > 0 else { return } + let d = Constants.lookSpeed * deltaTime + look(dx: lookInput.x * d, dy: lookInput.y * d) + } + + /// Fly the orbit target through the scene. Moving the target carries the camera + /// with it, since the camera is derived as target + rotation * (0, 0, distance). + private func applyMovement(deltaTime: Float) { + guard moveInput != .zero, deltaTime > 0 else { return } + takeManualControl() + + let basis = cameraBasis + let right = SIMD3(basis.columns.0.x, basis.columns.0.y, basis.columns.0.z) + let up = SIMD3(basis.columns.1.x, basis.columns.1.y, basis.columns.1.z) + let forward = -SIMD3(basis.columns.2.x, basis.columns.2.y, basis.columns.2.z) + + // Scale by distance so flying feels the same zoomed in or out. + let speed = max(0.05, cameraDistance) * Constants.movementSpeed * moveBoost * deltaTime + cameraTarget += (right * moveInput.x + up * moveInput.y + forward * moveInput.z) * speed + } + + // MARK: - Camera interaction + + /// Freeze the idle spin at its current angle and hand control to the user. + private func takeManualControl() { + guard autoRotate else { return } + autoRotate = false + cameraYaw += Float(rotation.radians) + rotation = .zero + } + + func toggleAutoRotate() { + if autoRotate { takeManualControl() } else { autoRotate = true } + } + + /// Camera orientation as a rotation matrix: columns are right, up, and backward. + private var cameraBasis: matrix_float4x4 { + matrix4x4_rotation(radians: cameraYaw + Float(rotation.radians), axis: Constants.rotationAxis) + * matrix4x4_rotation(radians: cameraPitch, axis: SIMD3(1, 0, 0)) + } + + /// Where the eye actually sits. The orbit target is the pivot, not the camera. + private var cameraPosition: SIMD3 { + let backward = SIMD3(cameraBasis.columns.2.x, cameraBasis.columns.2.y, cameraBasis.columns.2.z) + return cameraTarget + backward * cameraDistance + } + + private func applyRotationDelta(dx: Float, dy: Float) { + let sensitivity: Float = 0.006 + cameraYaw += dx * sensitivity + let limit = Float.pi / 2 - 0.01 + cameraPitch = min(limit, max(-limit, cameraPitch - dy * sensitivity)) + } + + /// Turntable orbit: swings the eye around the pivot, so the eye moves. + func orbit(dx: Float, dy: Float) { + takeManualControl() + applyRotationDelta(dx: dx, dy: dy) + } + + /// Free-look: re-aims the camera while the eye stays put. Same rotation as + /// `orbit`, but the pivot is moved afterwards to hold the eye in place. + func look(dx: Float, dy: Float) { + takeManualControl() + let eye = cameraPosition + applyRotationDelta(dx: dx, dy: dy) + let backward = SIMD3(cameraBasis.columns.2.x, cameraBasis.columns.2.y, cameraBasis.columns.2.z) + cameraTarget = eye - backward * cameraDistance + } + + /// Slide the orbit target across the camera plane. Scaled by distance so it + /// feels the same whether you're across the room or up against a surface. + func pan(dx: Float, dy: Float) { + takeManualControl() + let basis = cameraBasis + let right = SIMD3(basis.columns.0.x, basis.columns.0.y, basis.columns.0.z) + let up = SIMD3(basis.columns.1.x, basis.columns.1.y, basis.columns.1.z) + let scale = cameraDistance * 0.0015 + cameraTarget += (-right * dx + up * dy) * scale + } + + /// Multiplicative dolly, so each notch moves a constant fraction of the way in. + func zoom(_ factor: Float) { + cameraDistance = min(1000, max(0.01, cameraDistance * factor)) + } + + func toggleUpCalibration() { + applyUpCalibration.toggle() + Self.log.info("up-axis calibration: \(self.applyUpCalibration ? "on (default)" : "off (flipped)")") + } - guard let lastRotationUpdateTimestamp else { return } - rotation += Constants.rotationPerSecond * now.timeIntervalSince(lastRotationUpdateTimestamp) + func resetCamera() { + autoRotate = true + rotation = .zero + cameraYaw = 0 + cameraPitch = 0 + cameraDistance = -Constants.modelCenterZ + cameraTarget = .zero } func draw(in view: MTKView) { @@ -125,7 +253,7 @@ class MetalKitSceneRenderer: NSObject, MTKViewDelegate { } let semaphore = inFlightSemaphore - commandBuffer.addCompletedHandler { (_ commandBuffer)-> Swift.Void in + commandBuffer.addCompletedHandler { @Sendable _ in semaphore.signal() } diff --git a/SampleApp/Scene/MetalKitSceneView.swift b/SampleApp/Scene/MetalKitSceneView.swift index f17b51f7..466757ed 100644 --- a/SampleApp/Scene/MetalKitSceneView.swift +++ b/SampleApp/Scene/MetalKitSceneView.swift @@ -2,6 +2,7 @@ import SwiftUI import MetalKit +import simd #if os(macOS) private typealias ViewRepresentable = NSViewRepresentable @@ -9,6 +10,217 @@ private typealias ViewRepresentable = NSViewRepresentable private typealias ViewRepresentable = UIViewRepresentable #endif + +/// A camera action that can be bound to a held key. +private enum CameraKey { + case forward, back, left, right, up, down + case lookUp, lookDown, lookLeft, lookRight +} + +/// MTKView subclass that turns platform input into camera moves. +/// Left-drag = orbit, right/cmd-drag = free-look (eye stays put), +/// shift/option/middle-drag = pan, scroll or pinch = zoom, +/// WASD or arrows = fly, Q/E = down/up, IJKL = look, shift = go faster. +class InteractiveMTKView: MTKView { + weak var camera: MetalKitSceneRenderer? + + /// Movement keys are tracked as a held set and applied per-frame by the renderer, + /// so motion is smooth and independent of the OS key-repeat rate. + private var held: Set = [] + + @discardableResult + private func setHeld(_ direction: CameraKey?, down: Bool) -> Bool { + guard let direction else { return false } + if down { held.insert(direction) } else { held.remove(direction) } + publishMovement() + return true + } + + private func publishMovement() { + var v = SIMD3(repeating: 0) + if held.contains(.right) { v.x += 1 } + if held.contains(.left) { v.x -= 1 } + if held.contains(.up) { v.y += 1 } + if held.contains(.down) { v.y -= 1 } + if held.contains(.forward) { v.z += 1 } + if held.contains(.back) { v.z -= 1 } + // Normalize so diagonals aren't faster than the cardinal directions. + camera?.moveInput = (v == .zero) ? .zero : normalize(v) + + var l = SIMD2(repeating: 0) + if held.contains(.lookRight) { l.x += 1 } + if held.contains(.lookLeft) { l.x -= 1 } + if held.contains(.lookDown) { l.y += 1 } // +y matches mouse deltaY (down) + if held.contains(.lookUp) { l.y -= 1 } + camera?.lookInput = l +#if os(macOS) + camera?.moveBoost = NSEvent.modifierFlags.contains(.shift) ? Constants.movementBoost : 1 +#endif + } + +#if os(macOS) + override var acceptsFirstResponder: Bool { true } + + override func scrollWheel(with event: NSEvent) { + // Trackpads report far finer deltas than a wheel's discrete notches. + let step: Float = event.hasPreciseScrollingDeltas ? 0.004 : 0.03 + camera?.zoom(exp(Float(-event.scrollingDeltaY) * step)) + } + + override func magnify(with event: NSEvent) { + camera?.zoom(1 / Float(1 + event.magnification)) + } + + override func mouseDragged(with event: NSEvent) { + if event.modifierFlags.contains(.command) { + camera?.look(dx: Float(event.deltaX), dy: Float(event.deltaY)) + } else if event.modifierFlags.contains(.shift) || event.modifierFlags.contains(.option) { + camera?.pan(dx: Float(event.deltaX), dy: Float(event.deltaY)) + } else { + camera?.orbit(dx: Float(event.deltaX), dy: Float(event.deltaY)) + } + } + + override func rightMouseDragged(with event: NSEvent) { + camera?.look(dx: Float(event.deltaX), dy: Float(event.deltaY)) + } + + override func otherMouseDragged(with event: NSEvent) { + camera?.pan(dx: Float(event.deltaX), dy: Float(event.deltaY)) + } + + override func keyDown(with event: NSEvent) { + // Movement keys are positional (keyCode), so WASD stays under the left hand + // regardless of keyboard layout. + if setHeld(Self.direction(forKeyCode: event.keyCode), down: true) { return } + switch event.charactersIgnoringModifiers?.lowercased() { + case "r": camera?.resetCamera() + case "f": camera?.toggleUpCalibration() + case " ": camera?.toggleAutoRotate() + case "=", "+": camera?.zoom(0.9) + case "-", "_": camera?.zoom(1 / 0.9) + default: super.keyDown(with: event) + } + } + + override func keyUp(with event: NSEvent) { + if setHeld(Self.direction(forKeyCode: event.keyCode), down: false) { return } + super.keyUp(with: event) + } + + override func flagsChanged(with event: NSEvent) { + publishMovement() + super.flagsChanged(with: event) + } + + override func resignFirstResponder() -> Bool { + // Otherwise a key held while focus moves away stays stuck down forever. + held.removeAll() + publishMovement() + return super.resignFirstResponder() + } + + private static func direction(forKeyCode code: UInt16) -> CameraKey? { + switch code { + case 13, 126: return .forward // W, up arrow + case 1, 125: return .back // S, down arrow + case 0, 123: return .left // A, left arrow + case 2, 124: return .right // D, right arrow + case 14, 116: return .up // E, page up + case 12, 121: return .down // Q, page down + case 34: return .lookUp // I + case 40: return .lookDown // K + case 38: return .lookLeft // J + case 37: return .lookRight // L + default: return nil + } + } +#elseif os(iOS) + override var canBecomeFirstResponder: Bool { true } + + private var lastPoint: CGPoint? + private var lastPinch: CGFloat? + + override func touchesBegan(_ touches: Set, with event: UIEvent?) { + lastPoint = nil + lastPinch = nil + } + + override func touchesMoved(_ touches: Set, with event: UIEvent?) { + let all = (event?.allTouches ?? touches).sorted { $0.hashValue < $1.hashValue } + if all.count >= 2 { + let a = all[0].location(in: self), b = all[1].location(in: self) + let spread = hypot(b.x - a.x, b.y - a.y) + let mid = CGPoint(x: (a.x + b.x) / 2, y: (a.y + b.y) / 2) + if let lastPinch, lastPinch > 0, spread > 0 { + camera?.zoom(Float(lastPinch / spread)) + } + if let lastPoint { + camera?.pan(dx: Float(mid.x - lastPoint.x), dy: Float(mid.y - lastPoint.y)) + } + lastPinch = spread + lastPoint = mid + } else if let touch = all.first { + let p = touch.location(in: self) + if let lastPoint { + camera?.orbit(dx: Float(p.x - lastPoint.x), dy: Float(p.y - lastPoint.y)) + } + lastPoint = p + lastPinch = nil + } + } + + override func touchesEnded(_ touches: Set, with event: UIEvent?) { + lastPoint = nil + lastPinch = nil + } + + override func touchesCancelled(_ touches: Set, with event: UIEvent?) { + lastPoint = nil + lastPinch = nil + } + + // Hardware keyboards attached to an iPad. + override func pressesBegan(_ presses: Set, with event: UIPressesEvent?) { + var handled = false + for press in presses where setHeld(Self.direction(for: press.key?.keyCode), down: true) { + handled = true + } + if !handled { super.pressesBegan(presses, with: event) } + } + + override func pressesEnded(_ presses: Set, with event: UIPressesEvent?) { + var handled = false + for press in presses where setHeld(Self.direction(for: press.key?.keyCode), down: false) { + handled = true + } + if !handled { super.pressesEnded(presses, with: event) } + } + + override func pressesCancelled(_ presses: Set, with event: UIPressesEvent?) { + held.removeAll() + publishMovement() + super.pressesCancelled(presses, with: event) + } + + private static func direction(for usage: UIKeyboardHIDUsage?) -> CameraKey? { + switch usage { + case .keyboardW, .keyboardUpArrow: return .forward + case .keyboardS, .keyboardDownArrow: return .back + case .keyboardA, .keyboardLeftArrow: return .left + case .keyboardD, .keyboardRightArrow: return .right + case .keyboardE, .keyboardPageUp: return .up + case .keyboardQ, .keyboardPageDown: return .down + case .keyboardI: return .lookUp + case .keyboardK: return .lookDown + case .keyboardJ: return .lookLeft + case .keyboardL: return .lookRight + default: return nil + } + } +#endif +} + struct MetalKitSceneView: ViewRepresentable { var modelIdentifier: ModelIdentifier? @@ -31,7 +243,7 @@ struct MetalKitSceneView: ViewRepresentable { #endif private func makeView(_ coordinator: Coordinator) -> MTKView { - let metalKitView = MTKView() + let metalKitView = InteractiveMTKView() if let metalDevice = MTLCreateSystemDefaultDevice() { metalKitView.device = metalDevice @@ -40,6 +252,7 @@ struct MetalKitSceneView: ViewRepresentable { let renderer = MetalKitSceneRenderer(metalKitView) coordinator.renderer = renderer metalKitView.delegate = renderer + metalKitView.camera = renderer Task { do { @@ -64,6 +277,15 @@ struct MetalKitSceneView: ViewRepresentable { private func updateView(_ coordinator: Coordinator) { guard let renderer = coordinator.renderer else { return } + if let view = renderer.metalKitView as? InteractiveMTKView { +#if os(macOS) + if let window = view.window, window.firstResponder !== view { + window.makeFirstResponder(view) + } +#elseif os(iOS) + if !view.isFirstResponder { view.becomeFirstResponder() } +#endif + } Task { do { try await renderer.load(modelIdentifier) diff --git a/SampleApp/Scene/VisionSceneRenderer.swift b/SampleApp/Scene/VisionSceneRenderer.swift index 1230cae2..509d234f 100644 --- a/SampleApp/Scene/VisionSceneRenderer.swift +++ b/SampleApp/Scene/VisionSceneRenderer.swift @@ -23,7 +23,7 @@ extension LayerRenderer.Clock.Instant.Duration { /// - State changes are synchronized through the RendererTaskExecutor final class VisionSceneRenderer: @unchecked Sendable { private static let log = - Logger(subsystem: Bundle.main.bundleIdentifier!, + Logger(subsystem: Bundle.main.bundleIdentifier ?? "com.metalsplatter.sampleapp", category: "VisionSceneRenderer") let layerRenderer: LayerRenderer