Hand tracking

2026-06-10 14:54:56 +10:00
parent c1fbfd3b5e
commit ba3c2785d8
6 changed files with 436 additions and 32 deletions
--- a/src/vr180player/dom/icons.ts
+++ b/src/vr180player/dom/icons.ts
@@ -3,6 +3,7 @@ export type LucideIconName =
 	| 'play'
 	| 'pause'
 	| 'maximize'
+	| 'arrow-left'
 	| 'rotate-ccw'
 	| 'rotate-cw'
 	| 'volume-2'
@@ -32,6 +33,10 @@ const ICONS: Record<LucideIconName, readonly IconNode[]> = {
 		['path', { d: 'M3 16v3a2 2 0 0 0 2 2h3' }],
 		['path', { d: 'M16 21h3a2 2 0 0 0 2-2v-3' }]
 	],
+	'arrow-left': [
+		['path', { d: 'm12 19-7-7 7-7' }],
+		['path', { d: 'M19 12H5' }]
+	],
 	'rotate-ccw': [
 		['path', { d: 'M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8' }],
 		['path', { d: 'M3 3v5h5' }]
--- a/src/vr180player/vr180-player.ts
+++ b/src/vr180player/vr180-player.ts
@@ -273,6 +273,10 @@ function hidePanel() {
 	vrPanelVisibility.hide();
 }

+function getVisibleVrPanelInteractables() {
+	return vrPanelVisibility.isVisible ? (vrPanel?.interactables ?? []) : [];
+}
+
 function onWindowResize() {
 	if (!renderer) return;

@@ -563,7 +567,10 @@ function renderXR(timestamp, frame) {
 	if (vrPanelVisibility.isFading) {
 		animatePanelFade(timestamp);
 	}
-	xrInputRig?.update(timestamp);
+	const isInputHoveringVrPanel = xrInputRig?.update(timestamp, getVisibleVrPanelInteractables()) ?? false;
+	if (isInputHoveringVrPanel) {
+		vrPanelVisibility.show();
+	}

 	if (!frame) {
 		console.warn("renderXR called without an XRFrame. Skipping render.");
--- a/src/vr180player/xr/hand-aim.ts
+++ b/src/vr180player/xr/hand-aim.ts
@@ -18,8 +18,20 @@ export type PalmAimRay = {
 	origin: VectorLike;
 };

+export type TimedPalmAimRay = PalmAimRay & {
+	timestamp: number;
+};
+
+export type PalmAimLatch = {
+	isSelecting: boolean;
+	selectedRay: TimedPalmAimRay | null;
+	stableRay: TimedPalmAimRay | null;
+};
+
 const MIN_AXIS_LENGTH_SQ = 0.000001;
+const PALM_AIM_FORWARD_TILT_DEGREES = 40;
 const PALM_SURFACE_OFFSET_METERS = 0.035;
+export const DEFAULT_PALM_AIM_LATCH_MAX_AGE_MS = 300;

 export function computePalmAimRay(input: PalmAimInput): PalmAimRay | null {
 	const { indexMetacarpal, pinkyMetacarpal, wrist } = input;
@@ -43,7 +55,10 @@ export function computePalmAimRay(input: PalmAimInput): PalmAimRay | null {
 		return null;
 	}

-	const direction = normalize(getPalmNormal(fingerAxis, acrossPalmAxis, input.handedness));
+	const direction = normalize(getTiltedPalmDirection(
+		getPalmNormal(fingerAxis, acrossPalmAxis, input.handedness),
+		fingerAxis
+	));
 	if (!direction) {
 		return null;
 	}
@@ -53,6 +68,71 @@ export function computePalmAimRay(input: PalmAimInput): PalmAimRay | null {
 	return { direction, origin };
 }

+export function createPalmAimLatch(): PalmAimLatch {
+	return {
+		isSelecting: false,
+		selectedRay: null,
+		stableRay: null
+	};
+}
+
+export function recordStablePalmAimRay(
+	latch: PalmAimLatch | null | undefined,
+	ray: PalmAimRay,
+	timestamp: number
+): void {
+	if (!latch) {
+		return;
+	}
+
+	if (latch.isSelecting) {
+		return;
+	}
+
+	latch.stableRay = withTimestamp(ray, timestamp);
+}
+
+export function beginPalmAimSelection(
+	latch: PalmAimLatch | null | undefined,
+	timestamp: number,
+	maxAgeMs = DEFAULT_PALM_AIM_LATCH_MAX_AGE_MS
+): PalmAimRay | null {
+	if (!latch) {
+		return null;
+	}
+
+	latch.isSelecting = true;
+	const stableRay = getFreshTimedRay(latch.stableRay, timestamp, maxAgeMs);
+	latch.selectedRay = stableRay ? cloneTimedRay(stableRay) : null;
+	return latch.selectedRay ? clonePalmAimRay(latch.selectedRay) : null;
+}
+
+export function endPalmAimSelection(latch: PalmAimLatch | null | undefined): void {
+	if (!latch) {
+		return;
+	}
+
+	latch.isSelecting = false;
+	latch.selectedRay = null;
+}
+
+export function getPalmAimSelectionRay(
+	latch: PalmAimLatch | null | undefined,
+	timestamp: number,
+	maxAgeMs = DEFAULT_PALM_AIM_LATCH_MAX_AGE_MS
+): PalmAimRay | null {
+	if (!latch) {
+		return null;
+	}
+
+	if (latch.isSelecting && latch.selectedRay) {
+		return clonePalmAimRay(latch.selectedRay);
+	}
+
+	const stableRay = getFreshTimedRay(latch.stableRay, timestamp, maxAgeMs);
+	return stableRay ? clonePalmAimRay(stableRay) : null;
+}
+
 function getPalmNormal(fingerAxis: VectorLike, acrossPalmAxis: VectorLike, handedness?: string | null): VectorLike {
 	if (handedness === 'left') {
 		return cross(acrossPalmAxis, fingerAxis);
@@ -61,6 +141,56 @@ function getPalmNormal(fingerAxis: VectorLike, acrossPalmAxis: VectorLike, hande
 	return cross(fingerAxis, acrossPalmAxis);
 }

+function getTiltedPalmDirection(palmNormal: VectorLike, fingerAxis: VectorLike): VectorLike {
+	const tiltRadians = (PALM_AIM_FORWARD_TILT_DEGREES * Math.PI) / 180;
+	return add(
+		scale(palmNormal, Math.cos(tiltRadians)),
+		scale(fingerAxis, Math.sin(tiltRadians))
+	);
+}
+
+function withTimestamp(ray: PalmAimRay, timestamp: number): TimedPalmAimRay {
+	return {
+		...clonePalmAimRay(ray),
+		timestamp
+	};
+}
+
+function cloneTimedRay(ray: TimedPalmAimRay): TimedPalmAimRay {
+	return {
+		...clonePalmAimRay(ray),
+		timestamp: ray.timestamp
+	};
+}
+
+function clonePalmAimRay(ray: PalmAimRay): PalmAimRay {
+	return {
+		direction: cloneVector(ray.direction),
+		origin: cloneVector(ray.origin)
+	};
+}
+
+function cloneVector(vector: VectorLike): VectorLike {
+	return {
+		x: vector.x,
+		y: vector.y,
+		z: vector.z
+	};
+}
+
+function getFreshTimedRay(
+	ray: TimedPalmAimRay | null,
+	timestamp: number,
+	maxAgeMs: number
+): TimedPalmAimRay | null {
+	if (!ray) {
+		return null;
+	}
+
+	const ageMs = Math.max(0, timestamp - ray.timestamp);
+	return ageMs <= maxAgeMs ? ray : null;
+}
+
 function averageVectors(vectors: Array<VectorLike | null | undefined>): VectorLike | null {
 	const usableVectors = vectors.filter(Boolean) as VectorLike[];
 	if (usableVectors.length === 0) {
--- a/src/vr180player/xr/vr-control-panel.ts
+++ b/src/vr180player/xr/vr-control-panel.ts
@@ -182,7 +182,7 @@ export function createVrControlPanel(
 		centerY: FIGMA_EXIT_BUTTON_Y_PX,
 		name: 'vrExitButton',
 		size: FIGMA_EXIT_BUTTON_SIZE_PX,
-		texture: createLucideButtonTexture('log-out', '#ffffff', VR_BUTTON_TEXTURE_SIZE, VR_BUTTON_ICON_SIZE)
+		texture: createLucideButtonTexture('arrow-left', '#ffffff', VR_BUTTON_TEXTURE_SIZE, VR_BUTTON_ICON_SIZE)
 	});
 	group.add(exitButtonMesh);
 	interactables.push(exitButtonMesh);
--- a/src/vr180player/xr/vr-controller-interactions.ts
+++ b/src/vr180player/xr/vr-controller-interactions.ts
@@ -4,7 +4,14 @@ import {
 	type VrControlPanel
 } from './vr-control-panel.js';
 import {
+	beginPalmAimSelection,
 	computePalmAimRay,
+	createPalmAimLatch,
+	endPalmAimSelection,
+	getPalmAimSelectionRay,
+	recordStablePalmAimRay,
+	type PalmAimLatch,
+	type PalmAimRay,
 	type VectorLike
 } from './hand-aim.js';

@@ -27,7 +34,7 @@ type VrInputRig = {
 	hideOverlays: () => void;
 	raycaster: any;
 	showOverlays: (timestamp?: number) => void;
-	update: (timestamp: number) => void;
+	update: (timestamp: number, hoverTargets?: any[]) => boolean;
 };

 type AimRay = {
@@ -38,9 +45,18 @@ type AimRay = {
 type HandPointerOverlay = {
 	fallbackPointerOverlay: any;
 	hand: any;
+	handAimLatch: PalmAimLatch;
 	pointerOverlay: any;
 };

+type VrInputSource = {
+	controller: any;
+	controllerPointerOverlay: any;
+	hand?: any;
+	handAimLatch?: PalmAimLatch;
+	handPointerOverlay?: any;
+};
+
 type VrOverlayVisibilityOptions = {
 	fadeDurationMs?: number;
 	hideDelayMs?: number;
@@ -50,6 +66,8 @@ const INPUT_OVERLAY_HIDE_DELAY_MS = 2500;
 const INPUT_OVERLAY_FADE_DURATION_MS = 200;
 const INPUT_OVERLAY_RENDER_ORDER = 10000;
 const POINTER_LENGTH = 5;
+const POINTER_MIN_LENGTH = 0.06;
+const POINTER_HIT_SURFACE_OFFSET = 0.015;
 const HAND_JOINT_NAMES = [
 	'wrist',
 	'thumb-metacarpal',
@@ -83,18 +101,35 @@ const tempMatrix = new THREE.Matrix4();
 export function createVrInputRig(scene: any, renderer: any, onSelectStart: (event: any) => void): VrInputRig {
 	const overlayVisibility = new VrOverlayVisibility();
 	const handPointerOverlays: HandPointerOverlay[] = [];
+	const inputSources: VrInputSource[] = [];
 	const raycaster = new THREE.Raycaster();
 	raycaster.near = 0.1;
 	raycaster.far = POINTER_LENGTH;
+	const hoverRaycaster = new THREE.Raycaster();
+	hoverRaycaster.near = 0.1;
+	hoverRaycaster.far = POINTER_LENGTH;

 	for (let index = 0; index < 2; index += 1) {
 		const controller = renderer.xr.getController(index);
+		const controllerPointerOverlay = createPointerOverlay(index, overlayVisibility);
+		const inputSource: VrInputSource = {
+			controller,
+			controllerPointerOverlay
+		};
+		inputSources.push(inputSource);
 		controller.addEventListener('selectstart', (event: any) => {
-			overlayVisibility.show();
+			const timestamp = getEventTimestamp(event);
+			beginPalmAimSelection(controller.userData?.vrwpHandAimLatch, timestamp);
+			overlayVisibility.show(timestamp);
 			onSelectStart(event);
 		});
+		controller.addEventListener('selectend', () => {
+			endPalmAimSelection(controller.userData?.vrwpHandAimLatch);
+		});
+		controller.addEventListener('select', () => {
+			endPalmAimSelection(controller.userData?.vrwpHandAimLatch);
+		});
 		bindOverlayActivity(controller, overlayVisibility);
-		const controllerPointerOverlay = createPointerOverlay(index, overlayVisibility);
 		controller.add(controllerPointerOverlay);
 		scene.add(controller);

@@ -107,9 +142,17 @@ export function createVrInputRig(scene: any, renderer: any, onSelectStart: (even

 		const hand = renderer.xr.getHand?.(index);
 		if (hand) {
+			const handAimLatch = createPalmAimLatch();
+			inputSource.hand = hand;
+			inputSource.handAimLatch = handAimLatch;
 			controller.userData = {
 				...controller.userData,
-				vrwpHand: hand
+				vrwpHand: hand,
+				vrwpHandAimLatch: handAimLatch
+			};
+			hand.userData = {
+				...hand.userData,
+				vrwpAimLatch: handAimLatch
 			};
 			bindOverlayActivity(hand, overlayVisibility);
 			rememberHandedness(hand, { data: hand.inputState });
@@ -122,10 +165,12 @@ export function createVrInputRig(scene: any, renderer: any, onSelectStart: (even
 			scene.add(hand);

 			const handPointerOverlay = createWorldPointerOverlay(index, overlayVisibility);
+			inputSource.handPointerOverlay = handPointerOverlay;
 			scene.add(handPointerOverlay);
 			handPointerOverlays.push({
 				fallbackPointerOverlay: controllerPointerOverlay,
 				hand,
+				handAimLatch,
 				pointerOverlay: handPointerOverlay
 			});
 		}
@@ -137,9 +182,14 @@ export function createVrInputRig(scene: any, renderer: any, onSelectStart: (even
 		hideOverlays: () => overlayVisibility.hideImmediately(),
 		raycaster,
 		showOverlays: (timestamp?: number) => overlayVisibility.show(timestamp),
-		update: (timestamp: number) => {
-			updateHandPointerOverlays(handPointerOverlays);
+		update: (timestamp: number, hoverTargets: any[] = []) => {
+			updateHandPointerOverlays(handPointerOverlays, timestamp);
+			const isHovering = updateInputPointerIntersections(inputSources, hoverTargets, hoverRaycaster, timestamp);
+			if (isHovering) {
+				overlayVisibility.show(timestamp);
+			}
 			overlayVisibility.update(timestamp);
+			return isHovering;
 		}
 	};
 }
@@ -207,7 +257,7 @@ function togglePanel(options: VrControllerSelectionOptions): void {
 }

 function applySelectionRay(controller: any, raycaster: any): void {
-	const handRay = getHandAimRay(controller.userData?.vrwpHand);
+	const handRay = getSelectionHandAimRay(controller) || getHandAimRay(controller.userData?.vrwpHand);
 	if (handRay) {
 		raycaster.ray.origin.copy(handRay.origin);
 		raycaster.ray.direction.copy(handRay.direction);
@@ -220,10 +270,126 @@ function applySelectionRay(controller: any, raycaster: any): void {
 	raycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix);
 }

-function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): void {
-	handPointerOverlays.forEach(({ fallbackPointerOverlay, hand, pointerOverlay }) => {
-		const handRay = getHandAimRay(hand);
-		const hasHandRay = Boolean(handRay);
+function updateInputPointerIntersections(
+	inputSources: VrInputSource[],
+	hoverTargets: any[],
+	hoverRaycaster: any,
+	timestamp: number
+): boolean {
+	let isHoveringAnyTarget = false;
+
+	inputSources.forEach((inputSource) => {
+		resetInputPointerLengths(inputSource);
+		const aimRay = getInputSourceAimRay(inputSource, timestamp);
+		const pointerOverlay = getActivePointerOverlay(inputSource);
+		if (!aimRay || !pointerOverlay || hoverTargets.length === 0) {
+			return;
+		}
+
+		hoverRaycaster.ray.origin.copy(aimRay.origin);
+		hoverRaycaster.ray.direction.copy(aimRay.direction);
+		const intersections = hoverRaycaster.intersectObjects(hoverTargets, true);
+		if (intersections.length === 0) {
+			return;
+		}
+
+		isHoveringAnyTarget = true;
+		setPointerOverlayLength(pointerOverlay, getPointerIntersectionLength(intersections[0].distance));
+	});
+
+	return isHoveringAnyTarget;
+}
+
+function getInputSourceAimRay(inputSource: VrInputSource, timestamp: number): AimRay | null {
+	if (inputSource.hand && inputSource.handAimLatch) {
+		const latchedRay = getPalmAimSelectionRay(inputSource.handAimLatch, timestamp);
+		if (latchedRay) {
+			return toAimRay(latchedRay);
+		}
+
+		const handRay = getHandAimRay(inputSource.hand);
+		if (handRay) {
+			return handRay;
+		}
+	}
+
+	return getControllerAimRay(inputSource.controller);
+}
+
+function getControllerAimRay(controller: any): AimRay | null {
+	if (!controller) {
+		return null;
+	}
+
+	controller.updateMatrixWorld();
+	tempMatrix.identity().extractRotation(controller.matrixWorld);
+	const origin = new THREE.Vector3().setFromMatrixPosition(controller.matrixWorld);
+	const direction = new THREE.Vector3(0, 0, -1).applyMatrix4(tempMatrix);
+	return { direction, origin };
+}
+
+function resetInputPointerLengths(inputSource: VrInputSource): void {
+	setPointerOverlayLength(inputSource.controllerPointerOverlay, POINTER_LENGTH);
+	if (inputSource.handPointerOverlay) {
+		setPointerOverlayLength(inputSource.handPointerOverlay, POINTER_LENGTH);
+	}
+}
+
+function getActivePointerOverlay(inputSource: VrInputSource): any {
+	if (inputSource.handPointerOverlay && inputSource.handPointerOverlay.userData?.vrwpOverlayAvailable !== false) {
+		return inputSource.handPointerOverlay;
+	}
+
+	if (inputSource.controllerPointerOverlay && inputSource.controllerPointerOverlay.userData?.vrwpOverlayAvailable !== false) {
+		return inputSource.controllerPointerOverlay;
+	}
+
+	return null;
+}
+
+function getPointerIntersectionLength(distance: number): number {
+	return Math.max(
+		POINTER_MIN_LENGTH,
+		Math.min(POINTER_LENGTH, distance - POINTER_HIT_SURFACE_OFFSET)
+	);
+}
+
+function setPointerOverlayLength(pointerOverlay: any, length: number): void {
+	if (!pointerOverlay || pointerOverlay.userData?.vrwpPointerLength === length) {
+		return;
+	}
+
+	const pointerLine = pointerOverlay.userData?.vrwpPointerLine;
+	const pointerTip = pointerOverlay.userData?.vrwpPointerTip;
+	const positionAttribute = pointerLine?.geometry?.getAttribute?.('position') ||
+		pointerLine?.geometry?.attributes?.position;
+
+	if (positionAttribute?.setXYZ) {
+		positionAttribute.setXYZ(1, 0, 0, -length);
+		positionAttribute.needsUpdate = true;
+		pointerLine.geometry.computeBoundingSphere?.();
+	}
+
+	if (pointerTip) {
+		pointerTip.position.z = -length;
+	}
+
+	pointerOverlay.userData = {
+		...pointerOverlay.userData,
+		vrwpPointerLength: length
+	};
+}
+
+function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[], timestamp: number): void {
+	handPointerOverlays.forEach(({ fallbackPointerOverlay, hand, handAimLatch, pointerOverlay }) => {
+		const currentHandRay = getHandAimRay(hand);
+		if (currentHandRay) {
+			recordStablePalmAimRay(handAimLatch, toPalmAimRay(currentHandRay), timestamp);
+		}
+
+		const selectionPalmRay = getPalmAimSelectionRay(handAimLatch, timestamp);
+		const displayHandRay = selectionPalmRay ? toAimRay(selectionPalmRay) : currentHandRay;
+		const hasHandRay = Boolean(displayHandRay);
 		pointerOverlay.userData = {
 			...pointerOverlay.userData,
 			vrwpOverlayAvailable: hasHandRay
@@ -233,15 +399,26 @@ function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): v
 			vrwpOverlayAvailable: !hasHandRay
 		};

-		if (!handRay) {
+		if (!displayHandRay) {
 			return;
 		}

-		pointerOverlay.position.copy(handRay.origin);
-		pointerOverlay.quaternion.setFromUnitVectors(DEFAULT_RAY_DIRECTION, handRay.direction);
+		pointerOverlay.position.copy(displayHandRay.origin);
+		pointerOverlay.quaternion.setFromUnitVectors(DEFAULT_RAY_DIRECTION, displayHandRay.direction);
 	});
 }

+function getSelectionHandAimRay(controller: any): AimRay | null {
+	const latch = controller.userData?.vrwpHandAimLatch ||
+		controller.userData?.vrwpHand?.userData?.vrwpAimLatch;
+	if (!latch) {
+		return null;
+	}
+
+	const palmAimRay = getPalmAimSelectionRay(latch, performance.now());
+	return palmAimRay ? toAimRay(palmAimRay) : null;
+}
+
 function getHandAimRay(hand: any): AimRay | null {
 	const joints = hand?.joints;
 	if (!joints) {
@@ -265,6 +442,20 @@ function getHandAimRay(hand: any): AimRay | null {
 	return { direction, origin };
 }

+function toPalmAimRay(ray: AimRay): PalmAimRay {
+	return {
+		direction: fromThreeVector(ray.direction),
+		origin: fromThreeVector(ray.origin)
+	};
+}
+
+function toAimRay(ray: PalmAimRay): AimRay {
+	return {
+		direction: toThreeVector(ray.direction),
+		origin: toThreeVector(ray.origin)
+	};
+}
+
 function rememberHandedness(hand: any, event: any): void {
 	const handedness = event?.data?.handedness ||
 		event?.data?.inputSource?.handedness ||
@@ -299,6 +490,18 @@ function toThreeVector(vector: VectorLike): any {
 	return new THREE.Vector3(vector.x, vector.y, vector.z);
 }

+function fromThreeVector(vector: any): VectorLike {
+	return {
+		x: vector.x,
+		y: vector.y,
+		z: vector.z
+	};
+}
+
+function getEventTimestamp(event: any): number {
+	return Number.isFinite(event?.timeStamp) ? event.timeStamp : performance.now();
+}
+
 class VrOverlayVisibility {
 	private readonly fadeDurationMs: number;
 	private readonly hideDelayMs: number;
@@ -421,6 +624,13 @@ function createPointerOverlay(index: number, overlayVisibility: VrOverlayVisibil
 	tipMesh.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 1;
 	group.add(tipMesh);

+	group.userData = {
+		...group.userData,
+		vrwpPointerLength: POINTER_LENGTH,
+		vrwpPointerLine: pointerLine,
+		vrwpPointerTip: tipMesh
+	};
+
 	overlayVisibility.register(group);
 	return group;
 }