clean up and hand tracking

2026-06-10 14:29:15 +10:00
parent 95b9bc7cdc
commit 82d5c31ab2
10 changed files with 431 additions and 39 deletions
--- a/src/vr180player/vr180-player.ts
+++ b/src/vr180player/vr180-player.ts
@@ -15,7 +15,7 @@ import { createMediaTexture as createMediaTextureCore } from './rendering/three-
 import { FallbackCameraControls } from './modes/fallback-camera-controls.js';
 import { MediaController } from './media/media-controller.js';
 import {
-	createVrController,
+	createVrInputRig,
 	handleVrControllerSelect
 } from './xr/vr-controller-interactions.js';
 import { bindVideoEvents } from './media/video-events.js';
@@ -42,6 +42,7 @@ let scene, camera, renderer, video, sphereMaterial;
 let vr180Mesh, planeMesh, activeContentMesh;
 let xrSession = null;
 let raycaster, uiElements = [];
+let xrInputRig;
 let mediaAdapter: SupportedMediaAdapter | undefined;
 let playBtn;
 let frameCounter = 0;
@@ -202,7 +203,8 @@ function init() {
 		vrPanelVisibility.setPanel(vrPanel);
 		uiElements.push(...vrPanel.interactables);

-		raycaster = createVrController(scene, renderer, onSelectStartVR).raycaster;
+		xrInputRig = createVrInputRig(scene, renderer, onSelectStartVR);
+		raycaster = xrInputRig.raycaster;
 	} catch (e) {
 		console.error("INIT_ERROR (Phase 2 - VR Controls Setup):", e);
 	}
@@ -422,6 +424,7 @@ async function actualSessionToggle() {
 		try {
 			const session = await navigator.xr.requestSession('immersive-vr', {
 				requiredFeatures: ['local-floor'],
+				optionalFeatures: ['hand-tracking'],
 			});
 			if (!session) { throw new Error("requestSession returned no session."); }

@@ -462,6 +465,7 @@ async function actualSessionToggle() {
 			}

 			await renderer.xr.setSession(xrSession);
+			xrInputRig?.showOverlays();
 			isXrLoopActive = true;
 			renderer.setAnimationLoop(renderXR);
 			frameCounter = 0;
@@ -521,6 +525,7 @@ function onVRSessionEnd(event) {
 	if (vrControlPanel) {
 		vrPanelVisibility.hideImmediately();
 	}
+	xrInputRig?.hideOverlays();

 	if (endedSession && typeof endedSession.removeEventListener === 'function') {
 		endedSession.removeEventListener('end', onVRSessionEnd);
@@ -558,6 +563,7 @@ function renderXR(timestamp, frame) {
 	if (vrPanelVisibility.isFading) {
 		animatePanelFade(timestamp);
 	}
+	xrInputRig?.update(timestamp);

 	if (!frame) {
 		console.warn("renderXR called without an XRFrame. Skipping render.");
--- a/src/vr180player/xr/vr-controller-interactions.ts
+++ b/src/vr180player/xr/vr-controller-interactions.ts
@@ -19,38 +19,130 @@ type VrControllerSelectionOptions = {
 	vrPanel: VrControlPanel | undefined;
 };

+type VrInputRig = {
+	hideOverlays: () => void;
+	raycaster: any;
+	showOverlays: (timestamp?: number) => void;
+	update: (timestamp: number) => void;
+};
+
+type AimRay = {
+	direction: any;
+	origin: any;
+};
+
+type HandPointerOverlay = {
+	fallbackPointerOverlay: any;
+	hand: any;
+	pointerOverlay: any;
+};
+
+type VrOverlayVisibilityOptions = {
+	fadeDurationMs?: number;
+	hideDelayMs?: number;
+};
+
+const INPUT_OVERLAY_HIDE_DELAY_MS = 2500;
+const INPUT_OVERLAY_FADE_DURATION_MS = 200;
+const INPUT_OVERLAY_RENDER_ORDER = 10000;
+const POINTER_LENGTH = 5;
+const HAND_JOINT_NAMES = [
+	'wrist',
+	'thumb-metacarpal',
+	'thumb-phalanx-proximal',
+	'thumb-phalanx-distal',
+	'thumb-tip',
+	'index-finger-metacarpal',
+	'index-finger-phalanx-proximal',
+	'index-finger-phalanx-intermediate',
+	'index-finger-phalanx-distal',
+	'index-finger-tip',
+	'middle-finger-metacarpal',
+	'middle-finger-phalanx-proximal',
+	'middle-finger-phalanx-intermediate',
+	'middle-finger-phalanx-distal',
+	'middle-finger-tip',
+	'ring-finger-metacarpal',
+	'ring-finger-phalanx-proximal',
+	'ring-finger-phalanx-intermediate',
+	'ring-finger-phalanx-distal',
+	'ring-finger-tip',
+	'pinky-finger-metacarpal',
+	'pinky-finger-phalanx-proximal',
+	'pinky-finger-phalanx-intermediate',
+	'pinky-finger-phalanx-distal',
+	'pinky-finger-tip'
+];
+const DEFAULT_RAY_DIRECTION = new THREE.Vector3(0, 0, -1);
 const tempMatrix = new THREE.Matrix4();

-export function createVrController(scene: any, renderer: any, onSelectStart: (event: any) => void): {
-	controller: any;
-	raycaster: any;
-} {
-	const controller = renderer.xr.getController(0);
-	controller.addEventListener('selectstart', onSelectStart);
-
-	const lineMaterial = new THREE.LineBasicMaterial({ color: 0xffffff, transparent: true, opacity: 0.5 });
-	const lineGeometry = new THREE.BufferGeometry().setFromPoints([
-		new THREE.Vector3(0, 0, 0),
-		new THREE.Vector3(0, 0, -5)
-	]);
-	controller.add(new THREE.Line(lineGeometry, lineMaterial));
-	scene.add(controller);
-
+export function createVrInputRig(scene: any, renderer: any, onSelectStart: (event: any) => void): VrInputRig {
+	const overlayVisibility = new VrOverlayVisibility();
+	const handPointerOverlays: HandPointerOverlay[] = [];
 	const raycaster = new THREE.Raycaster();
 	raycaster.near = 0.1;
-	raycaster.far = 5;
+	raycaster.far = POINTER_LENGTH;

-	return { controller, raycaster };
+	for (let index = 0; index < 2; index += 1) {
+		const controller = renderer.xr.getController(index);
+		controller.addEventListener('selectstart', (event: any) => {
+			overlayVisibility.show();
+			onSelectStart(event);
+		});
+		bindOverlayActivity(controller, overlayVisibility);
+		const controllerPointerOverlay = createPointerOverlay(index, overlayVisibility);
+		controller.add(controllerPointerOverlay);
+		scene.add(controller);
+
+		const grip = renderer.xr.getControllerGrip?.(index);
+		if (grip) {
+			bindOverlayActivity(grip, overlayVisibility);
+			grip.add(createControllerOverlay(index, overlayVisibility));
+			scene.add(grip);
+		}
+
+		const hand = renderer.xr.getHand?.(index);
+		if (hand) {
+			controller.userData = {
+				...controller.userData,
+				vrwpHand: hand
+			};
+			bindOverlayActivity(hand, overlayVisibility);
+			createHandOverlay(hand, index, overlayVisibility);
+			hand.addEventListener?.('connected', () => {
+				createHandOverlay(hand, index, overlayVisibility);
+				overlayVisibility.show();
+			});
+			scene.add(hand);
+
+			const handPointerOverlay = createWorldPointerOverlay(index, overlayVisibility);
+			scene.add(handPointerOverlay);
+			handPointerOverlays.push({
+				fallbackPointerOverlay: controllerPointerOverlay,
+				hand,
+				pointerOverlay: handPointerOverlay
+			});
+		}
+	}
+
+	overlayVisibility.hideImmediately();
+
+	return {
+		hideOverlays: () => overlayVisibility.hideImmediately(),
+		raycaster,
+		showOverlays: (timestamp?: number) => overlayVisibility.show(timestamp),
+		update: (timestamp: number) => {
+			updateHandPointerOverlays(handPointerOverlays);
+			overlayVisibility.update(timestamp);
+		}
+	};
 }

 export function handleVrControllerSelect(event: any, options: VrControllerSelectionOptions): void {
 	const controller = event.target;
 	if (!options.raycaster) return;

-	controller.updateMatrixWorld();
-	tempMatrix.identity().extractRotation(controller.matrixWorld);
-	options.raycaster.ray.origin.setFromMatrixPosition(controller.matrixWorld);
-	options.raycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix);
+	applySelectionRay(controller, options.raycaster);

 	const directIntersects = options.raycaster.intersectObjects(options.uiElements, true);
 	if (directIntersects.length === 0) {
@@ -107,3 +199,296 @@ function togglePanel(options: VrControllerSelectionOptions): void {
 		options.showPanel();
 	}
 }
+
+function applySelectionRay(controller: any, raycaster: any): void {
+	const handRay = getHandAimRay(controller.userData?.vrwpHand);
+	if (handRay) {
+		raycaster.ray.origin.copy(handRay.origin);
+		raycaster.ray.direction.copy(handRay.direction);
+		return;
+	}
+
+	controller.updateMatrixWorld();
+	tempMatrix.identity().extractRotation(controller.matrixWorld);
+	raycaster.ray.origin.setFromMatrixPosition(controller.matrixWorld);
+	raycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix);
+}
+
+function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): void {
+	handPointerOverlays.forEach(({ fallbackPointerOverlay, hand, pointerOverlay }) => {
+		const handRay = getHandAimRay(hand);
+		const hasHandRay = Boolean(handRay);
+		pointerOverlay.userData = {
+			...pointerOverlay.userData,
+			vrwpOverlayAvailable: hasHandRay
+		};
+		fallbackPointerOverlay.userData = {
+			...fallbackPointerOverlay.userData,
+			vrwpOverlayAvailable: !hasHandRay
+		};
+
+		if (!handRay) {
+			return;
+		}
+
+		pointerOverlay.position.copy(handRay.origin);
+		pointerOverlay.quaternion.setFromUnitVectors(DEFAULT_RAY_DIRECTION, handRay.direction);
+	});
+}
+
+function getHandAimRay(hand: any): AimRay | null {
+	const joints = hand?.joints;
+	const tipJoint = joints?.['index-finger-tip'];
+	const baseJoint = joints?.['index-finger-phalanx-proximal'] ||
+		joints?.['index-finger-metacarpal'] ||
+		joints?.wrist;
+
+	if (!tipJoint || !baseJoint) {
+		return null;
+	}
+
+	tipJoint.updateMatrixWorld?.(true);
+	baseJoint.updateMatrixWorld?.(true);
+
+	const origin = tipJoint.getWorldPosition(new THREE.Vector3());
+	const base = baseJoint.getWorldPosition(new THREE.Vector3());
+	const direction = origin.clone().sub(base);
+	if (direction.lengthSq() < 0.000001) {
+		return null;
+	}
+
+	direction.normalize();
+	return { direction, origin };
+}
+
+class VrOverlayVisibility {
+	private readonly fadeDurationMs: number;
+	private readonly hideDelayMs: number;
+	private readonly objects: any[] = [];
+	private opacity = 0;
+	private targetOpacity = 0;
+	private visibleUntil = 0;
+
+	constructor({
+		fadeDurationMs = INPUT_OVERLAY_FADE_DURATION_MS,
+		hideDelayMs = INPUT_OVERLAY_HIDE_DELAY_MS
+	}: VrOverlayVisibilityOptions = {}) {
+		this.fadeDurationMs = fadeDurationMs;
+		this.hideDelayMs = hideDelayMs;
+	}
+
+	register(object: any): void {
+		this.objects.push(object);
+		this.setObjectVisible(object, this.targetOpacity > 0 || this.opacity > 0.001);
+		this.setObjectOpacity(object, this.opacity);
+	}
+
+	show(timestamp = performance.now()): void {
+		this.visibleUntil = timestamp + this.hideDelayMs;
+		this.targetOpacity = 1;
+		this.objects.forEach((object) => this.setObjectVisible(object, true));
+	}
+
+	hideImmediately(): void {
+		this.visibleUntil = 0;
+		this.opacity = 0;
+		this.targetOpacity = 0;
+		this.objects.forEach((object) => {
+			this.setObjectOpacity(object, 0);
+			this.setObjectVisible(object, false);
+		});
+	}
+
+	update(timestamp: number): void {
+		if (this.targetOpacity > 0 && timestamp >= this.visibleUntil) {
+			this.targetOpacity = 0;
+		}
+
+		if (this.opacity === this.targetOpacity) {
+			return;
+		}
+
+		const fadeStep = this.fadeDurationMs <= 0
+			? 1
+			: Math.min(1, 16.67 / this.fadeDurationMs);
+		const direction = this.opacity < this.targetOpacity ? 1 : -1;
+		this.opacity = Math.max(0, Math.min(1, this.opacity + fadeStep * direction));
+
+		if ((direction > 0 && this.opacity >= this.targetOpacity) || (direction < 0 && this.opacity <= this.targetOpacity)) {
+			this.opacity = this.targetOpacity;
+		}
+
+		this.objects.forEach((object) => {
+			this.setObjectOpacity(object, this.opacity);
+			this.setObjectVisible(object, this.opacity > 0.001);
+		});
+	}
+
+	private setObjectVisible(object: any, isVisible: boolean): void {
+		const objectVisible = isVisible && object.userData?.vrwpOverlayAvailable !== false;
+		object.visible = objectVisible;
+		object.traverse?.((child: any) => {
+			child.visible = objectVisible;
+		});
+	}
+
+	private setObjectOpacity(object: any, opacity: number): void {
+		object.traverse?.((child: any) => {
+			const materials = Array.isArray(child.material) ? child.material : [child.material];
+			materials.filter(Boolean).forEach((material: any) => {
+				material.opacity = opacity;
+				material.transparent = true;
+				material.depthTest = false;
+				material.depthWrite = false;
+				material.needsUpdate = true;
+			});
+		});
+	}
+}
+
+function bindOverlayActivity(target: any, overlayVisibility: VrOverlayVisibility): void {
+	[
+		'connected',
+		'disconnected',
+		'select',
+		'selectend',
+		'squeezestart',
+		'squeeze',
+		'squeezeend',
+		'pinchstart',
+		'pinchend'
+	].forEach((eventName) => {
+		target.addEventListener?.(eventName, () => overlayVisibility.show());
+	});
+}
+
+function createPointerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+	const group = new THREE.Group();
+	group.name = `vrPointerOverlay${index}`;
+
+	const pointerMaterial = createOverlayLineMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.75);
+	const pointerGeometry = new THREE.BufferGeometry().setFromPoints([
+		new THREE.Vector3(0, 0, 0),
+		new THREE.Vector3(0, 0, -POINTER_LENGTH)
+	]);
+	const pointerLine = new THREE.Line(pointerGeometry, pointerMaterial);
+	pointerLine.name = `vrPointerRay${index}`;
+	pointerLine.renderOrder = INPUT_OVERLAY_RENDER_ORDER;
+	group.add(pointerLine);
+
+	const tipMaterial = createOverlayMeshMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.9);
+	const tipMesh = new THREE.Mesh(new THREE.SphereGeometry(0.018, 12, 8), tipMaterial);
+	tipMesh.name = `vrPointerTip${index}`;
+	tipMesh.position.z = -POINTER_LENGTH;
+	tipMesh.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 1;
+	group.add(tipMesh);
+
+	overlayVisibility.register(group);
+	return group;
+}
+
+function createWorldPointerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+	const group = createPointerOverlay(index, overlayVisibility);
+	group.name = `vrHandPointerOverlay${index}`;
+	group.userData = {
+		...group.userData,
+		vrwpOverlayAvailable: false
+	};
+	return group;
+}
+
+function createControllerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+	const group = new THREE.Group();
+	group.name = `vrControllerOverlay${index}`;
+
+	const material = createOverlayLineMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.8);
+	const outlineGeometry = new THREE.BufferGeometry().setFromPoints([
+		new THREE.Vector3(-0.045, -0.025, -0.08),
+		new THREE.Vector3(0.045, -0.025, -0.08),
+		new THREE.Vector3(0.055, 0.025, -0.02),
+		new THREE.Vector3(0.025, 0.035, 0.05),
+		new THREE.Vector3(-0.025, 0.035, 0.05),
+		new THREE.Vector3(-0.055, 0.025, -0.02),
+		new THREE.Vector3(-0.045, -0.025, -0.08)
+	]);
+	const outline = new THREE.Line(outlineGeometry, material);
+	outline.name = `vrControllerOutline${index}`;
+	outline.renderOrder = INPUT_OVERLAY_RENDER_ORDER;
+	group.add(outline);
+
+	const originMaterial = createOverlayMeshMaterial(0xffffff, 0.75);
+	const origin = new THREE.Mesh(new THREE.SphereGeometry(0.012, 10, 6), originMaterial);
+	origin.name = `vrControllerOrigin${index}`;
+	origin.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 1;
+	group.add(origin);
+
+	overlayVisibility.register(group);
+	return group;
+}
+
+function createHandOverlay(hand: any, index: number, overlayVisibility: VrOverlayVisibility): void {
+	const joints = getHandJoints(hand);
+	if (joints.length === 0) {
+		return;
+	}
+
+	const material = createOverlayMeshMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.85);
+	joints.forEach(({ joint, name }) => {
+		if (!joint || joint.userData?.vrwpHandOverlayMarker) {
+			return;
+		}
+
+		const isTip = name.endsWith('tip');
+		const isWrist = name === 'wrist';
+		const radius = isWrist ? 0.014 : isTip ? 0.011 : 0.008;
+		const marker = new THREE.Mesh(new THREE.SphereGeometry(radius, 10, 6), material);
+		marker.name = `vrHandJointOverlay${index}-${name}`;
+		marker.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 2;
+		marker.frustumCulled = false;
+		joint.add(marker);
+		joint.userData = {
+			...joint.userData,
+			vrwpHandOverlayMarker: marker
+		};
+		overlayVisibility.register(marker);
+	});
+}
+
+function getHandJoints(hand: any): Array<{ joint: any; name: string }> {
+	const joints = hand?.joints;
+	if (!joints) {
+		return [];
+	}
+
+	const namedJoints = HAND_JOINT_NAMES
+		.map((name) => ({ joint: joints[name], name }))
+		.filter(({ joint }) => Boolean(joint));
+
+	if (namedJoints.length > 0) {
+		return namedJoints;
+	}
+
+	return Object.entries(joints)
+		.map(([name, joint]) => ({ joint, name }))
+		.filter(({ joint }) => Boolean(joint));
+}
+
+function createOverlayLineMaterial(color: number, opacity: number): any {
+	return new THREE.LineBasicMaterial({
+		color,
+		depthTest: false,
+		depthWrite: false,
+		opacity,
+		transparent: true
+	});
+}
+
+function createOverlayMeshMaterial(color: number, opacity: number): any {
+	return new THREE.MeshBasicMaterial({
+		color,
+		depthTest: false,
+		depthWrite: false,
+		opacity,
+		transparent: true
+	});
+}