From 82d5c31ab2c65b7896576613a36e2cf688f3dc14 Mon Sep 17 00:00:00 2001
From: Aiden <68633820+awils27@users.noreply.github.com>
Date: Wed, 10 Jun 2026 14:29:15 +1000
Subject: [PATCH] clean up and hand tracking
---
README.md | 3 +-
src/vr180player/vr180-player.ts | 10 +-
.../xr/vr-controller-interactions.ts | 427 +++++++++++++++++-
.../demo-xr-status.js | 0
demo.css => test-pages/demo.css | 0
index.html => test-pages/index.html | 2 +-
.../test-3d-image.html | 6 +-
.../test-3d-video.html | 8 +-
.../test-vr180-3d-image.html | 6 +-
.../test-vr180-3d-video.html | 8 +-
10 files changed, 431 insertions(+), 39 deletions(-)
rename demo-xr-status.js => test-pages/demo-xr-status.js (100%)
rename demo.css => test-pages/demo.css (100%)
rename index.html => test-pages/index.html (91%)
rename test-3d-image.html => test-pages/test-3d-image.html (75%)
rename test-3d-video.html => test-pages/test-3d-video.html (71%)
rename test-vr180-3d-image.html => test-pages/test-vr180-3d-image.html (74%)
rename test-vr180-3d-video.html => test-pages/test-vr180-3d-video.html (71%)
diff --git a/README.md b/README.md
index 3be5ed4..ffe1be4 100644
--- a/README.md
+++ b/README.md
@@ -54,10 +54,11 @@ When the page loads, the media is embedded normally with an entry button over it
- In WebXR, `plane` maps the left and right halves onto the matching eyes of a floating 16:9 plane.
- Outside WebXR, both modes render only the left half of the SBS media so viewers do not see the raw double image.
- Static images show only applicable controls; playback, seek, and mute controls are video-only.
+- Controller pointers and lightweight hand/controller overlays appear after controller interaction, then fade away after a short idle period so they do not distract from viewing.
- Control icons are embedded from [Lucide](https://lucide.dev/) SVG definitions, so no PNG icon assets are required.
## Demo
-Run `npm run build`, then open this repository's `index.html` through a local web server. The index page links to focused test pages for flat 3D image, VR180 3D image, flat 3D video, and VR180 3D video.
+Run `npm run build`, then open `test-pages/index.html` through a local web server. The test hub links to focused pages for flat 3D image, VR180 3D image, flat 3D video, and VR180 3D video. The root `index.html` redirects there for convenience.
For local experimentation, run:
diff --git a/src/vr180player/vr180-player.ts b/src/vr180player/vr180-player.ts
index 1a01e48..3276980 100644
--- a/src/vr180player/vr180-player.ts
+++ b/src/vr180player/vr180-player.ts
@@ -15,7 +15,7 @@ import { createMediaTexture as createMediaTextureCore } from './rendering/three-
import { FallbackCameraControls } from './modes/fallback-camera-controls.js';
import { MediaController } from './media/media-controller.js';
import {
- createVrController,
+ createVrInputRig,
handleVrControllerSelect
} from './xr/vr-controller-interactions.js';
import { bindVideoEvents } from './media/video-events.js';
@@ -42,6 +42,7 @@ let scene, camera, renderer, video, sphereMaterial;
let vr180Mesh, planeMesh, activeContentMesh;
let xrSession = null;
let raycaster, uiElements = [];
+let xrInputRig;
let mediaAdapter: SupportedMediaAdapter | undefined;
let playBtn;
let frameCounter = 0;
@@ -202,7 +203,8 @@ function init() {
vrPanelVisibility.setPanel(vrPanel);
uiElements.push(...vrPanel.interactables);
- raycaster = createVrController(scene, renderer, onSelectStartVR).raycaster;
+ xrInputRig = createVrInputRig(scene, renderer, onSelectStartVR);
+ raycaster = xrInputRig.raycaster;
} catch (e) {
console.error("INIT_ERROR (Phase 2 - VR Controls Setup):", e);
}
@@ -422,6 +424,7 @@ async function actualSessionToggle() {
try {
const session = await navigator.xr.requestSession('immersive-vr', {
requiredFeatures: ['local-floor'],
+ optionalFeatures: ['hand-tracking'],
});
if (!session) { throw new Error("requestSession returned no session."); }
@@ -462,6 +465,7 @@ async function actualSessionToggle() {
}
await renderer.xr.setSession(xrSession);
+ xrInputRig?.showOverlays();
isXrLoopActive = true;
renderer.setAnimationLoop(renderXR);
frameCounter = 0;
@@ -521,6 +525,7 @@ function onVRSessionEnd(event) {
if (vrControlPanel) {
vrPanelVisibility.hideImmediately();
}
+ xrInputRig?.hideOverlays();
if (endedSession && typeof endedSession.removeEventListener === 'function') {
endedSession.removeEventListener('end', onVRSessionEnd);
@@ -558,6 +563,7 @@ function renderXR(timestamp, frame) {
if (vrPanelVisibility.isFading) {
animatePanelFade(timestamp);
}
+ xrInputRig?.update(timestamp);
if (!frame) {
console.warn("renderXR called without an XRFrame. Skipping render.");
diff --git a/src/vr180player/xr/vr-controller-interactions.ts b/src/vr180player/xr/vr-controller-interactions.ts
index 9c645f6..b89469b 100644
--- a/src/vr180player/xr/vr-controller-interactions.ts
+++ b/src/vr180player/xr/vr-controller-interactions.ts
@@ -19,38 +19,130 @@ type VrControllerSelectionOptions = {
vrPanel: VrControlPanel | undefined;
};
+type VrInputRig = {
+ hideOverlays: () => void;
+ raycaster: any;
+ showOverlays: (timestamp?: number) => void;
+ update: (timestamp: number) => void;
+};
+
+type AimRay = {
+ direction: any;
+ origin: any;
+};
+
+type HandPointerOverlay = {
+ fallbackPointerOverlay: any;
+ hand: any;
+ pointerOverlay: any;
+};
+
+type VrOverlayVisibilityOptions = {
+ fadeDurationMs?: number;
+ hideDelayMs?: number;
+};
+
+const INPUT_OVERLAY_HIDE_DELAY_MS = 2500;
+const INPUT_OVERLAY_FADE_DURATION_MS = 200;
+const INPUT_OVERLAY_RENDER_ORDER = 10000;
+const POINTER_LENGTH = 5;
+const HAND_JOINT_NAMES = [
+ 'wrist',
+ 'thumb-metacarpal',
+ 'thumb-phalanx-proximal',
+ 'thumb-phalanx-distal',
+ 'thumb-tip',
+ 'index-finger-metacarpal',
+ 'index-finger-phalanx-proximal',
+ 'index-finger-phalanx-intermediate',
+ 'index-finger-phalanx-distal',
+ 'index-finger-tip',
+ 'middle-finger-metacarpal',
+ 'middle-finger-phalanx-proximal',
+ 'middle-finger-phalanx-intermediate',
+ 'middle-finger-phalanx-distal',
+ 'middle-finger-tip',
+ 'ring-finger-metacarpal',
+ 'ring-finger-phalanx-proximal',
+ 'ring-finger-phalanx-intermediate',
+ 'ring-finger-phalanx-distal',
+ 'ring-finger-tip',
+ 'pinky-finger-metacarpal',
+ 'pinky-finger-phalanx-proximal',
+ 'pinky-finger-phalanx-intermediate',
+ 'pinky-finger-phalanx-distal',
+ 'pinky-finger-tip'
+];
+const DEFAULT_RAY_DIRECTION = new THREE.Vector3(0, 0, -1);
const tempMatrix = new THREE.Matrix4();
-export function createVrController(scene: any, renderer: any, onSelectStart: (event: any) => void): {
- controller: any;
- raycaster: any;
-} {
- const controller = renderer.xr.getController(0);
- controller.addEventListener('selectstart', onSelectStart);
-
- const lineMaterial = new THREE.LineBasicMaterial({ color: 0xffffff, transparent: true, opacity: 0.5 });
- const lineGeometry = new THREE.BufferGeometry().setFromPoints([
- new THREE.Vector3(0, 0, 0),
- new THREE.Vector3(0, 0, -5)
- ]);
- controller.add(new THREE.Line(lineGeometry, lineMaterial));
- scene.add(controller);
-
+export function createVrInputRig(scene: any, renderer: any, onSelectStart: (event: any) => void): VrInputRig {
+ const overlayVisibility = new VrOverlayVisibility();
+ const handPointerOverlays: HandPointerOverlay[] = [];
const raycaster = new THREE.Raycaster();
raycaster.near = 0.1;
- raycaster.far = 5;
+ raycaster.far = POINTER_LENGTH;
- return { controller, raycaster };
+ for (let index = 0; index < 2; index += 1) {
+ const controller = renderer.xr.getController(index);
+ controller.addEventListener('selectstart', (event: any) => {
+ overlayVisibility.show();
+ onSelectStart(event);
+ });
+ bindOverlayActivity(controller, overlayVisibility);
+ const controllerPointerOverlay = createPointerOverlay(index, overlayVisibility);
+ controller.add(controllerPointerOverlay);
+ scene.add(controller);
+
+ const grip = renderer.xr.getControllerGrip?.(index);
+ if (grip) {
+ bindOverlayActivity(grip, overlayVisibility);
+ grip.add(createControllerOverlay(index, overlayVisibility));
+ scene.add(grip);
+ }
+
+ const hand = renderer.xr.getHand?.(index);
+ if (hand) {
+ controller.userData = {
+ ...controller.userData,
+ vrwpHand: hand
+ };
+ bindOverlayActivity(hand, overlayVisibility);
+ createHandOverlay(hand, index, overlayVisibility);
+ hand.addEventListener?.('connected', () => {
+ createHandOverlay(hand, index, overlayVisibility);
+ overlayVisibility.show();
+ });
+ scene.add(hand);
+
+ const handPointerOverlay = createWorldPointerOverlay(index, overlayVisibility);
+ scene.add(handPointerOverlay);
+ handPointerOverlays.push({
+ fallbackPointerOverlay: controllerPointerOverlay,
+ hand,
+ pointerOverlay: handPointerOverlay
+ });
+ }
+ }
+
+ overlayVisibility.hideImmediately();
+
+ return {
+ hideOverlays: () => overlayVisibility.hideImmediately(),
+ raycaster,
+ showOverlays: (timestamp?: number) => overlayVisibility.show(timestamp),
+ update: (timestamp: number) => {
+ updateHandPointerOverlays(handPointerOverlays);
+ overlayVisibility.update(timestamp);
+ }
+ };
}
export function handleVrControllerSelect(event: any, options: VrControllerSelectionOptions): void {
const controller = event.target;
if (!options.raycaster) return;
- controller.updateMatrixWorld();
- tempMatrix.identity().extractRotation(controller.matrixWorld);
- options.raycaster.ray.origin.setFromMatrixPosition(controller.matrixWorld);
- options.raycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix);
+ applySelectionRay(controller, options.raycaster);
const directIntersects = options.raycaster.intersectObjects(options.uiElements, true);
if (directIntersects.length === 0) {
@@ -107,3 +199,296 @@ function togglePanel(options: VrControllerSelectionOptions): void {
options.showPanel();
}
}
+
+function applySelectionRay(controller: any, raycaster: any): void {
+ const handRay = getHandAimRay(controller.userData?.vrwpHand);
+ if (handRay) {
+ raycaster.ray.origin.copy(handRay.origin);
+ raycaster.ray.direction.copy(handRay.direction);
+ return;
+ }
+
+ controller.updateMatrixWorld();
+ tempMatrix.identity().extractRotation(controller.matrixWorld);
+ raycaster.ray.origin.setFromMatrixPosition(controller.matrixWorld);
+ raycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix);
+}
+
+function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): void {
+ handPointerOverlays.forEach(({ fallbackPointerOverlay, hand, pointerOverlay }) => {
+ const handRay = getHandAimRay(hand);
+ const hasHandRay = Boolean(handRay);
+ pointerOverlay.userData = {
+ ...pointerOverlay.userData,
+ vrwpOverlayAvailable: hasHandRay
+ };
+ fallbackPointerOverlay.userData = {
+ ...fallbackPointerOverlay.userData,
+ vrwpOverlayAvailable: !hasHandRay
+ };
+
+ if (!handRay) {
+ return;
+ }
+
+ pointerOverlay.position.copy(handRay.origin);
+ pointerOverlay.quaternion.setFromUnitVectors(DEFAULT_RAY_DIRECTION, handRay.direction);
+ });
+}
+
+function getHandAimRay(hand: any): AimRay | null {
+ const joints = hand?.joints;
+ const tipJoint = joints?.['index-finger-tip'];
+ const baseJoint = joints?.['index-finger-phalanx-proximal'] ||
+ joints?.['index-finger-metacarpal'] ||
+ joints?.wrist;
+
+ if (!tipJoint || !baseJoint) {
+ return null;
+ }
+
+ tipJoint.updateMatrixWorld?.(true);
+ baseJoint.updateMatrixWorld?.(true);
+
+ const origin = tipJoint.getWorldPosition(new THREE.Vector3());
+ const base = baseJoint.getWorldPosition(new THREE.Vector3());
+ const direction = origin.clone().sub(base);
+ if (direction.lengthSq() < 0.000001) {
+ return null;
+ }
+
+ direction.normalize();
+ return { direction, origin };
+}
+
+class VrOverlayVisibility {
+ private readonly fadeDurationMs: number;
+ private readonly hideDelayMs: number;
+ private readonly objects: any[] = [];
+ private opacity = 0;
+ private targetOpacity = 0;
+ private visibleUntil = 0;
+
+ constructor({
+ fadeDurationMs = INPUT_OVERLAY_FADE_DURATION_MS,
+ hideDelayMs = INPUT_OVERLAY_HIDE_DELAY_MS
+ }: VrOverlayVisibilityOptions = {}) {
+ this.fadeDurationMs = fadeDurationMs;
+ this.hideDelayMs = hideDelayMs;
+ }
+
+ register(object: any): void {
+ this.objects.push(object);
+ this.setObjectVisible(object, this.targetOpacity > 0 || this.opacity > 0.001);
+ this.setObjectOpacity(object, this.opacity);
+ }
+
+ show(timestamp = performance.now()): void {
+ this.visibleUntil = timestamp + this.hideDelayMs;
+ this.targetOpacity = 1;
+ this.objects.forEach((object) => this.setObjectVisible(object, true));
+ }
+
+ hideImmediately(): void {
+ this.visibleUntil = 0;
+ this.opacity = 0;
+ this.targetOpacity = 0;
+ this.objects.forEach((object) => {
+ this.setObjectOpacity(object, 0);
+ this.setObjectVisible(object, false);
+ });
+ }
+
+ update(timestamp: number): void {
+ if (this.targetOpacity > 0 && timestamp >= this.visibleUntil) {
+ this.targetOpacity = 0;
+ }
+
+ if (this.opacity === this.targetOpacity) {
+ return;
+ }
+
+ const fadeStep = this.fadeDurationMs <= 0
+ ? 1
+ : Math.min(1, 16.67 / this.fadeDurationMs);
+ const direction = this.opacity < this.targetOpacity ? 1 : -1;
+ this.opacity = Math.max(0, Math.min(1, this.opacity + fadeStep * direction));
+
+ if ((direction > 0 && this.opacity >= this.targetOpacity) || (direction < 0 && this.opacity <= this.targetOpacity)) {
+ this.opacity = this.targetOpacity;
+ }
+
+ this.objects.forEach((object) => {
+ this.setObjectOpacity(object, this.opacity);
+ this.setObjectVisible(object, this.opacity > 0.001);
+ });
+ }
+
+ private setObjectVisible(object: any, isVisible: boolean): void {
+ const objectVisible = isVisible && object.userData?.vrwpOverlayAvailable !== false;
+ object.visible = objectVisible;
+ object.traverse?.((child: any) => {
+ child.visible = objectVisible;
+ });
+ }
+
+ private setObjectOpacity(object: any, opacity: number): void {
+ object.traverse?.((child: any) => {
+ const materials = Array.isArray(child.material) ? child.material : [child.material];
+ materials.filter(Boolean).forEach((material: any) => {
+ material.opacity = opacity;
+ material.transparent = true;
+ material.depthTest = false;
+ material.depthWrite = false;
+ material.needsUpdate = true;
+ });
+ });
+ }
+}
+
+function bindOverlayActivity(target: any, overlayVisibility: VrOverlayVisibility): void {
+ [
+ 'connected',
+ 'disconnected',
+ 'select',
+ 'selectend',
+ 'squeezestart',
+ 'squeeze',
+ 'squeezeend',
+ 'pinchstart',
+ 'pinchend'
+ ].forEach((eventName) => {
+ target.addEventListener?.(eventName, () => overlayVisibility.show());
+ });
+}
+
+function createPointerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+ const group = new THREE.Group();
+ group.name = `vrPointerOverlay${index}`;
+
+ const pointerMaterial = createOverlayLineMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.75);
+ const pointerGeometry = new THREE.BufferGeometry().setFromPoints([
+ new THREE.Vector3(0, 0, 0),
+ new THREE.Vector3(0, 0, -POINTER_LENGTH)
+ ]);
+ const pointerLine = new THREE.Line(pointerGeometry, pointerMaterial);
+ pointerLine.name = `vrPointerRay${index}`;
+ pointerLine.renderOrder = INPUT_OVERLAY_RENDER_ORDER;
+ group.add(pointerLine);
+
+ const tipMaterial = createOverlayMeshMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.9);
+ const tipMesh = new THREE.Mesh(new THREE.SphereGeometry(0.018, 12, 8), tipMaterial);
+ tipMesh.name = `vrPointerTip${index}`;
+ tipMesh.position.z = -POINTER_LENGTH;
+ tipMesh.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 1;
+ group.add(tipMesh);
+
+ overlayVisibility.register(group);
+ return group;
+}
+
+function createWorldPointerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+ const group = createPointerOverlay(index, overlayVisibility);
+ group.name = `vrHandPointerOverlay${index}`;
+ group.userData = {
+ ...group.userData,
+ vrwpOverlayAvailable: false
+ };
+ return group;
+}
+
+function createControllerOverlay(index: number, overlayVisibility: VrOverlayVisibility): any {
+ const group = new THREE.Group();
+ group.name = `vrControllerOverlay${index}`;
+
+ const material = createOverlayLineMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.8);
+ const outlineGeometry = new THREE.BufferGeometry().setFromPoints([
+ new THREE.Vector3(-0.045, -0.025, -0.08),
+ new THREE.Vector3(0.045, -0.025, -0.08),
+ new THREE.Vector3(0.055, 0.025, -0.02),
+ new THREE.Vector3(0.025, 0.035, 0.05),
+ new THREE.Vector3(-0.025, 0.035, 0.05),
+ new THREE.Vector3(-0.055, 0.025, -0.02),
+ new THREE.Vector3(-0.045, -0.025, -0.08)
+ ]);
+ const outline = new THREE.Line(outlineGeometry, material);
+ outline.name = `vrControllerOutline${index}`;
+ outline.renderOrder = INPUT_OVERLAY_RENDER_ORDER;
+ group.add(outline);
+
+ const originMaterial = createOverlayMeshMaterial(0xffffff, 0.75);
+ const origin = new THREE.Mesh(new THREE.SphereGeometry(0.012, 10, 6), originMaterial);
+ origin.name = `vrControllerOrigin${index}`;
+ origin.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 1;
+ group.add(origin);
+
+ overlayVisibility.register(group);
+ return group;
+}
+
+function createHandOverlay(hand: any, index: number, overlayVisibility: VrOverlayVisibility): void {
+ const joints = getHandJoints(hand);
+ if (joints.length === 0) {
+ return;
+ }
+
+ const material = createOverlayMeshMaterial(index === 0 ? 0x8fd8ff : 0xffc16b, 0.85);
+ joints.forEach(({ joint, name }) => {
+ if (!joint || joint.userData?.vrwpHandOverlayMarker) {
+ return;
+ }
+
+ const isTip = name.endsWith('tip');
+ const isWrist = name === 'wrist';
+ const radius = isWrist ? 0.014 : isTip ? 0.011 : 0.008;
+ const marker = new THREE.Mesh(new THREE.SphereGeometry(radius, 10, 6), material);
+ marker.name = `vrHandJointOverlay${index}-${name}`;
+ marker.renderOrder = INPUT_OVERLAY_RENDER_ORDER + 2;
+ marker.frustumCulled = false;
+ joint.add(marker);
+ joint.userData = {
+ ...joint.userData,
+ vrwpHandOverlayMarker: marker
+ };
+ overlayVisibility.register(marker);
+ });
+}
+
+function getHandJoints(hand: any): Array<{ joint: any; name: string }> {
+ const joints = hand?.joints;
+ if (!joints) {
+ return [];
+ }
+
+ const namedJoints = HAND_JOINT_NAMES
+ .map((name) => ({ joint: joints[name], name }))
+ .filter(({ joint }) => Boolean(joint));
+
+ if (namedJoints.length > 0) {
+ return namedJoints;
+ }
+
+ return Object.entries(joints)
+ .map(([name, joint]) => ({ joint, name }))
+ .filter(({ joint }) => Boolean(joint));
+}
+
+function createOverlayLineMaterial(color: number, opacity: number): any {
+ return new THREE.LineBasicMaterial({
+ color,
+ depthTest: false,
+ depthWrite: false,
+ opacity,
+ transparent: true
+ });
+}
+
+function createOverlayMeshMaterial(color: number, opacity: number): any {
+ return new THREE.MeshBasicMaterial({
+ color,
+ depthTest: false,
+ depthWrite: false,
+ opacity,
+ transparent: true
+ });
+}
diff --git a/demo-xr-status.js b/test-pages/demo-xr-status.js
similarity index 100%
rename from demo-xr-status.js
rename to test-pages/demo-xr-status.js
diff --git a/demo.css b/test-pages/demo.css
similarity index 100%
rename from demo.css
rename to test-pages/demo.css
diff --git a/index.html b/test-pages/index.html
similarity index 91%
rename from index.html
rename to test-pages/index.html
index f60822f..5a31ca3 100644
--- a/index.html
+++ b/test-pages/index.html
@@ -39,7 +39,7 @@
- Image tests use media/169_3d_test.png. Video tests expect media/sbs-video.mp4.
+ Image tests use files in ../media/. Video tests expect ../media/sbs-video.mp4.
diff --git a/test-3d-image.html b/test-pages/test-3d-image.html
similarity index 75%
rename from test-3d-image.html
rename to test-pages/test-3d-image.html
index 23419b3..30b2e0b 100644
--- a/test-3d-image.html
+++ b/test-pages/test-3d-image.html
@@ -5,7 +5,7 @@