hand adjsutments

package.json

@@ -7,7 +7,7 @@
     "dev": "npm run build && vite --host 0.0.0.0",
     "build": "tsc",
     "check": "tsc --noEmit",
-    "test": "npm run build && node --test tests/time.test.mjs tests/projection.test.mjs tests/media-controller.test.mjs tests/texture-manager.test.mjs tests/media-adapter.test.mjs",
+    "test": "npm run build && node --test tests/time.test.mjs tests/projection.test.mjs tests/media-controller.test.mjs tests/texture-manager.test.mjs tests/media-adapter.test.mjs tests/hand-aim.test.mjs",
     "preview": "npm run build && vite preview --host 127.0.0.1"
   },
   "devDependencies": {

src/vr180player/xr/hand-aim.ts

@@ -0,0 +1,125 @@
+export type VectorLike = {
+	x: number;
+	y: number;
+	z: number;
+};
+
+export type PalmAimInput = {
+	handedness?: string | null;
+	indexMetacarpal?: VectorLike | null;
+	middleMetacarpal?: VectorLike | null;
+	pinkyMetacarpal?: VectorLike | null;
+	ringMetacarpal?: VectorLike | null;
+	wrist?: VectorLike | null;
+};
+
+export type PalmAimRay = {
+	direction: VectorLike;
+	origin: VectorLike;
+};
+
+const MIN_AXIS_LENGTH_SQ = 0.000001;
+const PALM_SURFACE_OFFSET_METERS = 0.035;
+
+export function computePalmAimRay(input: PalmAimInput): PalmAimRay | null {
+	const { indexMetacarpal, pinkyMetacarpal, wrist } = input;
+	if (!indexMetacarpal || !pinkyMetacarpal || !wrist) {
+		return null;
+	}
+
+	const knuckleCenter = averageVectors([
+		indexMetacarpal,
+		input.middleMetacarpal,
+		input.ringMetacarpal,
+		pinkyMetacarpal
+	]);
+	if (!knuckleCenter) {
+		return null;
+	}
+
+	const fingerAxis = normalize(subtract(knuckleCenter, wrist));
+	const acrossPalmAxis = normalize(subtract(pinkyMetacarpal, indexMetacarpal));
+	if (!fingerAxis || !acrossPalmAxis) {
+		return null;
+	}
+
+	const direction = normalize(getPalmNormal(fingerAxis, acrossPalmAxis, input.handedness));
+	if (!direction) {
+		return null;
+	}
+
+	const palmCenter = lerp(wrist, knuckleCenter, 0.62);
+	const origin = add(palmCenter, scale(direction, PALM_SURFACE_OFFSET_METERS));
+	return { direction, origin };
+}
+
+function getPalmNormal(fingerAxis: VectorLike, acrossPalmAxis: VectorLike, handedness?: string | null): VectorLike {
+	if (handedness === 'left') {
+		return cross(acrossPalmAxis, fingerAxis);
+	}
+
+	return cross(fingerAxis, acrossPalmAxis);
+}
+
+function averageVectors(vectors: Array<VectorLike | null | undefined>): VectorLike | null {
+	const usableVectors = vectors.filter(Boolean) as VectorLike[];
+	if (usableVectors.length === 0) {
+		return null;
+	}
+
+	const total = usableVectors.reduce(
+		(sum, vector) => add(sum, vector),
+		{ x: 0, y: 0, z: 0 }
+	);
+	return scale(total, 1 / usableVectors.length);
+}
+
+function normalize(vector: VectorLike): VectorLike | null {
+	const lengthSq = vector.x * vector.x + vector.y * vector.y + vector.z * vector.z;
+	if (lengthSq < MIN_AXIS_LENGTH_SQ) {
+		return null;
+	}
+
+	const length = Math.sqrt(lengthSq);
+	return scale(vector, 1 / length);
+}
+
+function add(a: VectorLike, b: VectorLike): VectorLike {
+	return {
+		x: a.x + b.x,
+		y: a.y + b.y,
+		z: a.z + b.z
+	};
+}
+
+function subtract(a: VectorLike, b: VectorLike): VectorLike {
+	return {
+		x: a.x - b.x,
+		y: a.y - b.y,
+		z: a.z - b.z
+	};
+}
+
+function scale(vector: VectorLike, scalar: number): VectorLike {
+	return {
+		x: vector.x * scalar,
+		y: vector.y * scalar,
+		z: vector.z * scalar
+	};
+}
+
+function lerp(a: VectorLike, b: VectorLike, amount: number): VectorLike {
+	return {
+		x: a.x + (b.x - a.x) * amount,
+		y: a.y + (b.y - a.y) * amount,
+		z: a.z + (b.z - a.z) * amount
+	};
+}
+
+function cross(a: VectorLike, b: VectorLike): VectorLike {
+	return {
+		x: a.y * b.z - a.z * b.y,
+		y: a.z * b.x - a.x * b.z,
+		z: a.x * b.y - a.y * b.x
+	};
+}

src/vr180player/xr/vr-controller-interactions.ts

@@ -3,6 +3,10 @@ import {
 	getSeekProgressFromIntersection,
 	type VrControlPanel
 } from './vr-control-panel.js';
+import {
+	computePalmAimRay,
+	type VectorLike
+} from './hand-aim.js';
 
 type VrControllerSelectionOptions = {
 	exitVr: () => void;
@@ -108,8 +112,10 @@ export function createVrInputRig(scene: any, renderer: any, onSelectStart: (even
 				vrwpHand: hand
 			};
 			bindOverlayActivity(hand, overlayVisibility);
+			rememberHandedness(hand, { data: hand.inputState });
 			createHandOverlay(hand, index, overlayVisibility);
-			hand.addEventListener?.('connected', () => {
+			hand.addEventListener?.('connected', (event: any) => {
+				rememberHandedness(hand, event);
 				createHandOverlay(hand, index, overlayVisibility);
 				overlayVisibility.show();
 			});
@@ -238,29 +244,61 @@ function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): v
 
 function getHandAimRay(hand: any): AimRay | null {
 	const joints = hand?.joints;
-	const tipJoint = joints?.['index-finger-tip'];
+	if (!joints) {
-	const baseJoint = joints?.['index-finger-phalanx-proximal'] ||
-		joints?.['index-finger-metacarpal'] ||
-		joints?.wrist;
-
-	if (!tipJoint || !baseJoint) {
 		return null;
 	}
 
-	tipJoint.updateMatrixWorld?.(true);
+	const palmAimRay = computePalmAimRay({
-	baseJoint.updateMatrixWorld?.(true);
+		handedness: getHandedness(hand),
-
+		indexMetacarpal: getJointWorldPosition(joints['index-finger-metacarpal']),
-	const origin = tipJoint.getWorldPosition(new THREE.Vector3());
+		middleMetacarpal: getJointWorldPosition(joints['middle-finger-metacarpal']),
-	const base = baseJoint.getWorldPosition(new THREE.Vector3());
+		pinkyMetacarpal: getJointWorldPosition(joints['pinky-finger-metacarpal']),
-	const direction = origin.clone().sub(base);
+		ringMetacarpal: getJointWorldPosition(joints['ring-finger-metacarpal']),
-	if (direction.lengthSq() < 0.000001) {
+		wrist: getJointWorldPosition(joints.wrist)
+	});
+	if (!palmAimRay) {
 		return null;
 	}
 
-	direction.normalize();
+	const origin = toThreeVector(palmAimRay.origin);
+	const direction = toThreeVector(palmAimRay.direction);
 	return { direction, origin };
 }
 
+function rememberHandedness(hand: any, event: any): void {
+	const handedness = event?.data?.handedness ||
+		event?.data?.inputSource?.handedness ||
+		hand?.inputState?.handedness;
+
+	if (handedness !== 'left' && handedness !== 'right') {
+		return;
+	}
+
+	hand.userData = {
+		...hand.userData,
+		vrwpHandedness: handedness
+	};
+}
+
+function getHandedness(hand: any): string | undefined {
+	return hand?.userData?.vrwpHandedness ||
+		hand?.inputState?.handedness ||
+		hand?.userData?.inputSource?.handedness;
+}
+
+function getJointWorldPosition(joint: any): VectorLike | null {
+	if (!joint?.getWorldPosition) {
+		return null;
+	}
+
+	joint.updateMatrixWorld?.(true);
+	return joint.getWorldPosition(new THREE.Vector3());
+}
+
+function toThreeVector(vector: VectorLike): any {
+	return new THREE.Vector3(vector.x, vector.y, vector.z);
+}
+
 class VrOverlayVisibility {
 	private readonly fadeDurationMs: number;
 	private readonly hideDelayMs: number;

tests/hand-aim.test.mjs

@@ -0,0 +1,49 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { computePalmAimRay } from '../vr180player/xr/hand-aim.js';
+
+const wrist = { x: 0, y: 0, z: 0 };
+const middleMetacarpal = { x: 0, y: 0.1, z: 0 };
+const ringMetacarpal = { x: 0.025, y: 0.1, z: 0 };
+
+test('computePalmAimRay points out from a right palm instead of following the index finger', () => {
+	const ray = computePalmAimRay({
+		handedness: 'right',
+		indexMetacarpal: { x: -0.035, y: 0.1, z: 0 },
+		middleMetacarpal,
+		pinkyMetacarpal: { x: 0.055, y: 0.1, z: 0 },
+		ringMetacarpal,
+		wrist
+	});
+
+	assert.ok(ray);
+	assert.equal(Math.round(ray.direction.x * 1000) / 1000, 0);
+	assert.equal(Math.round(ray.direction.y * 1000) / 1000, 0);
+	assert.equal(Math.round(ray.direction.z * 1000) / 1000, -1);
+	assert.equal(Math.round(ray.origin.z * 1000) / 1000, -0.035);
+});
+
+test('computePalmAimRay flips the palm normal for a left hand', () => {
+	const ray = computePalmAimRay({
+		handedness: 'left',
+		indexMetacarpal: { x: 0.035, y: 0.1, z: 0 },
+		middleMetacarpal,
+		pinkyMetacarpal: { x: -0.055, y: 0.1, z: 0 },
+		ringMetacarpal: { x: -0.025, y: 0.1, z: 0 },
+		wrist
+	});
+
+	assert.ok(ray);
+	assert.equal(Math.round(ray.direction.z * 1000) / 1000, -1);
+});
+
+test('computePalmAimRay returns null when palm joints cannot define a stable ray', () => {
+	assert.equal(computePalmAimRay({ handedness: 'right', wrist }), null);
+	assert.equal(computePalmAimRay({
+		handedness: 'right',
+		indexMetacarpal: { x: 0, y: 0.1, z: 0 },
+		pinkyMetacarpal: { x: 0, y: 0.1, z: 0 },
+		wrist
+	}), null);
+});