diff --git a/package.json b/package.json index ca4b391..10e3108 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "dev": "npm run build && vite --host 0.0.0.0", "build": "tsc", "check": "tsc --noEmit", - "test": "npm run build && node --test tests/time.test.mjs tests/projection.test.mjs tests/media-controller.test.mjs tests/texture-manager.test.mjs tests/media-adapter.test.mjs", + "test": "npm run build && node --test tests/time.test.mjs tests/projection.test.mjs tests/media-controller.test.mjs tests/texture-manager.test.mjs tests/media-adapter.test.mjs tests/hand-aim.test.mjs", "preview": "npm run build && vite preview --host 127.0.0.1" }, "devDependencies": { diff --git a/src/vr180player/xr/hand-aim.ts b/src/vr180player/xr/hand-aim.ts new file mode 100644 index 0000000..8029476 --- /dev/null +++ b/src/vr180player/xr/hand-aim.ts @@ -0,0 +1,125 @@ +export type VectorLike = { + x: number; + y: number; + z: number; +}; + +export type PalmAimInput = { + handedness?: string | null; + indexMetacarpal?: VectorLike | null; + middleMetacarpal?: VectorLike | null; + pinkyMetacarpal?: VectorLike | null; + ringMetacarpal?: VectorLike | null; + wrist?: VectorLike | null; +}; + +export type PalmAimRay = { + direction: VectorLike; + origin: VectorLike; +}; + +const MIN_AXIS_LENGTH_SQ = 0.000001; +const PALM_SURFACE_OFFSET_METERS = 0.035; + +export function computePalmAimRay(input: PalmAimInput): PalmAimRay | null { + const { indexMetacarpal, pinkyMetacarpal, wrist } = input; + if (!indexMetacarpal || !pinkyMetacarpal || !wrist) { + return null; + } + + const knuckleCenter = averageVectors([ + indexMetacarpal, + input.middleMetacarpal, + input.ringMetacarpal, + pinkyMetacarpal + ]); + if (!knuckleCenter) { + return null; + } + + const fingerAxis = normalize(subtract(knuckleCenter, wrist)); + const acrossPalmAxis = normalize(subtract(pinkyMetacarpal, indexMetacarpal)); + if (!fingerAxis || !acrossPalmAxis) { + return null; + } + + const direction = normalize(getPalmNormal(fingerAxis, acrossPalmAxis, input.handedness)); + if (!direction) { + return null; + } + + const palmCenter = lerp(wrist, knuckleCenter, 0.62); + const origin = add(palmCenter, scale(direction, PALM_SURFACE_OFFSET_METERS)); + return { direction, origin }; +} + +function getPalmNormal(fingerAxis: VectorLike, acrossPalmAxis: VectorLike, handedness?: string | null): VectorLike { + if (handedness === 'left') { + return cross(acrossPalmAxis, fingerAxis); + } + + return cross(fingerAxis, acrossPalmAxis); +} + +function averageVectors(vectors: Array): VectorLike | null { + const usableVectors = vectors.filter(Boolean) as VectorLike[]; + if (usableVectors.length === 0) { + return null; + } + + const total = usableVectors.reduce( + (sum, vector) => add(sum, vector), + { x: 0, y: 0, z: 0 } + ); + return scale(total, 1 / usableVectors.length); +} + +function normalize(vector: VectorLike): VectorLike | null { + const lengthSq = vector.x * vector.x + vector.y * vector.y + vector.z * vector.z; + if (lengthSq < MIN_AXIS_LENGTH_SQ) { + return null; + } + + const length = Math.sqrt(lengthSq); + return scale(vector, 1 / length); +} + +function add(a: VectorLike, b: VectorLike): VectorLike { + return { + x: a.x + b.x, + y: a.y + b.y, + z: a.z + b.z + }; +} + +function subtract(a: VectorLike, b: VectorLike): VectorLike { + return { + x: a.x - b.x, + y: a.y - b.y, + z: a.z - b.z + }; +} + +function scale(vector: VectorLike, scalar: number): VectorLike { + return { + x: vector.x * scalar, + y: vector.y * scalar, + z: vector.z * scalar + }; +} + +function lerp(a: VectorLike, b: VectorLike, amount: number): VectorLike { + return { + x: a.x + (b.x - a.x) * amount, + y: a.y + (b.y - a.y) * amount, + z: a.z + (b.z - a.z) * amount + }; +} + +function cross(a: VectorLike, b: VectorLike): VectorLike { + return { + x: a.y * b.z - a.z * b.y, + y: a.z * b.x - a.x * b.z, + z: a.x * b.y - a.y * b.x + }; +} diff --git a/src/vr180player/xr/vr-controller-interactions.ts b/src/vr180player/xr/vr-controller-interactions.ts index b89469b..b7679bd 100644 --- a/src/vr180player/xr/vr-controller-interactions.ts +++ b/src/vr180player/xr/vr-controller-interactions.ts @@ -3,6 +3,10 @@ import { getSeekProgressFromIntersection, type VrControlPanel } from './vr-control-panel.js'; +import { + computePalmAimRay, + type VectorLike +} from './hand-aim.js'; type VrControllerSelectionOptions = { exitVr: () => void; @@ -108,8 +112,10 @@ export function createVrInputRig(scene: any, renderer: any, onSelectStart: (even vrwpHand: hand }; bindOverlayActivity(hand, overlayVisibility); + rememberHandedness(hand, { data: hand.inputState }); createHandOverlay(hand, index, overlayVisibility); - hand.addEventListener?.('connected', () => { + hand.addEventListener?.('connected', (event: any) => { + rememberHandedness(hand, event); createHandOverlay(hand, index, overlayVisibility); overlayVisibility.show(); }); @@ -238,29 +244,61 @@ function updateHandPointerOverlays(handPointerOverlays: HandPointerOverlay[]): v function getHandAimRay(hand: any): AimRay | null { const joints = hand?.joints; - const tipJoint = joints?.['index-finger-tip']; - const baseJoint = joints?.['index-finger-phalanx-proximal'] || - joints?.['index-finger-metacarpal'] || - joints?.wrist; - - if (!tipJoint || !baseJoint) { + if (!joints) { return null; } - tipJoint.updateMatrixWorld?.(true); - baseJoint.updateMatrixWorld?.(true); - - const origin = tipJoint.getWorldPosition(new THREE.Vector3()); - const base = baseJoint.getWorldPosition(new THREE.Vector3()); - const direction = origin.clone().sub(base); - if (direction.lengthSq() < 0.000001) { + const palmAimRay = computePalmAimRay({ + handedness: getHandedness(hand), + indexMetacarpal: getJointWorldPosition(joints['index-finger-metacarpal']), + middleMetacarpal: getJointWorldPosition(joints['middle-finger-metacarpal']), + pinkyMetacarpal: getJointWorldPosition(joints['pinky-finger-metacarpal']), + ringMetacarpal: getJointWorldPosition(joints['ring-finger-metacarpal']), + wrist: getJointWorldPosition(joints.wrist) + }); + if (!palmAimRay) { return null; } - direction.normalize(); + const origin = toThreeVector(palmAimRay.origin); + const direction = toThreeVector(palmAimRay.direction); return { direction, origin }; } +function rememberHandedness(hand: any, event: any): void { + const handedness = event?.data?.handedness || + event?.data?.inputSource?.handedness || + hand?.inputState?.handedness; + + if (handedness !== 'left' && handedness !== 'right') { + return; + } + + hand.userData = { + ...hand.userData, + vrwpHandedness: handedness + }; +} + +function getHandedness(hand: any): string | undefined { + return hand?.userData?.vrwpHandedness || + hand?.inputState?.handedness || + hand?.userData?.inputSource?.handedness; +} + +function getJointWorldPosition(joint: any): VectorLike | null { + if (!joint?.getWorldPosition) { + return null; + } + + joint.updateMatrixWorld?.(true); + return joint.getWorldPosition(new THREE.Vector3()); +} + +function toThreeVector(vector: VectorLike): any { + return new THREE.Vector3(vector.x, vector.y, vector.z); +} + class VrOverlayVisibility { private readonly fadeDurationMs: number; private readonly hideDelayMs: number; diff --git a/tests/hand-aim.test.mjs b/tests/hand-aim.test.mjs new file mode 100644 index 0000000..57e18d7 --- /dev/null +++ b/tests/hand-aim.test.mjs @@ -0,0 +1,49 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { computePalmAimRay } from '../vr180player/xr/hand-aim.js'; + +const wrist = { x: 0, y: 0, z: 0 }; +const middleMetacarpal = { x: 0, y: 0.1, z: 0 }; +const ringMetacarpal = { x: 0.025, y: 0.1, z: 0 }; + +test('computePalmAimRay points out from a right palm instead of following the index finger', () => { + const ray = computePalmAimRay({ + handedness: 'right', + indexMetacarpal: { x: -0.035, y: 0.1, z: 0 }, + middleMetacarpal, + pinkyMetacarpal: { x: 0.055, y: 0.1, z: 0 }, + ringMetacarpal, + wrist + }); + + assert.ok(ray); + assert.equal(Math.round(ray.direction.x * 1000) / 1000, 0); + assert.equal(Math.round(ray.direction.y * 1000) / 1000, 0); + assert.equal(Math.round(ray.direction.z * 1000) / 1000, -1); + assert.equal(Math.round(ray.origin.z * 1000) / 1000, -0.035); +}); + +test('computePalmAimRay flips the palm normal for a left hand', () => { + const ray = computePalmAimRay({ + handedness: 'left', + indexMetacarpal: { x: 0.035, y: 0.1, z: 0 }, + middleMetacarpal, + pinkyMetacarpal: { x: -0.055, y: 0.1, z: 0 }, + ringMetacarpal: { x: -0.025, y: 0.1, z: 0 }, + wrist + }); + + assert.ok(ray); + assert.equal(Math.round(ray.direction.z * 1000) / 1000, -1); +}); + +test('computePalmAimRay returns null when palm joints cannot define a stable ray', () => { + assert.equal(computePalmAimRay({ handedness: 'right', wrist }), null); + assert.equal(computePalmAimRay({ + handedness: 'right', + indexMetacarpal: { x: 0, y: 0.1, z: 0 }, + pinkyMetacarpal: { x: 0, y: 0.1, z: 0 }, + wrist + }), null); +});