Merge branch 'main' into fingertracking

server/core/protobuf_update.bat

@@ -1 +0,0 @@
-protoc --proto_path=../SlimeVR-OpenVR-Driver/src/bridge --java_out=./src/main/java ProtobufMessages.proto

server/core/src/main/java/dev/slimevr/VRServer.kt

@@ -10,6 +10,8 @@ import dev.slimevr.firmware.SerialFlashingHandler
 import dev.slimevr.games.vrchat.VRCConfigHandler
 import dev.slimevr.games.vrchat.VRCConfigHandlerStub
 import dev.slimevr.games.vrchat.VRChatConfigManager
+import dev.slimevr.inputs.Input
+import dev.slimevr.inputs.TapInputManager
 import dev.slimevr.osc.OSCHandler
 import dev.slimevr.osc.OSCRouter
 import dev.slimevr.osc.VMCHandler
@@ -117,6 +119,9 @@ class VRServer @JvmOverloads constructor(
 	@JvmField
 	val handshakeHandler = HandshakeHandler()
 
+	val tapInputManager: TapInputManager
+	val inputs: MutableList<Input> = FastList()
+
 	init {
 		// UwU
 		configManager = ConfigManager(configPath)
@@ -147,7 +152,9 @@ class VRServer @JvmOverloads constructor(
 		for (bridge in bridgeProvider(this, computedTrackers) + sequenceOf(WebSocketVRBridge(computedTrackers, this))) {
 			tasks.add(Runnable { bridge.startBridge() })
 			bridges.add(bridge)
+			bridge.addFingerBones(humanPoseManager.shareableFingerBones)
 		}
+		tapInputManager = TapInputManager(humanPoseManager.skeleton, this)
 
 		// Initialize OSC handlers
 		vrcOSCHandler = VRCOSCHandler(
@@ -245,11 +252,23 @@ class VRServer @JvmOverloads constructor(
 				tracker.tick(fpsTimer.timePerFrame)
 			}
 			humanPoseManager.update()
+
+			// Check for tap inputs
+			tapInputManager.update()
+			// Update bridges
 			for (bridge in bridges) {
+				// Send inputs to each bridge
+				for (input in inputs) {
+					bridge.sendInput(input)
+				}
 				bridge.dataWrite()
 			}
+			// Don't forget to clear inputs so that we don't send the same ones again
+			inputs.clear()
+
 			vrcOSCHandler.update()
 			vMCHandler.update()
+
 			// final long time = System.currentTimeMillis() - start;
 			try {
 				sleep(1) // 1000Hz

server/core/src/main/java/dev/slimevr/bridge/Bridge.kt

@@ -1,5 +1,7 @@
 package dev.slimevr.bridge
 
+import dev.slimevr.inputs.Input
+import dev.slimevr.tracking.processor.ShareableBone
 import dev.slimevr.tracking.trackers.Tracker
 import dev.slimevr.tracking.trackers.TrackerRole
 import dev.slimevr.util.ann.VRServerThread
@@ -38,6 +40,24 @@ interface Bridge {
 	@VRServerThread
 	fun removeSharedTracker(tracker: Tracker?)
 
+	/**
+	 * Adds a list of finger bones to the bridge. This should only be set
+	 * once the skeleton has initialized. Bridge will send finger data for
+	 * the hand trackers if it serves them.
+	 *
+	 * @param bones
+	 */
+	@VRServerThread
+	fun addFingerBones(bones: List<ShareableBone>)
+
+	/**
+	 * Reports a virtual controller input to the driver.
+	 *
+	 * @param input the object containing the data about the input
+	 */
+	@VRServerThread
+	fun sendInput(input: Input)
+
 	@VRServerThread
 	fun startBridge()
 

server/core/src/main/java/dev/slimevr/inputs/Input.kt

@@ -0,0 +1,8 @@
+package dev.slimevr.inputs
+
+class Input(val rightHand: Boolean, val type: InputType)
+
+enum class InputType {
+	DOUBLE_TAP,
+	TRIPLE_TAP,
+}

server/core/src/main/java/dev/slimevr/inputs/TapInputManager.kt

@@ -0,0 +1,98 @@
+package dev.slimevr.inputs
+
+import dev.slimevr.VRServer
+import dev.slimevr.tracking.processor.skeleton.HumanSkeleton
+import dev.slimevr.tracking.processor.skeleton.TapDetection
+import dev.slimevr.tracking.trackers.Tracker
+import dev.slimevr.util.ann.VRServerThread
+
+/**
+ * Handles tap detection for SteamVR virtual controller input
+ */
+class TapInputManager(
+	private val skeleton: HumanSkeleton,
+	private val vrServer: VRServer,
+) {
+
+	// tap detectors
+	private lateinit var leftDoubleDetector: TapDetection
+	private lateinit var leftTripleDetector: TapDetection
+	private lateinit var rightDoubleDetector: TapDetection
+	private lateinit var rightTripleDetector: TapDetection
+
+	val NS_CONVERTER: Float = 1.0e9f
+	private var doubleTapDelay = 0.25f * NS_CONVERTER
+
+	init {
+		reinit()
+	}
+
+	fun reinit() {
+		// Create tap detectors for both hands
+		leftDoubleDetector = TapDetection(skeleton, trackerToWatchLeftHand)
+		leftDoubleDetector.enabled = true
+		leftDoubleDetector.setMaxTaps(2)
+
+		leftTripleDetector = TapDetection(skeleton, trackerToWatchLeftHand)
+		leftTripleDetector.enabled = true
+		leftTripleDetector.setMaxTaps(3)
+
+		rightDoubleDetector = TapDetection(skeleton, trackerToWatchRightHand)
+		rightDoubleDetector.enabled = true
+		rightDoubleDetector.setMaxTaps(2)
+
+		rightTripleDetector = TapDetection(skeleton, trackerToWatchRightHand)
+		rightTripleDetector.enabled = true
+		rightTripleDetector.setMaxTaps(3)
+	}
+
+	@VRServerThread
+	fun update() {
+		// update the tap detectors
+		leftDoubleDetector.update()
+		leftTripleDetector.update()
+		rightDoubleDetector.update()
+		rightTripleDetector.update()
+
+		// check if any tap detectors have detected taps
+		if (3 <= leftTripleDetector.taps) {
+			leftTripleDetector.resetDetector()
+			leftDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(false, InputType.TRIPLE_TAP))
+		} else if (2 <= leftDoubleDetector.taps && System.nanoTime() - rightDoubleDetector.detectionTime > doubleTapDelay) {
+			leftTripleDetector.resetDetector()
+			leftDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(false, InputType.DOUBLE_TAP))
+		}
+
+		if (3 <= rightTripleDetector.taps) {
+			rightTripleDetector.resetDetector()
+			rightDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(true, InputType.TRIPLE_TAP))
+		} else if (2 <= rightDoubleDetector.taps && System.nanoTime() - rightDoubleDetector.detectionTime > doubleTapDelay) {
+			rightTripleDetector.resetDetector()
+			rightDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(true, InputType.DOUBLE_TAP))
+		}
+	}
+
+	private val trackerToWatchLeftHand: Tracker?
+		get() = listOfNotNull(
+			skeleton.leftHandTracker,
+			skeleton.leftLowerArmTracker,
+			skeleton.leftUpperArmTracker,
+			skeleton.leftThumbDistalTracker,
+			skeleton.leftThumbProximalTracker,
+			skeleton.leftThumbMetacarpalTracker,
+		).firstOrNull()
+
+	private val trackerToWatchRightHand: Tracker?
+		get() = listOfNotNull(
+			skeleton.rightHandTracker,
+			skeleton.rightLowerArmTracker,
+			skeleton.rightUpperArmTracker,
+			skeleton.rightThumbDistalTracker,
+			skeleton.rightThumbProximalTracker,
+			skeleton.rightThumbMetacarpalTracker,
+		).firstOrNull()
+}

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.java

@@ -61,8 +61,8 @@ public class RPCSettingsBuilder {
 					&& config.getOSCTrackerRole(TrackerRole.RIGHT_FOOT, false),
 				config.getOSCTrackerRole(TrackerRole.LEFT_ELBOW, false)
 					&& config.getOSCTrackerRole(TrackerRole.RIGHT_ELBOW, false),
-				config.getOSCTrackerRole(TrackerRole.LEFT_HAND, false)
-					&& config.getOSCTrackerRole(TrackerRole.RIGHT_HAND, false)
+				config.getOSCTrackerRole(TrackerRole.LEFT_CONTROLLER, false)
+					&& config.getOSCTrackerRole(TrackerRole.RIGHT_CONTROLLER, false)
 			);
 		VRCOSCSettings.startVRCOSCSettings(fbb);
 		VRCOSCSettings.addOscSettings(fbb, generalSettingOffset);
@@ -164,8 +164,8 @@ public class RPCSettingsBuilder {
 					bridge.getShareSetting(TrackerRole.RIGHT_KNEE),
 					bridge.getShareSetting(TrackerRole.LEFT_ELBOW),
 					bridge.getShareSetting(TrackerRole.RIGHT_ELBOW),
-					bridge.getShareSetting(TrackerRole.LEFT_HAND),
-					bridge.getShareSetting(TrackerRole.RIGHT_HAND)
+					bridge.getShareSetting(TrackerRole.LEFT_CONTROLLER),
+					bridge.getShareSetting(TrackerRole.RIGHT_CONTROLLER)
 				);
 		}
 		return steamvrTrackerSettings;

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt

@@ -59,8 +59,8 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) {
 				bridge.changeShareSettings(TrackerRole.RIGHT_KNEE, req.steamVrTrackers().rightKnee())
 				bridge.changeShareSettings(TrackerRole.LEFT_ELBOW, req.steamVrTrackers().leftElbow())
 				bridge.changeShareSettings(TrackerRole.RIGHT_ELBOW, req.steamVrTrackers().rightElbow())
-				bridge.changeShareSettings(TrackerRole.LEFT_HAND, req.steamVrTrackers().leftHand())
-				bridge.changeShareSettings(TrackerRole.RIGHT_HAND, req.steamVrTrackers().rightHand())
+				bridge.changeShareSettings(TrackerRole.LEFT_CONTROLLER, req.steamVrTrackers().leftHand())
+				bridge.changeShareSettings(TrackerRole.RIGHT_CONTROLLER, req.steamVrTrackers().rightHand())
 				bridge.setAutomaticSharedTrackers(req.steamVrTrackers().automaticTrackerToggle())
 			}
 		}
@@ -128,8 +128,8 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) {
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_FOOT, trackers.feet())
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_ELBOW, trackers.elbows())
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_ELBOW, trackers.elbows())
-				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_HAND, trackers.hands())
-				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_HAND, trackers.hands())
+				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_CONTROLLER, trackers.hands())
+				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_CONTROLLER, trackers.hands())
 			}
 			vrcOSCConfig.oscqueryEnabled = req.vrcOsc().oscqueryEnabled()
 

server/core/src/main/java/dev/slimevr/tracking/processor/BoneType.kt

@@ -1,14 +1,58 @@
-package dev.slimevr.tracking.processor;
+@file:Suppress("ktlint:standard:no-wildcard-imports")
 
-import solarxr_protocol.datatypes.BodyPart;
+package dev.slimevr.tracking.processor
 
+import dev.slimevr.tracking.processor.BoneType.*
+import solarxr_protocol.datatypes.BodyPart
+
+fun BoneType?.isLeftFinger(): Boolean {
+	this?.let {
+		return it == LEFT_THUMB_METACARPAL ||
+			it == LEFT_THUMB_PROXIMAL ||
+			it == LEFT_THUMB_DISTAL ||
+			it == LEFT_INDEX_PROXIMAL ||
+			it == LEFT_INDEX_INTERMEDIATE ||
+			it == LEFT_INDEX_DISTAL ||
+			it == LEFT_MIDDLE_PROXIMAL ||
+			it == LEFT_MIDDLE_INTERMEDIATE ||
+			it == LEFT_MIDDLE_DISTAL ||
+			it == LEFT_RING_PROXIMAL ||
+			it == LEFT_RING_INTERMEDIATE ||
+			it == LEFT_RING_DISTAL ||
+			it == LEFT_LITTLE_PROXIMAL ||
+			it == LEFT_LITTLE_INTERMEDIATE ||
+			it == LEFT_LITTLE_DISTAL
+	}
+	return false
+}
+
+fun BoneType?.isRightFinger(): Boolean {
+	this?.let {
+		return it == RIGHT_THUMB_METACARPAL ||
+			it == RIGHT_THUMB_PROXIMAL ||
+			it == RIGHT_THUMB_DISTAL ||
+			it == RIGHT_INDEX_PROXIMAL ||
+			it == RIGHT_INDEX_INTERMEDIATE ||
+			it == RIGHT_INDEX_DISTAL ||
+			it == RIGHT_MIDDLE_PROXIMAL ||
+			it == RIGHT_MIDDLE_INTERMEDIATE ||
+			it == RIGHT_MIDDLE_DISTAL ||
+			it == RIGHT_RING_PROXIMAL ||
+			it == RIGHT_RING_INTERMEDIATE ||
+			it == RIGHT_RING_DISTAL ||
+			it == RIGHT_LITTLE_PROXIMAL ||
+			it == RIGHT_LITTLE_INTERMEDIATE ||
+			it == RIGHT_LITTLE_DISTAL
+	}
+	return false
+}
 
 /**
  * Keys for all the bones in the skeleton.
  */
-public enum BoneType {
+enum class BoneType {
 	HEAD(BodyPart.HEAD),
-	HEAD_TRACKER(),
+	HEAD_TRACKER,
 	NECK(BodyPart.NECK),
 	UPPER_CHEST(BodyPart.UPPER_CHEST),
 	CHEST_TRACKER,
@@ -71,17 +115,21 @@ public enum BoneType {
 	RIGHT_RING_DISTAL(BodyPart.RIGHT_RING_DISTAL),
 	RIGHT_LITTLE_PROXIMAL(BodyPart.RIGHT_LITTLE_PROXIMAL),
 	RIGHT_LITTLE_INTERMEDIATE(BodyPart.RIGHT_LITTLE_INTERMEDIATE),
-	RIGHT_LITTLE_DISTAL(BodyPart.RIGHT_LITTLE_DISTAL);
+	RIGHT_LITTLE_DISTAL(BodyPart.RIGHT_LITTLE_DISTAL),
+	;
 
-	public static final BoneType[] values = values();
+	@JvmField
+	val bodyPart: Int
 
-	public final int bodyPart;
-
-	BoneType() {
-		this.bodyPart = BodyPart.NONE;
+	constructor() {
+		this.bodyPart = BodyPart.NONE
 	}
 
-	BoneType(int associatedBodyPart) {
-		this.bodyPart = associatedBodyPart;
+	constructor(associatedBodyPart: Int) {
+		this.bodyPart = associatedBodyPart
+	}
+
+	companion object {
+		val values: Array<BoneType> = entries.toTypedArray()
 	}
 }

server/core/src/main/java/dev/slimevr/tracking/processor/HumanPoseManager.kt

@@ -34,6 +34,7 @@ import kotlin.math.*
  */
 class HumanPoseManager(val server: VRServer?) {
 	val computedTrackers: MutableList<Tracker> = FastList()
+	val shareableFingerBones: MutableList<ShareableBone> = FastList()
 	private val onSkeletonUpdated: MutableList<Consumer<HumanSkeleton>> = FastList()
 	private val skeletonConfigManager = SkeletonConfigManager(true, this)
 
@@ -45,6 +46,7 @@ class HumanPoseManager(val server: VRServer?) {
 	// #region Constructors
 	init {
 		initializeComputedHumanPoseTracker()
+		initializeShareableFingerBones()
 	}
 
 	init {
@@ -230,6 +232,39 @@ class HumanPoseManager(val server: VRServer?) {
 		connectComputedHumanPoseTrackers()
 	}
 
+	private fun initializeShareableFingerBones() {
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_METACARPAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_METACARPAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_DISTAL))
+	}
+
 	fun loadFromConfig(configManager: ConfigManager) {
 		skeletonConfigManager.loadFromConfig(configManager)
 	}

server/core/src/main/java/dev/slimevr/tracking/processor/ShareableBone.kt

@@ -0,0 +1,11 @@
+package dev.slimevr.tracking.processor
+
+import io.github.axisangles.ktmath.Quaternion
+
+class ShareableBone(val boneType: BoneType) {
+
+	/**
+	 * Returns the rotation of the bone relative to its parent
+	 */
+	var localRotation: Quaternion = Quaternion.IDENTITY
+}

server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/HumanSkeleton.kt

@@ -7,6 +7,7 @@ import dev.slimevr.tracking.processor.BoneType
 import dev.slimevr.tracking.processor.Constraint
 import dev.slimevr.tracking.processor.Constraint.Companion.ConstraintType
 import dev.slimevr.tracking.processor.HumanPoseManager
+import dev.slimevr.tracking.processor.ShareableBone
 import dev.slimevr.tracking.processor.config.SkeletonConfigToggles
 import dev.slimevr.tracking.processor.config.SkeletonConfigValues
 import dev.slimevr.tracking.processor.stayaligned.StayAligned
@@ -190,6 +191,38 @@ class HumanSkeleton(
 	var computedLeftHandTracker: Tracker? = null
 	var computedRightHandTracker: Tracker? = null
 
+	// Output bones
+	var shareableLeftThumbMetacarpalBone: ShareableBone? = null
+	var shareableLeftThumbProximalBone: ShareableBone? = null
+	var shareableLeftThumbDistalBone: ShareableBone? = null
+	var shareableLeftIndexProximalBone: ShareableBone? = null
+	var shareableLeftIndexIntermediateBone: ShareableBone? = null
+	var shareableLeftIndexDistalBone: ShareableBone? = null
+	var shareableLeftMiddleProximalBone: ShareableBone? = null
+	var shareableLeftMiddleIntermediateBone: ShareableBone? = null
+	var shareableLeftMiddleDistalBone: ShareableBone? = null
+	var shareableLeftRingProximalBone: ShareableBone? = null
+	var shareableLeftRingIntermediateBone: ShareableBone? = null
+	var shareableLeftRingDistalBone: ShareableBone? = null
+	var shareableLeftLittleProximalBone: ShareableBone? = null
+	var shareableLeftLittleIntermediateBone: ShareableBone? = null
+	var shareableLeftLittleDistalBone: ShareableBone? = null
+	var shareableRightThumbMetacarpalBone: ShareableBone? = null
+	var shareableRightThumbProximalBone: ShareableBone? = null
+	var shareableRightThumbDistalBone: ShareableBone? = null
+	var shareableRightIndexProximalBone: ShareableBone? = null
+	var shareableRightIndexIntermediateBone: ShareableBone? = null
+	var shareableRightIndexDistalBone: ShareableBone? = null
+	var shareableRightMiddleProximalBone: ShareableBone? = null
+	var shareableRightMiddleIntermediateBone: ShareableBone? = null
+	var shareableRightMiddleDistalBone: ShareableBone? = null
+	var shareableRightRingProximalBone: ShareableBone? = null
+	var shareableRightRingIntermediateBone: ShareableBone? = null
+	var shareableRightRingDistalBone: ShareableBone? = null
+	var shareableRightLittleProximalBone: ShareableBone? = null
+	var shareableRightLittleIntermediateBone: ShareableBone? = null
+	var shareableRightLittleDistalBone: ShareableBone? = null
+
 	// Toggles
 	private var extendedSpineModel = false
 	private var extendedPelvisModel = false
@@ -223,6 +256,7 @@ class HumanSkeleton(
 	init {
 		assembleSkeleton()
 		setComputedTrackers(humanPoseManager.computedTrackers)
+		setShareableBones(humanPoseManager.shareableFingerBones)
 	}
 
 	constructor(
@@ -455,6 +489,7 @@ class HumanSkeleton(
 
 		// Update tap detection's trackers
 		tapDetectionManager.updateConfig(trackers)
+		humanPoseManager.server?.tapInputManager?.reinit()
 
 		// Update bones tracker field
 		refreshBoneTracker()
@@ -492,6 +527,54 @@ class HumanSkeleton(
 		}
 	}
 
+	/**
+	 * Set shareable bones from list
+	 */
+	private fun setShareableBones(bones: List<ShareableBone>) {
+		for (b in bones) {
+			setShareableBone(b)
+		}
+	}
+
+	/**
+	 * Set shareable bone
+	 */
+	private fun setShareableBone(bone: ShareableBone) {
+		when (bone.boneType) {
+			BoneType.LEFT_THUMB_METACARPAL -> shareableLeftThumbMetacarpalBone = bone
+			BoneType.LEFT_THUMB_PROXIMAL -> shareableLeftThumbProximalBone = bone
+			BoneType.LEFT_THUMB_DISTAL -> shareableLeftThumbDistalBone = bone
+			BoneType.LEFT_INDEX_PROXIMAL -> shareableLeftIndexProximalBone = bone
+			BoneType.LEFT_INDEX_INTERMEDIATE -> shareableLeftIndexIntermediateBone = bone
+			BoneType.LEFT_INDEX_DISTAL -> shareableLeftIndexDistalBone = bone
+			BoneType.LEFT_MIDDLE_PROXIMAL -> shareableLeftMiddleProximalBone = bone
+			BoneType.LEFT_MIDDLE_INTERMEDIATE -> shareableLeftMiddleIntermediateBone = bone
+			BoneType.LEFT_MIDDLE_DISTAL -> shareableLeftMiddleDistalBone = bone
+			BoneType.LEFT_RING_PROXIMAL -> shareableLeftRingProximalBone = bone
+			BoneType.LEFT_RING_INTERMEDIATE -> shareableLeftRingIntermediateBone = bone
+			BoneType.LEFT_RING_DISTAL -> shareableLeftRingDistalBone = bone
+			BoneType.LEFT_LITTLE_PROXIMAL -> shareableLeftLittleProximalBone = bone
+			BoneType.LEFT_LITTLE_INTERMEDIATE -> shareableLeftLittleIntermediateBone = bone
+			BoneType.LEFT_LITTLE_DISTAL -> shareableLeftLittleDistalBone = bone
+			BoneType.RIGHT_THUMB_METACARPAL -> shareableRightThumbMetacarpalBone = bone
+			BoneType.RIGHT_THUMB_PROXIMAL -> shareableRightThumbProximalBone = bone
+			BoneType.RIGHT_THUMB_DISTAL -> shareableRightThumbDistalBone = bone
+			BoneType.RIGHT_INDEX_PROXIMAL -> shareableRightIndexProximalBone = bone
+			BoneType.RIGHT_INDEX_INTERMEDIATE -> shareableRightIndexIntermediateBone = bone
+			BoneType.RIGHT_INDEX_DISTAL -> shareableRightIndexDistalBone = bone
+			BoneType.RIGHT_MIDDLE_PROXIMAL -> shareableRightMiddleProximalBone = bone
+			BoneType.RIGHT_MIDDLE_INTERMEDIATE -> shareableRightMiddleIntermediateBone = bone
+			BoneType.RIGHT_MIDDLE_DISTAL -> shareableRightMiddleDistalBone = bone
+			BoneType.RIGHT_RING_PROXIMAL -> shareableRightRingProximalBone = bone
+			BoneType.RIGHT_RING_INTERMEDIATE -> shareableRightRingIntermediateBone = bone
+			BoneType.RIGHT_RING_DISTAL -> shareableRightRingDistalBone = bone
+			BoneType.RIGHT_LITTLE_PROXIMAL -> shareableRightLittleProximalBone = bone
+			BoneType.RIGHT_LITTLE_INTERMEDIATE -> shareableRightLittleIntermediateBone = bone
+			BoneType.RIGHT_LITTLE_DISTAL -> shareableRightLittleDistalBone = bone
+			else -> {}
+		}
+	}
+
 	/**
 	 * Get output tracker from TrackerRole
 	 */
@@ -505,8 +588,8 @@ class HumanSkeleton(
 		TrackerRole.RIGHT_FOOT -> computedRightFootTracker!!
 		TrackerRole.LEFT_ELBOW -> computedLeftElbowTracker!!
 		TrackerRole.RIGHT_ELBOW -> computedRightElbowTracker!!
-		TrackerRole.LEFT_HAND -> computedLeftHandTracker!!
-		TrackerRole.RIGHT_HAND -> computedRightHandTracker!!
+		TrackerRole.LEFT_CONTROLLER -> computedLeftHandTracker!!
+		TrackerRole.RIGHT_CONTROLLER -> computedRightHandTracker!!
 		else -> throw IllegalArgumentException("Unsupported computed tracker's TrackerRole in HumanSkeleton")
 	}
 
@@ -527,6 +610,7 @@ class HumanSkeleton(
 			headBone.updateWithConstraints(false)
 		}
 		updateComputedTrackers()
+		updateShareableBones()
 
 		// Don't run post-processing if the tracking is paused
 		if (pauseTracking) return
@@ -1148,6 +1232,45 @@ class HumanSkeleton(
 		}
 	}
 
+	private fun updateShareableBones() {
+		this.updateShareableBone(shareableLeftThumbMetacarpalBone, leftThumbMetacarpalBone)
+		this.updateShareableBone(shareableLeftThumbProximalBone, leftThumbProximalBone)
+		this.updateShareableBone(shareableLeftThumbDistalBone, leftThumbDistalBone)
+		this.updateShareableBone(shareableLeftIndexProximalBone, leftIndexProximalBone)
+		this.updateShareableBone(shareableLeftIndexIntermediateBone, leftIndexIntermediateBone)
+		this.updateShareableBone(shareableLeftIndexDistalBone, leftIndexDistalBone)
+		this.updateShareableBone(shareableLeftMiddleProximalBone, leftMiddleProximalBone)
+		this.updateShareableBone(shareableLeftMiddleIntermediateBone, leftMiddleIntermediateBone)
+		this.updateShareableBone(shareableLeftMiddleDistalBone, leftMiddleDistalBone)
+		this.updateShareableBone(shareableLeftRingProximalBone, leftRingProximalBone)
+		this.updateShareableBone(shareableLeftRingIntermediateBone, leftRingIntermediateBone)
+		this.updateShareableBone(shareableLeftRingDistalBone, leftRingDistalBone)
+		this.updateShareableBone(shareableLeftLittleProximalBone, leftLittleProximalBone)
+		this.updateShareableBone(shareableLeftLittleIntermediateBone, leftLittleIntermediateBone)
+		this.updateShareableBone(shareableLeftLittleDistalBone, leftLittleDistalBone)
+		this.updateShareableBone(shareableRightThumbMetacarpalBone, rightThumbMetacarpalBone)
+		this.updateShareableBone(shareableRightThumbProximalBone, rightThumbProximalBone)
+		this.updateShareableBone(shareableRightThumbDistalBone, rightThumbDistalBone)
+		this.updateShareableBone(shareableRightIndexProximalBone, rightIndexProximalBone)
+		this.updateShareableBone(shareableRightIndexIntermediateBone, rightIndexIntermediateBone)
+		this.updateShareableBone(shareableRightIndexDistalBone, rightIndexDistalBone)
+		this.updateShareableBone(shareableRightMiddleProximalBone, rightMiddleProximalBone)
+		this.updateShareableBone(shareableRightMiddleIntermediateBone, rightMiddleIntermediateBone)
+		this.updateShareableBone(shareableRightMiddleDistalBone, rightMiddleDistalBone)
+		this.updateShareableBone(shareableRightRingProximalBone, rightRingProximalBone)
+		this.updateShareableBone(shareableRightRingIntermediateBone, rightRingIntermediateBone)
+		this.updateShareableBone(shareableRightRingDistalBone, rightRingDistalBone)
+		this.updateShareableBone(shareableRightLittleProximalBone, rightLittleProximalBone)
+		this.updateShareableBone(shareableRightLittleIntermediateBone, rightLittleIntermediateBone)
+		this.updateShareableBone(shareableRightLittleDistalBone, rightLittleDistalBone)
+	}
+
+	private fun updateShareableBone(shareableBone: ShareableBone?, bone: Bone) {
+		shareableBone?.let {
+			it.localRotation = bone.getLocalRotation()
+		}
+	}
+
 	// Skeleton Config toggles
 	fun updateToggleState(configToggle: SkeletonConfigToggles, newValue: Boolean) {
 		when (configToggle) {

server/core/src/main/java/dev/slimevr/tracking/trackers/TrackerPosition.kt

@@ -130,8 +130,8 @@ enum class TrackerPosition(
 	RIGHT_LOWER_ARM("body:right_lower_arm", null, BodyPart.RIGHT_LOWER_ARM, 14),
 	LEFT_UPPER_ARM("body:left_upper_arm", TrackerRole.LEFT_ELBOW, BodyPart.LEFT_UPPER_ARM, 15),
 	RIGHT_UPPER_ARM("body:right_upper_arm", TrackerRole.RIGHT_ELBOW, BodyPart.RIGHT_UPPER_ARM, 16),
-	LEFT_HAND("body:left_hand", TrackerRole.LEFT_HAND, BodyPart.LEFT_HAND, 17),
-	RIGHT_HAND("body:right_hand", TrackerRole.RIGHT_HAND, BodyPart.RIGHT_HAND, 18),
+	LEFT_HAND("body:left_hand", TrackerRole.LEFT_CONTROLLER, BodyPart.LEFT_HAND, 17),
+	RIGHT_HAND("body:right_hand", TrackerRole.RIGHT_CONTROLLER, BodyPart.RIGHT_HAND, 18),
 	LEFT_SHOULDER("body:left_shoulder", TrackerRole.LEFT_SHOULDER, BodyPart.LEFT_SHOULDER, 19),
 	RIGHT_SHOULDER("body:right_shoulder", TrackerRole.RIGHT_SHOULDER, BodyPart.RIGHT_SHOULDER, 20),
 	LEFT_THUMB_METACARPAL("body:left_thumb_metacarpal", null, BodyPart.LEFT_THUMB_METACARPAL, 21),
@@ -216,12 +216,10 @@ enum class TrackerPosition(
 
 		@JvmStatic
 		fun getByTrackerRole(role: TrackerRole): TrackerPosition? {
-			// Hands TrackerPositions are bound to the hands TrackerRoles,
-			// so we hardcode getting those.
-			if (role == TrackerRole.LEFT_CONTROLLER) {
+			// Hands TrackerPositions are bound to the controllers TrackerRoles so we hardcode that case.
+			if (role == TrackerRole.LEFT_HAND) {
 				return LEFT_HAND
-			}
-			if (role == TrackerRole.RIGHT_CONTROLLER) {
+			} else if (role == TrackerRole.RIGHT_HAND) {
 				return RIGHT_HAND
 			}
 			return byTrackerRole[role]

server/core/src/main/java/dev/slimevr/websocketapi/WebSocketVRBridge.kt

@@ -6,6 +6,8 @@ import dev.slimevr.VRServer
 import dev.slimevr.VRServer.Companion.getNextLocalTrackerId
 import dev.slimevr.VRServer.Companion.instance
 import dev.slimevr.bridge.Bridge
+import dev.slimevr.inputs.Input
+import dev.slimevr.tracking.processor.ShareableBone
 import dev.slimevr.tracking.trackers.Tracker
 import dev.slimevr.tracking.trackers.TrackerPosition
 import dev.slimevr.tracking.trackers.TrackerStatus
@@ -227,6 +229,14 @@ class WebSocketVRBridge(
 		// TODO Auto-generated method stub
 	}
 
+	override fun addFingerBones(bones: List<ShareableBone>) {
+		// TODO Auto-generated method stub
+	}
+
+	override fun sendInput(input: Input) {
+		// TODO Auto-generated method stub
+	}
+
 	override fun startBridge() {
 		start()
 	}

server/desktop/build.gradle.kts

@@ -60,7 +60,7 @@ dependencies {
 
 	implementation("commons-cli:commons-cli:1.8.0")
 	implementation("org.apache.commons:commons-lang3:3.15.0")
-	implementation("com.google.protobuf:protobuf-java:3.21.12")
+	implementation("com.google.protobuf:protobuf-java:4.31.1")
 	implementation("net.java.dev.jna:jna:5.+")
 	implementation("net.java.dev.jna:jna-platform:5.+")
 	implementation("com.fazecast:jSerialComm:2.11.2")

server/desktop/protobuf_update.bat

@@ -0,0 +1 @@
+protoc --proto_path=../../../SlimeVR-OpenVR-Driver/src/bridge --java_out=./src/main/java ProtobufMessages.proto

server/desktop/src/main/java/dev/slimevr/desktop/platform/ProtobufBridge.kt

@@ -4,24 +4,34 @@ import dev.slimevr.VRServer.Companion.instance
 import dev.slimevr.bridge.BridgeThread
 import dev.slimevr.bridge.ISteamVRBridge
 import dev.slimevr.desktop.platform.ProtobufMessages.*
+import dev.slimevr.inputs.InputType
+import dev.slimevr.tracking.processor.BoneType
+import dev.slimevr.tracking.processor.ShareableBone
+import dev.slimevr.tracking.processor.isLeftFinger
+import dev.slimevr.tracking.processor.isRightFinger
 import dev.slimevr.tracking.trackers.Tracker
+import dev.slimevr.tracking.trackers.TrackerPosition
 import dev.slimevr.tracking.trackers.TrackerStatus
 import dev.slimevr.tracking.trackers.TrackerStatus.Companion.getById
 import dev.slimevr.util.ann.VRServerThread
 import io.eiren.util.ann.Synchronize
 import io.eiren.util.ann.ThreadSafe
 import io.eiren.util.collections.FastList
+import io.eiren.util.logging.LogManager
 import io.github.axisangles.ktmath.Quaternion
 import io.github.axisangles.ktmath.Vector3
-import java.util.Queue
+import java.util.*
 import java.util.concurrent.LinkedBlockingQueue
-import kotlin.collections.HashMap
 
 abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISteamVRBridge {
 	@JvmField
 	@VRServerThread
 	protected val sharedTrackers: MutableList<Tracker> = FastList()
 
+	@JvmField
+	@VRServerThread
+	protected val fingerBones: MutableList<ShareableBone> = FastList()
+
 	@ThreadSafe
 	private val inputQueue: Queue<ProtobufMessage> = LinkedBlockingQueue()
 
@@ -35,6 +45,10 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 	private val remoteTrackersByTrackerId: MutableMap<Int, Tracker> = HashMap()
 	private var hadNewData = false
 
+	private var remoteProtocolVersion: Int = 0
+
+	private val inputs: MutableList<dev.slimevr.inputs.Input> = FastList()
+
 	/**
 	 * Wakes the bridge thread, implementation is platform-specific.
 	 */
@@ -99,15 +113,19 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 
 	@VRServerThread
 	protected fun writeTrackerUpdate(localTracker: Tracker?) {
-		val builder = ProtobufMessages.Position.newBuilder().setTrackerId(
+		val builder = Position.newBuilder().setTrackerId(
 			localTracker!!.id,
 		)
+
+		// localTracker position
 		if (localTracker.hasPosition) {
 			val pos = localTracker.position
 			builder.setX(pos.x)
 			builder.setY(pos.y)
 			builder.setZ(pos.z)
 		}
+
+		// localTracker rotation
 		if (localTracker.hasRotation) {
 			val rot = localTracker.getRotation()
 			builder.setQx(rot.x)
@@ -115,9 +133,66 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 			builder.setQz(rot.z)
 			builder.setQw(rot.w)
 		}
+
+		// localTracker's associated fingers' rotations
+		if (remoteProtocolVersion >= 1) {
+			val trackerIsLeftHand = localTracker.trackerPosition == TrackerPosition.LEFT_HAND
+			val trackerIsRightHand = localTracker.trackerPosition == TrackerPosition.RIGHT_HAND
+			if (trackerIsLeftHand || trackerIsRightHand) {
+				for (fingerBone in fingerBones) {
+					// Only send finger data if the finger bone matches the hand tracker side
+					if ((trackerIsLeftHand && fingerBone.boneType.isLeftFinger()) ||
+						(trackerIsRightHand && fingerBone.boneType.isRightFinger())
+					) {
+						val fingerBuilder = FingerBoneRotation.newBuilder()
+							.setName(boneTypeToFingerBoneName(fingerBone.boneType))
+							.setX(fingerBone.localRotation.x)
+							.setY(fingerBone.localRotation.y)
+							.setZ(fingerBone.localRotation.z)
+							.setW(fingerBone.localRotation.w)
+
+						builder.addFingerBoneRotations(fingerBuilder.build())
+					}
+				}
+			}
+		}
+
+		// Double and triple tap inputs
+		if (remoteProtocolVersion >= 1) {
+			val trackerIsLeftHand = localTracker.trackerPosition == TrackerPosition.LEFT_HAND
+			val trackerIsRightHand = localTracker.trackerPosition == TrackerPosition.RIGHT_HAND
+			if (trackerIsLeftHand || trackerIsRightHand) {
+				val iterator = inputs.iterator()
+				while (iterator.hasNext()) {
+					val input = iterator.next()
+					if ((input.rightHand && trackerIsRightHand) || (!input.rightHand && trackerIsLeftHand)) {
+						val inputBuilder = Input.newBuilder()
+						if (input.type == InputType.DOUBLE_TAP) {
+							inputBuilder.setType(Input.InputType.DOUBLE_TAP)
+						} else if (input.type == InputType.TRIPLE_TAP) {
+							inputBuilder.setType(Input.InputType.TRIPLE_TAP)
+						} else {
+							LogManager.debug("[ProtobufBridge] Unsupported Input, skipping: " + input.type)
+							iterator.remove()
+							continue
+						}
+
+						builder.addInput(inputBuilder.build())
+
+						iterator.remove()
+					}
+				}
+			}
+		}
+
 		sendMessage(ProtobufMessage.newBuilder().setPosition(builder).build())
 	}
 
+	@VRServerThread
+	override fun sendInput(input: dev.slimevr.inputs.Input) {
+		inputs.add(input)
+	}
+
 	@VRServerThread
 	protected open fun writeBatteryUpdate(localTracker: Tracker) {
 		return
@@ -137,6 +212,8 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 			trackerAddedReceived(message.trackerAdded)
 		} else if (message.hasBattery()) {
 			batteryReceived(message.battery)
+		} else if (message.hasVersion()) {
+			versionReceived(message.version)
 		}
 	}
 
@@ -172,6 +249,15 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 		return
 	}
 
+	@VRServerThread
+	protected open fun versionReceived(versionMessage: Version) {
+		remoteProtocolVersion = versionMessage.protocolVersion
+		LogManager.info("[ProtobufBridge] Received driver protocol version: $remoteProtocolVersion")
+		if (remoteProtocolVersion != PROTOCOL_VERSION) {
+			LogManager.warning("[ProtobufBridge] Driver protocol version ($remoteProtocolVersion) doesn't match server protocol version ($PROTOCOL_VERSION)")
+		}
+	}
+
 	@VRServerThread
 	protected abstract fun createNewTracker(trackerAdded: TrackerAdded): Tracker
 
@@ -272,7 +358,50 @@ abstract class ProtobufBridge(@JvmField protected val bridgeName: String) : ISte
 		sendMessage(ProtobufMessage.newBuilder().setTrackerStatus(statusBuilder).build())
 	}
 
+	@VRServerThread
+	override fun addFingerBones(bones: List<ShareableBone>) {
+		fingerBones.addAll(bones)
+	}
+
+	private fun boneTypeToFingerBoneName(boneType: BoneType): FingerBoneRotation.FingerBoneName = when (boneType) {
+		BoneType.LEFT_THUMB_METACARPAL, BoneType.RIGHT_THUMB_METACARPAL -> FingerBoneRotation.FingerBoneName.THUMB_METACARPAL
+
+		BoneType.LEFT_THUMB_PROXIMAL, BoneType.RIGHT_THUMB_PROXIMAL -> FingerBoneRotation.FingerBoneName.THUMB_PROXIMAL
+
+		BoneType.LEFT_THUMB_DISTAL, BoneType.RIGHT_THUMB_DISTAL -> FingerBoneRotation.FingerBoneName.THUMB_DISTAL
+
+		BoneType.LEFT_INDEX_PROXIMAL, BoneType.RIGHT_INDEX_PROXIMAL -> FingerBoneRotation.FingerBoneName.INDEX_PROXIMAL
+
+		BoneType.LEFT_INDEX_INTERMEDIATE, BoneType.RIGHT_INDEX_INTERMEDIATE -> FingerBoneRotation.FingerBoneName.INDEX_INTERMEDIATE
+
+		BoneType.LEFT_INDEX_DISTAL, BoneType.RIGHT_INDEX_DISTAL -> FingerBoneRotation.FingerBoneName.INDEX_DISTAL
+
+		BoneType.LEFT_MIDDLE_PROXIMAL, BoneType.RIGHT_MIDDLE_PROXIMAL -> FingerBoneRotation.FingerBoneName.MIDDLE_PROXIMAL
+
+		BoneType.LEFT_MIDDLE_INTERMEDIATE, BoneType.RIGHT_MIDDLE_INTERMEDIATE -> FingerBoneRotation.FingerBoneName.MIDDLE_INTERMEDIATE
+
+		BoneType.LEFT_MIDDLE_DISTAL, BoneType.RIGHT_MIDDLE_DISTAL -> FingerBoneRotation.FingerBoneName.MIDDLE_DISTAL
+
+		BoneType.LEFT_RING_PROXIMAL, BoneType.RIGHT_RING_PROXIMAL -> FingerBoneRotation.FingerBoneName.RING_PROXIMAL
+
+		BoneType.LEFT_RING_INTERMEDIATE, BoneType.RIGHT_RING_INTERMEDIATE -> FingerBoneRotation.FingerBoneName.RING_INTERMEDIATE
+
+		BoneType.LEFT_RING_DISTAL, BoneType.RIGHT_RING_DISTAL -> FingerBoneRotation.FingerBoneName.RING_DISTAL
+
+		BoneType.LEFT_LITTLE_PROXIMAL, BoneType.RIGHT_LITTLE_PROXIMAL -> FingerBoneRotation.FingerBoneName.LITTLE_PROXIMAL
+
+		BoneType.LEFT_LITTLE_INTERMEDIATE, BoneType.RIGHT_LITTLE_INTERMEDIATE -> FingerBoneRotation.FingerBoneName.LITTLE_INTERMEDIATE
+
+		BoneType.LEFT_LITTLE_DISTAL, BoneType.RIGHT_LITTLE_DISTAL -> FingerBoneRotation.FingerBoneName.LITTLE_DISTAL
+
+		else -> {
+			LogManager.severe("[ProtobufBridge] Tried to get FingerBoneName from invalid BoneType " + boneType.name)
+			FingerBoneRotation.FingerBoneName.UNRECOGNIZED
+		}
+	}
+
 	companion object {
 		private const val resetSourceNamePrefix = "ProtobufBridge"
+		private const val PROTOCOL_VERSION = 1
 	}
 }

server/desktop/src/main/java/dev/slimevr/desktop/platform/linux/UnixSocketRpcBridge.java

@@ -1,12 +1,15 @@
 package dev.slimevr.desktop.platform.linux;
 
 import dev.slimevr.bridge.BridgeThread;
+import dev.slimevr.inputs.Input;
 import dev.slimevr.protocol.GenericConnection;
 import dev.slimevr.protocol.ProtocolAPI;
+import dev.slimevr.tracking.processor.ShareableBone;
 import dev.slimevr.tracking.trackers.Tracker;
 import dev.slimevr.util.ann.VRServerThread;
 import dev.slimevr.VRServer;
 import io.eiren.util.logging.LogManager;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
 import java.io.IOException;
@@ -74,6 +77,10 @@ public class UnixSocketRpcBridge implements dev.slimevr.bridge.Bridge,
 	public void removeSharedTracker(Tracker tracker) {
 	}
 
+	@Override
+	public void addFingerBones(@NotNull List<ShareableBone> bones) {
+	}
+
 	@Override
 	@VRServerThread
 	public void startBridge() {
@@ -136,4 +143,9 @@ public class UnixSocketRpcBridge implements dev.slimevr.bridge.Bridge,
 			.map(key -> (GenericConnection) key.attachment())
 			.filter(conn -> conn != null);
 	}
+
+	@Override
+	public void sendInput(@NotNull Input input) {
+
+	}
 }