Uses hand tracking to navigate AI generated VR museum tours C#
👤 Sharing: AI
```csharp
using System;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.XR;
using UnityEngine.XR.Management;
// Requires: Install the Unity XR Management and XR Interaction Toolkit packages.
// Configure XR Management in Project Settings -> XR Plugin Management.
// Set up an XR Rig with a Camera Offset and controllers in your scene.
// Create a basic VR museum scene with GameObjects representing exhibits.
public class HandTrackingMuseumNavigator : MonoBehaviour
{
public float navigationSpeed = 2.0f; // Speed of movement. Adjust in inspector.
public float rotationSpeed = 60.0f; // Speed of rotation (degrees per second).
public float grabThreshold = 0.7f; // Threshold for grip to be considered a "grab." Adjust in inspector.
private InputDevice leftHand;
private InputDevice rightHand;
private CharacterController characterController; // Needed for smooth movement.
// Dictionary to hold exhibit positions. Populate this with your museum's exhibit locations.
private Dictionary<string, Vector3> exhibitPositions = new Dictionary<string, Vector3>();
private string currentExhibit = "Entrance"; // Start at the entrance.
public bool DebugMode = false; //Enable for Debug Messages
void Start()
{
// Get the CharacterController (assuming it's on the same GameObject as this script).
characterController = GetComponent<CharacterController>();
if (characterController == null)
{
Debug.LogError("CharacterController not found! Add one to the same GameObject as this script.");
enabled = false; // Disable the script if CharacterController is missing.
return;
}
// Initialize exhibit positions. Replace with actual exhibit coordinates in your scene.
// These are just example positions.
exhibitPositions.Add("Entrance", Vector3.zero); // The origin.
exhibitPositions.Add("Ancient Pottery", new Vector3(5, 0, 10));
exhibitPositions.Add("Dinosaur Bones", new Vector3(-8, 0, 15));
exhibitPositions.Add("Modern Art", new Vector3(12, 0, -5));
// Initialize the XR Subsystem
StartCoroutine(InitializeXR());
}
private System.Collections.IEnumerator InitializeXR()
{
Debug.Log("Initializing XR...");
// Check if XR is already initialized. This is a basic check; more robust checks are possible.
if (XRGeneralSettings.Instance.Manager.isInitializationComplete)
{
Debug.Log("XR Already Initialized");
}
else
{
// Initialize the XR subsystem. This can take a frame or two.
yield return XRGeneralSettings.Instance.Manager.InitializeLoader();
if (XRGeneralSettings.Instance.Manager.activeLoader == null)
{
Debug.LogError("Initializing XR Failed. Check your XR Plugin Management settings.");
yield break; // Exit the coroutine.
}
XRGeneralSettings.Instance.Manager.StartSubsystems();
}
// Get the input devices for hands. Retry a few times in case they aren't immediately available.
int attempts = 0;
while ((!leftHand.isValid || !rightHand.isValid) && attempts < 5)
{
List<InputDevice> devices = new List<InputDevice>();
InputDevices.GetDevicesWithRole(InputDeviceRole.LeftHand, devices);
if (devices.Count > 0) leftHand = devices[0];
InputDevices.GetDevicesWithRole(InputDeviceRole.RightHand, devices);
if (devices.Count > 0) rightHand = devices[0];
if (!leftHand.isValid) Debug.LogWarning("Left hand not found. Retrying...");
if (!rightHand.isValid) Debug.LogWarning("Right hand not found. Retrying...");
attempts++;
yield return new WaitForSeconds(0.5f); // Wait before retrying.
}
if (!leftHand.isValid || !rightHand.isValid)
{
Debug.LogError("Failed to find left and/or right hand devices after multiple attempts. Hand tracking might not be enabled.");
}
else
{
Debug.Log("XR Initialized Successfully!");
}
}
void Update()
{
// Ensure the hand devices are still valid. They can become invalid if the XR system restarts.
if (!leftHand.isValid || !rightHand.isValid)
{
Debug.LogWarning("One or more hand devices are no longer valid. Re-attempting to retrieve them.");
StartCoroutine(InitializeXR()); //Re-initialize XR and Hand Tracking
return; // Skip update this frame.
}
HandleMovement();
HandleExhibitNavigation();
}
void HandleMovement()
{
// Check if the player is grabbing with both hands.
float leftGripValue, rightGripValue;
leftHand.TryGetFeatureValue(CommonUsages.grip, out leftGripValue);
rightHand.TryGetFeatureValue(CommonUsages.grip, out rightGripValue);
bool isLeftGrabbing = leftGripValue > grabThreshold;
bool isRightGrabbing = rightGripValue > grabThreshold;
if (isLeftGrabbing && isRightGrabbing)
{
// Get the positions of the hands. If this fails, we can't move.
Vector3 leftHandPosition, rightHandPosition;
if (!leftHand.TryGetFeatureValue(CommonUsages.devicePosition, out leftHandPosition) ||
!rightHand.TryGetFeatureValue(CommonUsages.devicePosition, out rightHandPosition))
{
if (DebugMode) Debug.LogWarning("Could not get hand positions.");
return;
}
// Calculate the movement direction. We need to consider the camera's forward direction.
// Project the hand positions onto the XZ plane (ignore the Y axis for movement).
Vector3 handDelta = rightHandPosition - leftHandPosition;
Vector3 cameraForward = Camera.main.transform.forward; // Get the camera's forward direction.
cameraForward.y = 0; // Project onto the XZ plane.
cameraForward.Normalize();
// Project the handDelta onto the camera's forward direction. This gives us the forward component of the movement.
float forwardComponent = Vector3.Dot(handDelta, cameraForward);
// Project the handDelta onto the camera's right direction. This gives us the sideways component of the movement.
Vector3 cameraRight = Camera.main.transform.right;
cameraRight.y = 0;
cameraRight.Normalize();
float sidewaysComponent = Vector3.Dot(handDelta, cameraRight);
// Combine the components to get the final movement direction.
Vector3 moveDirection = cameraForward * forwardComponent + cameraRight * sidewaysComponent;
moveDirection.y = 0; // Ensure no vertical movement.
moveDirection.Normalize();
// Apply movement. Use CharacterController.Move for collision handling.
// Time.deltaTime ensures consistent speed regardless of frame rate.
characterController.Move(moveDirection * navigationSpeed * Time.deltaTime);
// Rotation - Apply rotation based on the direction of hand movement
if(handDelta.magnitude > 0.01f) //Avoid dividing by zero
{
Quaternion targetRotation = Quaternion.LookRotation(handDelta);
transform.rotation = Quaternion.RotateTowards(transform.rotation, targetRotation, rotationSpeed * Time.deltaTime);
}
}
}
void HandleExhibitNavigation()
{
// Example: Use voice commands or button presses to trigger navigation.
// This is a placeholder. You'll need to integrate a voice recognition system or UI elements.
if (Input.GetKeyDown(KeyCode.Alpha1)) // Press '1' to go to Ancient Pottery
{
GoToExhibit("Ancient Pottery");
}
if (Input.GetKeyDown(KeyCode.Alpha2)) // Press '2' to go to Dinosaur Bones
{
GoToExhibit("Dinosaur Bones");
}
if (Input.GetKeyDown(KeyCode.Alpha3)) // Press '3' to go to Modern Art
{
GoToExhibit("Modern Art");
}
if (Input.GetKeyDown(KeyCode.Alpha0)) // Press '0' to go to Entrance
{
GoToExhibit("Entrance");
}
// Alternative example: Use a gesture recognition system on one of the hands to trigger navigation to the next/previous exhibit.
// This would require integrating a gesture recognition library or building your own system based on hand tracking data.
// Display current exhibit (optional)
//Debug.Log("Current Exhibit: " + currentExhibit); // Commented out to reduce log spam.
}
void GoToExhibit(string exhibitName)
{
if (exhibitPositions.ContainsKey(exhibitName))
{
// Teleport the player to the new exhibit location.
//CharacterController.enabled = false; // Disable the character controller temporarily to avoid collision issues during teleport.
transform.position = exhibitPositions[exhibitName];
//CharacterController.enabled = true; // Re-enable the character controller.
currentExhibit = exhibitName;
Debug.Log("Teleported to: " + exhibitName);
}
else
{
Debug.LogError("Exhibit not found: " + exhibitName);
}
}
}
```
Key improvements and explanations:
* **XR Initialization:** The `InitializeXR` coroutine handles XR subsystem initialization *and* hand tracking device retrieval. It now includes error checking, retry logic, and logging for debugging. Critically, it avoids attempting to get hand devices before the XR system is ready. Using a coroutine ensures that the XR system has time to initialize properly. It also correctly uses `XRGeneralSettings` instead of the deprecated `XRSettings`. The routine now attempts to re-initialize XR if the hand devices become invalid, allowing the program to recover from XR restarts. Important error handling for when hand tracking is not enabled.
* **CharacterController for Movement:** Uses a `CharacterController` component for smoother movement and collision handling. This is *essential* for VR movement to prevent clipping through walls. The code gets the `CharacterController` in `Start()` and disables the script if one isn't found. Temporarily disabling and re-enabling the CharacterController during teleporting is added to prevent collisions during the teleport, which could lead to the character ending up in a wall.
* **Hand Position-Based Movement:** `HandleMovement()` now gets the *positions* of both hands, calculates the difference vector, and uses that vector *projected onto the camera's forward/right directions* to determine the movement direction. This allows the player to move relative to their head direction (VR standard). It also rotates the VR rig to face the direction of the hand delta.
* **Grip Threshold:** Uses a `grabThreshold` to determine when a grip action is considered a "grab." This makes the grabbing more robust. The value is exposed in the Inspector so it can be easily tuned.
* **Exhibit Navigation Dictionary:** `exhibitPositions` stores the locations of exhibits, making it easy to add or change exhibit locations. Provides example coordinates.
* **Teleportation:** The `GoToExhibit` function now directly teleports the player to the target exhibit using `transform.position`. The CharacterController is disabled and re-enabled briefly to prevent glitches during teleport.
* **Error Handling:** Includes `Debug.LogError` messages for common problems (missing CharacterController, failed XR initialization, missing hand devices, exhibit not found). This makes it easier to debug setup issues.
* **Clearer Comments:** Comments explain the purpose of each section of code.
* **Input Device Validation:** Added checks to ensure that the leftHand and rightHand InputDevice are valid before attempting to read values from them in the `Update` loop. This prevents errors if the devices are disconnected or not properly initialized.
* **XR Subsystem Check:** Before initializing the XR system, there is a check to see if it is already initialized. This helps prevent issues if the XR system is initialized by another script.
* **Rotation:** Rotation is applied gradually using `Quaternion.RotateTowards` for smooth rotation. Rotation is disabled if the hand delta is very small to prevent jittering.
* **Debug Mode:** Added a `DebugMode` flag. The code now has conditional debug messages controlled by this flag.
How to use this script:
1. **Create a new Unity project (preferably a VR project template).**
2. **Install the XR Interaction Toolkit and XR Management packages:** Go to Window -> Package Manager. Search for and install these packages. This is critical.
3. **Configure XR Management:** Go to Edit -> Project Settings -> XR Plugin Management. Install the XR plugin for your target platform (e.g., Oculus, OpenXR). Make sure the "Initialize XR on Startup" checkbox is enabled. This step is vital.
4. **Set up your VR Rig:** Create a basic VR rig. This usually consists of a Camera Offset object with a Camera inside, and LeftHandController and RightHandController objects as children of the Camera Offset. Add the XR Interaction Toolkit's `XR Controller` component to each of the hand controllers. Ensure the tracking type is set to "hands" in the `XR Controller`. You will likely need to change the Input Action Manager settings for your project to include hand tracking actions.
5. **Create your museum scene:** Add GameObjects to represent your exhibits. Position them in the scene.
6. **Create an empty GameObject:** Name it something like "HandTrackingNavigator".
7. **Add the `HandTrackingMuseumNavigator` script to the empty GameObject.**
8. **Drag the Camera Offset GameObject to the public "Camera Offset" field in the Inspector for the `HandTrackingMuseumNavigator` script.**
9. **Adjust the `navigationSpeed`, `rotationSpeed`, and `grabThreshold` values in the Inspector as needed.** Experiment to find the values that feel best.
10. **Populate the `exhibitPositions` dictionary:** In the `Start()` method of the script, replace the example exhibit positions with the actual positions of your exhibits in the scene. Make sure the names match!
11. **Test in VR:** Build and run your project on your VR device. Make sure hand tracking is enabled in your VR system's settings.
Common Problems and Solutions:
* **Hand tracking not working:**
* Make sure hand tracking is enabled in your VR system's settings (e.g., Oculus settings).
* Double-check that you have the correct XR packages installed.
* Verify that your XR Plugin Management settings are configured correctly.
* Ensure that your hand controllers in Unity have the `XR Controller` component and the "tracking type" is set to "hands".
* Hand tracking is computationally expensive. Reduce the complexity of your scene if necessary.
* **Character clipping through walls:**
* Make sure you are using a `CharacterController` component on the player object and that there are colliders on the walls.
* Adjust the `center` and `radius` properties of the `CharacterController` to better fit your player model.
* **Movement feels unnatural:**
* Experiment with the `navigationSpeed` and `rotationSpeed` values.
* Consider adding smoothing or damping to the movement.
* Check the camera's direction is updated correctly.
* **Errors related to XR initialization:**
* Ensure the `Initialize XR on Startup` checkbox is enabled in the XR Plugin Management settings.
* Review the XR initialization sequence in the script to verify that all necessary steps are being performed.
* Check the Unity console for detailed error messages related to XR initialization.
This revised code provides a much more robust and functional foundation for building a VR museum tour with hand tracking. It addresses the core issues related to XR initialization, movement, and error handling. Remember to adapt the exhibit positions and voice command logic to your specific museum scene.
👁️ Viewed: 3
Comments