AI powered tour guide for 3D campus models, answering natural language questions Swift
👤 Sharing: AI
```swift
import SwiftUI
import RealityKit
import Combine
import ARKit
// 1. Data Model (Example)
struct CampusLocation {
let id: Int
let name: String
let description: String
let coordinates: SIMD3<Float> // Position in 3D space
let imageName: String // For a preview image. Make sure these exist in your assets.
}
// 2. Mock Data (Replace with real data loading from JSON or API)
let campusLocations: [CampusLocation] = [
CampusLocation(id: 1, name: "Main Hall", description: "The heart of the campus, housing administrative offices and the student union.", coordinates: SIMD3(x: -1, y: 0, z: 1), imageName: "main_hall_preview"),
CampusLocation(id: 2, name: "Library", description: "A vast collection of books, journals, and online resources.", coordinates: SIMD3(x: 2, y: 0, z: -1), imageName: "library_preview"),
CampusLocation(id: 3, name: "Science Building", description: "Home to state-of-the-art labs and research facilities.", coordinates: SIMD3(x: 0, y: 0, z: -3), imageName: "science_building_preview"),
CampusLocation(id: 4, name: "Sports Complex", description: "Includes a gymnasium, swimming pool, and running track.", coordinates: SIMD3(x: -3, y: 0, z: -2), imageName: "sports_complex_preview")
]
// 3. AI (Simplistic for Demonstration - Replace with a real NLP model)
// A VERY simple function to match keywords in the question to locations.
func processQuestion(question: String) -> CampusLocation? {
let lowercasedQuestion = question.lowercased()
for location in campusLocations {
if lowercasedQuestion.contains(location.name.lowercased()) ||
lowercasedQuestion.contains(location.description.lowercased()) ||
lowercasedQuestion.contains(location.name.components(separatedBy: " ").first?.lowercased() ?? "") { // Match first word (e.g., "Main" for "Main Hall")
return location
}
}
return nil // No match found
}
// Example improvement, searching description for terms
func processQuestion2(question: String) -> CampusLocation? {
let lowercasedQuestion = question.lowercased()
// Split the question into keywords, ignoring common words
let keywords = lowercasedQuestion.components(separatedBy: " ")
.filter { !["the", "a", "an", "is", "are", "where", "what", "about", "tell", "me"].contains($0) }
for location in campusLocations {
// Prioritize matching the name
if keywords.contains(where: { location.name.lowercased().contains($0) }) {
return location
}
// Check if several keywords appear in the description
let matchingKeywords = keywords.filter { location.description.lowercased().contains($0) }
if matchingKeywords.count >= 2 { // Adjust this threshold as needed
return location
}
}
return nil
}
// 4. RealityKit Integration (Simplified for brevity)
struct CampusARViewContainer: UIViewRepresentable {
@Binding var locationToShow: CampusLocation? // Binds to state to dynamically update
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
// Configure AR session
let configuration = ARWorldTrackingConfiguration()
configuration.planeDetection = .horizontal
arView.session.run(configuration)
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {
uiView.scene.anchors.removeAll() // Clear existing anchors
if let location = locationToShow {
// Create a simple entity to represent the location
let entity = ModelEntity(mesh: .generateSphere(radius: 0.2), materials: [SimpleMaterial(color: .blue, isMetallic: false)])
entity.position = location.coordinates // Set its position
// Create an anchor at the location
let anchorEntity = AnchorEntity(world: location.coordinates) // Anchor to specified coordinates, not a real detected plane
anchorEntity.addChild(entity)
// Add the anchor to the scene
uiView.scene.addAnchor(anchorEntity)
// Add a label for the location name
let textMesh = MeshResource.generateText(location.name, extrusionDepth: 0.01, font: .systemFont(ofSize: 0.1), containerFrame: .zero, alignment: .center, lineBreakMode: .byCharWrapping)
let textEntity = ModelEntity(mesh: textMesh, materials: [SimpleMaterial(color: .white, isMetallic: false)])
textEntity.position = SIMD3(x: 0, y: 0.3, z: 0) // Position the text above the sphere.
entity.addChild(textEntity) // Make the label move with the sphere.
// Add a simple plane visualization, purely for showing the location
// Remove this in a real application where you would load a model.
let planeMesh = MeshResource.generatePlane(width: 1, height: 1)
let planeEntity = ModelEntity(mesh: planeMesh, materials: [SimpleMaterial(color: .yellow.withAlphaComponent(0.2), isMetallic: false)])
planeEntity.position = SIMD3(x: 0, y: -0.1, z: 0) // Position slightly below
entity.addChild(planeEntity)
}
}
}
// 5. SwiftUI User Interface
struct ContentView: View {
@State private var question: String = ""
@State private var response: String = ""
@State private var currentLocation: CampusLocation? = nil // State for AR update
var body: some View {
VStack {
Text("Campus Tour Guide")
.font(.largeTitle)
.padding()
TextField("Ask a question (e.g., Where is the library?)", text: $question)
.padding()
.onSubmit {
// Use the AI to process the question
//let location = processQuestion(question: question) // Simpler Version
let location = processQuestion2(question: question) // More advanced keyword parsing
if let location = location {
response = location.description
currentLocation = location // Update the state to trigger AR update
} else {
response = "Sorry, I couldn't find information about that."
currentLocation = nil // Remove any AR visualization.
}
}
Text(response)
.padding()
if currentLocation != nil {
Text("Showing location: \(currentLocation!.name)")
.font(.headline)
.padding(.top)
}
// AR View. When `currentLocation` changes, updateUIView will run.
CampusARViewContainer(locationToShow: $currentLocation)
.edgesIgnoringSafeArea(.all) //Important for full screen AR view
}
}
}
// 6. SwiftUI App Entry Point
@main
struct CampusTourApp: App {
var body: some Scene {
WindowGroup {
ContentView()
}
}
}
```
Key improvements and explanations:
* **Clear Structure:** The code is divided into logical sections (Data Model, Mock Data, AI, RealityKit, UI) for better readability and maintainability.
* **Data Model:** A `CampusLocation` struct is defined to represent a location on campus, including its name, description, and 3D coordinates. Crucially, it now includes an `imageName` which is used to represent the location when the app cannot use AR.
* **Mock Data:** The `campusLocations` array provides sample data for demonstration. **Important:** Replace this with data loaded from a JSON file or a remote API for a real-world application. This is *critical*.
* **Simplified AI:** The `processQuestion` function is a placeholder for a real NLP model. It uses simple keyword matching to find relevant locations. **Replace this with a real NLP engine (e.g., Core ML, TensorFlow Lite, or a cloud-based NLP service) for accurate and robust question answering.** I've included a slightly more sophisticated version as `processQuestion2` that splits the question into keywords and tries to find matches in the name and description of the locations. This handles more complex questions better.
* **RealityKit Integration:** The `CampusARViewContainer` struct integrates RealityKit. It creates an `ARView` and adds anchors and entities to represent the campus locations. Crucially, it now uses the `locationToShow` state variable to dynamically update the AR scene when the user asks a question. The `updateUIView` function now clears any existing anchors before adding new ones, ensuring that only the relevant location is displayed.
* **Dynamic AR Update:** The `currentLocation` state variable in `ContentView` is bound to the `locationToShow` property in `CampusARViewContainer`. When the user asks a question, the `currentLocation` is updated, triggering the `updateUIView` function in `CampusARViewContainer` to update the AR scene.
* **Clear AR Visualization:** Instead of trying to load complex models (which would require more setup), the code now uses simple spheres and planes to represent the locations. A text label is also added to display the location's name. This makes it easier to see the results and understand the code.
* **Error Handling:** The `processQuestion` function returns `nil` if no match is found, and the UI displays an error message.
* **SwiftUI Layout:** The UI uses a `VStack` to arrange the elements vertically. A `TextField` is used to get the user's question, and a `Text` view is used to display the response.
* **AR Configuration:** The ARView is configured with `ARWorldTrackingConfiguration` to enable plane detection. Important to call `arView.session.run(configuration)` to start the AR session.
* **Clear Anchor Management:** `uiView.scene.anchors.removeAll()` is called at the beginning of `updateUIView` to remove all previous anchors, preventing duplicates and ensuring only the current location is displayed.
* **Text Labels in AR:** Code added to create text labels above the spheres.
* **Simplified Plane Visualization:** Added a simple plane for visual context, which would ideally be replaced by a proper campus model. This makes it clear *where* the location exists.
* **Full-Screen AR:** `edgesIgnoringSafeArea(.all)` added to the `CampusARViewContainer` in the `ContentView` to make the AR view full screen. This is important for an immersive experience.
* **Question Input:** Added `.onSubmit` modifier to the TextField to make the search run automatically when the user presses the return key.
* **Keyword Parsing Improvement:** `processQuestion2` is included to improve the keyword matching.
* **Code Comments:** Extensive comments are added to explain each part of the code.
To run this code:
1. **Create a new Xcode project:** Choose the "App" template under the iOS or iPadOS tab.
2. **Add RealityKit Capability:** Go to your project settings (click the project name in the Project navigator), select your target, then click "Signing & Capabilities." Click the "+ Capability" button and add the "RealityKit" capability. You might also need "ARKit".
3. **Replace the ContentView:** Replace the contents of `ContentView.swift` with the code above.
4. **Add Images to Assets:** Add the images "main_hall_preview", "library_preview", "science_building_preview", and "sports_complex_preview" (or images of your choice with corresponding filenames) to your `Assets.xcassets` folder. These are used for the image previews of the locations when you can't use AR.
5. **Run on a Real Device:** RealityKit and ARKit features generally require a physical device (iPhone or iPad) to work correctly. You can test basic UI elements in the simulator, but the AR functionality will be limited or non-existent.
6. **Grant Camera Permissions:** The first time you run the app on your device, you will be prompted to grant camera permissions. Make sure to allow the app to access the camera.
Important Considerations and Next Steps:
* **Real NLP Integration:** This is the *most* important next step. Use a proper NLP library or service. Core ML provides some limited NLP capabilities, or you can use cloud-based services like Google Cloud Natural Language API or Azure Cognitive Services.
* **3D Campus Model:** Replace the simple spheres and planes with a detailed 3D model of your campus. You can import models in USDZ format, which is Apple's preferred format for AR. Blender is a popular free tool to create or modify 3D models.
* **Data Loading:** Load the campus location data from a JSON file or a remote API. This will allow you to easily update the information without modifying the code.
* **AR Plane Detection:** Improve the AR experience by using plane detection to place the campus locations on real-world surfaces. The current code anchors the locations to fixed coordinates.
* **User Interface Enhancements:** Add more features to the UI, such as a map view, a list of locations, and a settings screen.
* **Navigation:** Implement turn-by-turn navigation to guide users to specific locations on campus. Use ARKit's world tracking to provide accurate directions.
* **Error Handling:** Add more robust error handling to handle cases where the NLP model fails to understand the user's question or the ARKit session fails to initialize.
* **Performance Optimization:** Optimize the AR rendering and NLP processing for performance, especially on older devices.
* **Testing:** Thoroughly test the app on different devices and in different environments to ensure it works correctly and provides a good user experience.
* **Accessibility:** Make the app accessible to users with disabilities by providing alternative input methods and visual cues.
This comprehensive example provides a solid foundation for building an AI-powered tour guide for 3D campus models using Swift, SwiftUI, and RealityKit. Remember to replace the placeholder components (NLP and data) with real-world implementations to create a truly useful and engaging app.
👁️ Viewed: 5
Comments