RealityKit & AR

What is RealityKit?

RealityKit is Apple's high-performance 3D rendering framework for augmented reality. It uses an Entity-Component-System (ECS) architecture where entities represent objects, components add data, and systems process updates.

Setting Up an AR View

In SwiftUI, wrap ARView in UIViewRepresentable to display AR content.

import SwiftUI
import RealityKit
import ARKit

struct ARViewContainer: UIViewRepresentable {
    func makeUIView(context: Context) -> ARView {
        let arView = ARView(frame: .zero)
        let config = ARWorldTrackingConfiguration()
        config.planeDetection = [.horizontal, .vertical]
        config.environmentTexturing = .automatic
        arView.session.run(config)
        return arView
    }
    func updateUIView(_ uiView: ARView, context: Context) { }
}

Understanding Entities

AnchorEntity ties content to a real-world position. ModelEntity represents 3D objects with meshes and materials.

func createScene(in arView: ARView) {
    let anchor = AnchorEntity(world: [0, 0, 0])
    let mesh = MeshResource.generateBox(size: 0.1)
    let material = SimpleMaterial(color: .blue, roughness: 0.5, isMetallic: false)
    let boxEntity = ModelEntity(mesh: mesh, materials: [material])
    boxEntity.generateCollisionShapes(recursive: true)
    anchor.addChild(boxEntity)
    arView.scene.addAnchor(anchor)
}

Materials and Lighting

RealityKit supports PBR materials. Use SimpleMaterial for basic colors or PhysicallyBasedMaterial for advanced effects with roughness, metallic, and emissive properties.

let simpleMat = SimpleMaterial(color: .red, roughness: 0.8, isMetallic: false)

var pbrMat = PhysicallyBasedMaterial()
pbrMat.baseColor = .init(tint: .blue)
pbrMat.roughness = .init(floatLiteral: 0.3)
pbrMat.metallic = .init(floatLiteral: 0.8)

Plane Detection and Raycasting

Raycasting finds real-world surfaces and places objects on them.

func handleTap(at location: CGPoint, in arView: ARView) {
    let results = arView.raycast(from: location, allowing: .estimatedPlane, alignment: .horizontal)
    if let first = results.first {
        let point = first.worldTransform.columns.3.xyz
        let anchor = AnchorEntity(world: point)
        let mesh = MeshResource.generateSphere(radius: 0.05)
        let material = SimpleMaterial(color: .green, roughness: 0.3, isMetallic: true)
        let sphere = ModelEntity(mesh: mesh, materials: [material])
        anchor.addChild(sphere)
        arView.scene.addAnchor(anchor)
    }
}

Adding Gestures to AR Entities

Use RealityKit's gesture system to make 3D objects interactive with rotation, scale, and translation.

func enableGestures(for entity: ModelEntity, in arView: ARView) {
    entity.generateCollisionShapes(recursive: true)
    arView.installGestures([.rotation, .scale, .translation], for: entity)
}

Loading Reality Composer Scenes

Reality Composer provides a visual editor for creating AR scenes. Load these scenes at runtime using Entity.load(named:).

func loadRealityComposerScene(in arView: ARView) {
    guard let scene = try? Entity.load(named: "MyScene") else {
        print("Failed to load scene")
        return
    }
    let anchor = AnchorEntity(plane: .horizontal)
    anchor.addChild(scene)
    arView.scene.addAnchor(anchor)
}

Animating Entities

RealityKit provides built-in animations for entity transformations using move(to:relativeTo:duration:).

sphere.move(
    to: Transform(
        scale: SIMD3(repeating: 2.0),
        rotation: simd_quatf(angle: .pi, axis: [0, 1, 0]),
        translation: [0, 0.2, 0]
    ),
    relativeTo: sphere.parent,
    duration: 1.0,
    timingFunction: .easeInOut
)

Building an Object Placement UI

Combine SwiftUI with RealityKit to create a complete AR placement experience with a model selector.

struct ARPlacementView: View {
    @State private var selectedModel = "Chair"
    var body: some View {
        ZStack(alignment: .bottom) {
            ARViewContainer(selectedModel: $selectedModel)
                .ignoresSafeArea()
            HStack(spacing: 20) {
                ForEach(["Chair", "Table", "Lamp"], id: \.self) { model in
                    Button(model) { selectedModel = model }
                        .padding()
                        .background(selectedModel == model ? Color.blue : Color.gray)
                        .foregroundColor(.white)
                        .cornerRadius(10)
                }
            }.padding()
        }
    }
}

Distance Measurement

Use raycasting to measure distances between two points in AR space.

struct ARMeasurementView: View {
    @State private var firstPoint: simd_float3?
    @State private var secondPoint: simd_float3?
    @State private var distance: Float?
    var body: some View {
        VStack {
            ARViewContainer(firstPoint: $firstPoint, secondPoint: $secondPoint, distance: $distance)
                .ignoresSafeArea()
            if let dist = distance {
                Text("Distance: \(String(format: "%.2f", dist))m")
                    .font(.title).padding()
                    .background(.ultraThinMaterial).cornerRadius(12)
            }
        }
    }
}

Occlusion and Shadows

Make virtual objects appear more realistic by enabling occlusion so virtual objects behind real objects are hidden.

func enableOcclusion(in arView: ARView) {
    let config = ARWorldTrackingConfiguration()
    config.frameSemantics.insert(.personSegmentationWithDepth)
    arView.environment.background = .cameraFeed(feed: .passthrough)
}