ARKit PlaneDetectionProvider: adding collisions and physics
Converting anchor geometry into collision shapes.
Overview
Let’s build on what we learned in visualize detected planes. In the previous example, we generated some meshes based on data we received from the anchors. We can use a similar process to generate ShapeResources that we can pass to Collision and Physics components.
Reminder: Apple has an example called Placing content on detected planes. That project shows this process in a bit more detail, but the logic is spread across several files, extensions, etc.
So, we have anchors and we need to convert them to ShapeResource. Make sure to read the first example if you want a walkthrough of setting up PlaneDetectionProvider.
private func createCollisionShape(anchor: PlaneAnchor) async -> ShapeResource? {
// Generate a collision shape for the plane
var shape: ShapeResource? = nil
do {
// Convert vertices to SIMD3<Float>
let vertices = anchor.geometry.meshVertices
var vertexArray: [SIMD3<Float>] = []
for i in 0..<vertices.count {
let vertex = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * i).assumingMemoryBound(to: (Float, Float, Float).self).pointee
vertexArray.append(SIMD3<Float>(vertex.0, vertex.1, vertex.2))
}
// Convert faces to UInt16
let faces = anchor.geometry.meshFaces
var faceArray: [UInt16] = []
let totalFaces = faces.count * faces.primitive.indexCount
for i in 0..<totalFaces {
let face = faces.buffer.contents().advanced(by: i * MemoryLayout<Int32>.size).assumingMemoryBound(to: Int32.self).pointee
faceArray.append(UInt16(face))
}
shape = try await ShapeResource.generateStaticMesh(positions: vertexArray, faceIndices: faceArray)
return shape
} catch {
print("Failed to create a shape for collision \(error).")
}
return nil
}This is very similar to createMeshResource from the other example, and we could even abstract some of the shared logic into helpers. For now, let’s consider the changes.
ShapeResource.generateStaticMeshis an asynchronous process- We convert faces to an array of UInt16 instead of UInt32. Presumably, this is an optimization for collisions.
In principle, the concepts are the same. We have vertices and faces that we need to convert to a format that we can use to generate a shape.
Now that we have the ShapeResource, let’s add some collisions and physics.
let shape = await self.createCollisionShape(anchor: anchor)
if let shape = shape {
let collision = CollisionComponent(shapes: [shape], mode: .default)
entity.components.set(collision)
let physicsMaterial = PhysicsMaterialResource.generate(friction: 0, restitution: 0.8)
let physics = PhysicsBodyComponent(shapes: [shape], mass: 0.0, material: physicsMaterial, mode: .static)
entity.components.set(physics)
}To test this out, we can add a sphere to the scene. We’ll make it a dynamic physics body and apply some force to it when we tap it.
// Create the subject
@State var subject : ModelEntity = {
let subject = ModelEntity(
mesh: .generateSphere(radius: 0.1),
materials: [SimpleMaterial(color: .stepRed, isMetallic: false)])
subject.setPosition([1, 1, -1], relativeTo: nil)
let collision = CollisionComponent(shapes: [.generateSphere(radius: 0.1)])
var physics = PhysicsBodyComponent(
shapes: [.generateSphere(radius: 0.1)],
mass: 1.0,
material: .generate(friction: 0, restitution: 1),
mode: .dynamic
)
physics.isAffectedByGravity = false
let input = InputTargetComponent()
subject.components.set([collision, physics, input])
return subject
}()// Lazy way to add some random force when we tap the sphere
.gesture(TapGesture()
.targetedToEntity(subject)
.onEnded { value in
// Add some force when we tap the subject
let force = SIMD3<Float>(
x: Float.random(in: -1...1),
y: Float.random(in: -1...1),
z: Float.random(in: -1...1)
)
var motion = PhysicsMotionComponent()
motion.linearVelocity = force * 3
value.entity.components.set(motion)
})Video Demo
Example Code
This is a bit verbose, but I didn’t want to combine these two concepts yet.
struct Example070: View {
@State var session = ARKitSession()
@State private var planeAnchors: [UUID: Entity] = [:]
@State private var planeColors: [UUID: Color] = [:]
@State var subject : ModelEntity = {
let subject = ModelEntity(
mesh: .generateSphere(radius: 0.1),
materials: [SimpleMaterial(color: .stepRed, isMetallic: false)])
subject.setPosition([1, 1, -1], relativeTo: nil)
let collision = CollisionComponent(shapes: [.generateSphere(radius: 0.1)])
var physics = PhysicsBodyComponent(
shapes: [.generateSphere(radius: 0.1)],
mass: 1.0,
material: .generate(friction: 0, restitution: 1),
mode: .dynamic
)
physics.isAffectedByGravity = false
let input = InputTargetComponent()
subject.components.set([collision, physics, input])
return subject
}()
var body: some View {
RealityView { content in
content.add(subject)
} update: { content in
for (_, entity) in planeAnchors {
if !content.entities.contains(entity) {
content.add(entity)
}
}
}
.gesture(TapGesture()
.targetedToEntity(subject)
.onEnded { value in
// Add some force when we tap the subject
let force = SIMD3<Float>(
x: Float.random(in: -1...1),
y: Float.random(in: -1...1),
z: Float.random(in: -1...1)
)
var motion = PhysicsMotionComponent()
motion.linearVelocity = force * 3
value.entity.components.set(motion)
})
.task {
try! await setupAndRunPlaneDetection()
}
}
func setupAndRunPlaneDetection() async throws {
let planeData = PlaneDetectionProvider(alignments: [.horizontal, .vertical, .slanted])
if PlaneDetectionProvider.isSupported {
do {
try await session.run([planeData])
for await update in planeData.anchorUpdates {
switch update.event {
case .added, .updated:
let anchor = update.anchor
if planeColors[anchor.id] == nil {
planeColors[anchor.id] = generatePastelColor()
}
let planeEntity = createPlaneEntity(for: anchor, color: planeColors[anchor.id]!)
planeAnchors[anchor.id] = planeEntity
case .removed:
let anchor = update.anchor
if let entity = planeAnchors[anchor.id] {
entity.removeFromParent()
planeAnchors.removeValue(forKey: anchor.id)
planeColors.removeValue(forKey: anchor.id)
}
}
}
} catch {
print("ARKit session error \(error)")
}
}
}
private func generatePastelColor() -> Color {
let hue = Double.random(in: 0...1)
let saturation = Double.random(in: 0.2...0.4)
let brightness = Double.random(in: 0.8...1.0)
return Color(hue: hue, saturation: saturation, brightness: brightness)
}
private func createPlaneEntity(for anchor: PlaneAnchor, color: Color) -> Entity {
let entity = Entity()
entity.name = "Plane \(anchor.id)"
entity.setTransformMatrix(anchor.originFromAnchorTransform, relativeTo: nil)
var material = PhysicallyBasedMaterial()
material.baseColor.tint = UIColor(color)
if let meshResource = createMeshResource(anchor: anchor) {
entity.components.set(ModelComponent(mesh: meshResource, materials: [material]))
}
Task {
let shape = await self.createCollisionShape(anchor: anchor)
if let shape = shape {
let collision = CollisionComponent(shapes: [shape], mode: .default)
entity.components.set(collision)
let physicsMaterial = PhysicsMaterialResource.generate(friction: 0, restitution: 0.8)
let physics = PhysicsBodyComponent(shapes: [shape], mass: 0.0, material: physicsMaterial, mode: .static)
entity.components.set(physics)
}
}
return entity
}
private func createMeshResource(anchor: PlaneAnchor) -> MeshResource? {
// Generate a mesh for the plane (for occlusion).
var meshResource: MeshResource? = nil
do {
var contents = MeshResource.Contents()
contents.instances = [MeshResource.Instance(id: "main", model: "model")]
var part = MeshResource.Part(id: "part", materialIndex: 0)
// Convert vertices to SIMD3<Float>
let vertices = anchor.geometry.meshVertices
var vertexArray: [SIMD3<Float>] = []
for i in 0..<vertices.count {
let vertex = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * i).assumingMemoryBound(to: (Float, Float, Float).self).pointee
vertexArray.append(SIMD3<Float>(vertex.0, vertex.1, vertex.2))
}
part.positions = MeshBuffers.Positions(vertexArray)
// Convert faces to UInt32
let faces = anchor.geometry.meshFaces
var faceArray: [UInt32] = []
let totalFaces = faces.count * faces.primitive.indexCount
for i in 0..<totalFaces {
let face = faces.buffer.contents().advanced(by: i * MemoryLayout<Int32>.size).assumingMemoryBound(to: Int32.self).pointee
faceArray.append(UInt32(face))
}
part.triangleIndices = MeshBuffer(faceArray)
contents.models = [MeshResource.Model(id: "model", parts: [part])]
meshResource = try MeshResource.generate(from: contents)
return meshResource
} catch {
print("Failed to create a mesh resource for a plane anchor: \(error).")
}
return nil
}
private func createCollisionShape(anchor: PlaneAnchor) async -> ShapeResource? {
// Generate a collision shape for the plane
var shape: ShapeResource? = nil
do {
// Convert vertices to SIMD3<Float>
let vertices = anchor.geometry.meshVertices
var vertexArray: [SIMD3<Float>] = []
for i in 0..<vertices.count {
let vertex = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * i).assumingMemoryBound(to: (Float, Float, Float).self).pointee
vertexArray.append(SIMD3<Float>(vertex.0, vertex.1, vertex.2))
}
// Convert faces to UInt16
let faces = anchor.geometry.meshFaces
var faceArray: [UInt16] = []
let totalFaces = faces.count * faces.primitive.indexCount
for i in 0..<totalFaces {
let face = faces.buffer.contents().advanced(by: i * MemoryLayout<Int32>.size).assumingMemoryBound(to: Int32.self).pointee
faceArray.append(UInt16(face))
}
shape = try await ShapeResource.generateStaticMesh(positions: vertexArray, faceIndices: faceArray)
return shape
} catch {
print("Failed to create a shape for collision \(error).")
}
return nil
}
}Support our work so we can continue to bring you new examples and articles.
Download the Xcode project with this and many more examples from Step Into Vision.
Some examples are provided as standalone Xcode projects. You can find those here.
Follow Step Into Vision