ARKit PlaneDetectionProvider: classification and alignment

Joseph Simpson Updated on

We can filter anchors based on classification or alignment values.

Overview

This is the third post in a mini-series on PlaneDetectionProvide. Review part one and part two.

We looked at some options to create geometry and collision shapes for plane anchors. There are also two simple ways we can filter anchors based on our use case.

Alignment

ARKit provides the alignment of each anchor: horizontal, vertical, and slanted (new in visionOS 2). When we create a PlaneDetectionProvider, we can specify the alignments we want to track.

let planeData = PlaneDetectionProvider(alignments: [.horizontal, .vertical, .slanted])

We can also read the alignment from each anchor. For example we can select a different debug color for each one.

private func colorForPlaneAlignment(_ alignment: PlaneAnchor.Alignment?) -> UIColor {
    guard let alignment else {
        return .white
    }

    // Only three cases as of visionOS 2.4
    switch alignment {
    case .horizontal:
        return .systemOrange
    case .vertical:
        return .systemPurple
    case .slanted:
        return .systemCyan
    default:
        return .white
    }
}

Classification

ARKit classifies each anchor into a handful of real-world shapes. Examples include: wall, floor, ceiling, etc. Hopefully Apple expands these classifications in future versions of visionOS. We can switch on this value to too. We may want to exclude specific types from our scene, or add different components for each classification. Let’s add different debug colors for walls, floors, and ceilings, with a default color for anything else.

private func colorForPlaneClassification(_ classification: PlaneAnchor.Classification?) -> UIColor {
    guard let classification else {
        return .white
    }

    // Current cases: https://developer.apple.com/documentation/arkit/planeanchor/classification-swift.enum
    // We only care about wall, ceiling, and floor today
    switch classification {
    case .wall:
        return .systemRed
    case .ceiling:
        return .systemBlue
    case .floor:
        return .systemGreen
    default:
        return .white
    }
}

Example Code

You can change the value of useAlighmentColors to switch between these two color sets.

struct Example071: View {
    @State var session = ARKitSession()
    @State private var planeAnchors: [UUID: Entity] = [:]

    /// Debug value: use classification colors when false, alignment colors when true
    private var useAlighmentColors: Bool = false

    var body: some View {
        RealityView { content in

        } update: { content in
            for (_, entity) in planeAnchors {
                if !content.entities.contains(entity) {
                    content.add(entity)
                }
            }
        }

        .task {
            try! await setupAndRunPlaneDetection()
        }
    }

    func setupAndRunPlaneDetection() async throws {
        let planeData = PlaneDetectionProvider(alignments: [.horizontal, .vertical, .slanted])
        if PlaneDetectionProvider.isSupported {
            do {
                try await session.run([planeData])
                for await update in planeData.anchorUpdates {
                    switch update.event {
                    case .added, .updated:
                        let anchor = update.anchor

                        let planeEntity = createPlaneEntity(for: anchor)
                        planeAnchors[anchor.id] = planeEntity
                    case .removed:
                        let anchor = update.anchor
                        if let entity = planeAnchors[anchor.id] {
                            entity.removeFromParent()
                            planeAnchors.removeValue(forKey: anchor.id)
                        }
                    }
                }
            } catch {
                print("ARKit session error \(error)")
            }
        }
    }

    private func createPlaneEntity(for anchor: PlaneAnchor) -> Entity {
        let entity = Entity()
        entity.name = "Plane \(anchor.id)"
        entity.setTransformMatrix(anchor.originFromAnchorTransform, relativeTo: nil)

        var material = PhysicallyBasedMaterial()
        // Use eithr the classification or alignment colors
        material.baseColor.tint = useAlighmentColors ? colorForPlaneAlignment(anchor.alignment) : colorForPlaneClassification(anchor.surfaceClassification)

        if let meshResource = createMeshResource(anchor: anchor) {
            entity.components.set(ModelComponent(mesh: meshResource, materials: [material]))
        }

        return entity
    }

    // Helper functions to determine color from classification or alignment
    private func colorForPlaneClassification(_ classification: SurfaceClassification?) -> UIColor {
        guard let classification else {
            return .white
        }

        // Current cases: https://developer.apple.com/documentation/arkit/planeanchor/classification-swift.enum
        // We only care about wall, ceiling, and floor today
        switch classification {
        case .wall:
            return .systemRed
        case .ceiling:
            return .systemBlue
        case .floor:
            return .systemGreen
        default:
            return .white
        }
    }

    private func colorForPlaneAlignment(_ alignment: PlaneAnchor.Alignment?) -> UIColor {
        guard let alignment else {
            return .white
        }

        // Only three cases as of visionOS 2.4
        switch alignment {
        case .horizontal:
            return .systemOrange
        case .vertical:
            return .systemPurple
        case .slanted:
            return .systemCyan
        default:
            return .white
        }
    }


    private func createMeshResource(anchor: PlaneAnchor) -> MeshResource? {
        // Generate a mesh for the plane (for occlusion).
        var meshResource: MeshResource? = nil
        do {
            var contents = MeshResource.Contents()
            contents.instances = [MeshResource.Instance(id: "main", model: "model")]
            var part = MeshResource.Part(id: "part", materialIndex: 0)

            // Convert vertices to SIMD3<Float>
            let vertices = anchor.geometry.meshVertices
            var vertexArray: [SIMD3<Float>] = []
            for i in 0..<vertices.count {
                let vertex = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * i).assumingMemoryBound(to: (Float, Float, Float).self).pointee
                vertexArray.append(SIMD3<Float>(vertex.0, vertex.1, vertex.2))
            }
            part.positions = MeshBuffers.Positions(vertexArray)

            // Convert faces to UInt32
            let faces = anchor.geometry.meshFaces
            var faceArray: [UInt32] = []
            let totalFaces = faces.count * faces.primitive.indexCount
            for i in 0..<totalFaces {
                let face = faces.buffer.contents().advanced(by: i * MemoryLayout<Int32>.size).assumingMemoryBound(to: Int32.self).pointee
                faceArray.append(UInt32(face))
            }
            part.triangleIndices = MeshBuffer(faceArray)

            contents.models = [MeshResource.Model(id: "model", parts: [part])]
            meshResource = try MeshResource.generate(from: contents)
            return meshResource
        } catch {
            print("Failed to create a mesh resource for a plane anchor: \(error).")
        }
        return nil
    }

}

Updated for visionOS 26 to reflect how we access surface classification

// prior to visionOS 26 we use anchor.classification
anchor.classification

// Now we use anchor.surfaceClassification
anchor.surfaceClassification

Support our work so we can continue to bring you new examples and articles.

Support our Work

Download the Xcode project with this and many more examples from Step Into Vision.
Some examples are provided as standalone Xcode projects. You can find those here.

Follow Step Into Vision

Questions or feedback?