[orx-realsense2] Add realsense 2 support

build.gradle

@@ -18,10 +18,11 @@ def openrndrUseSnapshot = false
 apply plugin: 'org.jetbrains.dokka'
 
 project.ext {
-    openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.44"
+    openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.45.rc-1"
     kotlinVersion = "1.4.0"
     spekVersion = "2.0.12"
     libfreenectVersion = "0.5.7-1.5.4"
+    librealsense2Version = "2.29.0-1.5.4"
     gsonVersion = "2.8.6"
     antlrVersion = "4.8-1"
     tensorflowVersion = "0.2.0"
@@ -200,7 +201,8 @@ task collectScreenshots {
                 continue
             if (sub.name == "orx-video-profiles")
                 continue
-
+            if (sub.name == "orx-realsense2")
+                continue
             def set = sub.sourceSets.demo
             def ucl = new URLClassLoader(set.runtimeClasspath.collect { it.toURI().toURL() } as URL[])
 

orx-realsense2-natives-linux-x64/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:linux-x86_64"
+}

orx-realsense2-natives-macos/build.gradle

@@ -0,0 +1,4 @@
+
+dependencies {
+    runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:macosx-x86_64"
+}

orx-realsense2-natives-windows/build.gradle

@@ -0,0 +1,4 @@
+
+dependencies {
+    runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:windows-x86_64"
+}

orx-realsense2/build.gradle

@@ -0,0 +1,24 @@
+sourceSets {
+    demo {
+        java {
+            srcDirs = ["src/demo/kotlin"]
+            compileClasspath += main.getCompileClasspath()
+            runtimeClasspath += main.getRuntimeClasspath()
+        }
+    }
+}
+
+
+
+dependencies {
+    implementation "io.github.microutils:kotlin-logging:1.7.2"
+    api "org.bytedeco:librealsense2:$librealsense2Version"
+
+    demoImplementation("org.openrndr:openrndr-core:$openrndrVersion")
+    demoImplementation("org.openrndr:openrndr-extensions:$openrndrVersion")
+    demoRuntimeOnly(project(":orx-realsense2-natives-$openrndrOS"))
+    demoRuntimeOnly("org.openrndr:openrndr-gl3:$openrndrVersion")
+    demoRuntimeOnly("org.openrndr:openrndr-gl3-natives-$openrndrOS:$openrndrVersion")
+    demoImplementation(sourceSets.getByName("main").output)
+
+}

orx-realsense2/src/demo/kotlin/DemoRS201.kt

@@ -0,0 +1,26 @@
+import org.openrndr.application
+import org.openrndr.draw.ColorFormat
+import org.openrndr.draw.ColorType
+import org.openrndr.draw.colorBuffer
+import org.openrndr.extra.realsense2.RS2Sensor
+
+fun main() {
+    application {
+        program {
+            val sensors = RS2Sensor.listSensors()
+            val depthFrame = colorBuffer(640, 480, format = ColorFormat.R, type = ColorType.UINT16)
+            for (sensor in sensors) {
+                println(sensor)
+            }
+            val sensor = RS2Sensor.openFirstOrDummy()
+            println(sensor)
+            sensor.depthFrameReceived.listen {
+                it.copyTo(depthFrame)
+            }
+            extend {
+                sensor.waitForFrames()
+                drawer.image(depthFrame)
+            }
+        }
+    }
+}

orx-realsense2/src/main/kotlin/RS2Sensor.kt

@@ -0,0 +1,163 @@
+package org.openrndr.extra.realsense2
+
+import org.bytedeco.javacpp.BytePointer
+import org.bytedeco.librealsense2.*
+import org.bytedeco.librealsense2.global.realsense2.*
+import org.openrndr.draw.ColorBuffer
+import org.openrndr.events.Event
+import java.nio.ByteBuffer
+
+private val rs2Context by lazy {
+    val error = rs2_error()
+    val ctx = rs2_create_context(RS2_API_VERSION, error)
+    ctx
+}
+
+private fun rs2_error.check() {
+    if (!this.isNull) {
+        val function = rs2_get_failed_function(this)
+        val args = rs2_get_failed_args(this)
+        val errorMessage = rs2_get_error_message(this)
+        error("$errorMessage in $function with $args")
+    }
+}
+
+enum class RS2DepthFormat {
+    UINT16
+}
+class RS2SensorDescription(private val deviceList: rs2_device_list, private val deviceIndex: Int) {
+
+    /**
+     * open realsense sensor from description entry
+     */
+    fun open(
+            depthWidth: Int = 640,
+            depthHeight: Int = 480,
+            depthFps: Int = 30,
+            depthFormat: RS2DepthFormat = RS2DepthFormat.UINT16
+
+    ): RS2Sensor {
+        val error = rs2_error()
+        val device = rs2_create_device(deviceList, deviceIndex, error);
+        error.check()
+
+        val pipeline = rs2_create_pipeline(rs2Context, error)
+        error.check()
+
+        val config = rs2_create_config(error)
+        error.check()
+
+        rs2_config_enable_stream(config, RS2_STREAM_DEPTH, 0, depthWidth, depthHeight, RS2_FORMAT_Z16, depthFps, error)
+        error.check()
+
+        val pipelineProfile = rs2_pipeline_start_with_config(pipeline, config, error)
+        error.check()
+        return RS2Sensor(device, pipeline, pipelineProfile)
+    }
+}
+
+class RS2FrameEvent(private val frame: rs2_frame, val frameWidth: Int, val frameHeight: Int) {
+    val frameData: ByteBuffer
+    get() {
+        val error = rs2_error()
+        val pointer = rs2_get_frame_data(frame, error)
+        val buffer = BytePointer(pointer).capacity(frameWidth * frameHeight * 2L).asByteBuffer()
+        error.check()
+        error.close()
+        pointer.close()
+        return buffer
+    }
+
+    fun copyTo(target : ColorBuffer) {
+        target.write(frameData)
+    }
+}
+
+abstract class Sensor {
+    /**
+     * depth frame received event, triggered from [waitForFrames]
+     */
+    val depthFrameReceived = Event<RS2FrameEvent>()
+
+    /**
+     * wait for frames to arrives
+     */
+    abstract fun waitForFrames()
+
+    /**
+     * destroy the sensor
+     */
+    abstract fun destroy()
+}
+
+class DummySensor : Sensor() {
+    override fun waitForFrames() {
+    }
+
+    override fun destroy() {
+    }
+}
+
+class RS2Sensor(
+        private val device: rs2_device,
+        private val pipeline: rs2_pipeline,
+        private val pipelineProfile: rs2_pipeline_profile
+
+) : Sensor() {
+    override fun waitForFrames() {
+        val error = rs2_error()
+        val frames = rs2_pipeline_wait_for_frames(pipeline, RS2_DEFAULT_TIMEOUT, error)
+        error.check()
+
+        val frameCount = rs2_embedded_frames_count(frames, error)
+        error.check()
+
+        for (i in 0 until frameCount) {
+            val frame = rs2_extract_frame(frames, i, error)
+            error.check()
+            val cmp = rs2_is_frame_extendable_to(frame, RS2_EXTENSION_DEPTH_FRAME, error)
+            if (cmp != 0) {
+                val width = rs2_get_frame_width(frame, error)
+                error.check()
+                val height = rs2_get_frame_height(frame, error)
+                error.check()
+                val eventMessage = RS2FrameEvent(frame, width, height)
+                depthFrameReceived.trigger(eventMessage)
+                rs2_release_frame(frame)
+            }
+        }
+        rs2_release_frame(frames)
+    }
+
+    override fun destroy() {
+        val error = rs2_error()
+        rs2_pipeline_stop(pipeline, error)
+        error.check()
+    }
+
+    companion object {
+        /**
+         * list all connected Realsense devices
+         */
+        fun listSensors(): List<RS2SensorDescription> {
+            val error = rs2_error()
+            val deviceList = rs2_query_devices(rs2Context, error)
+            error.check()
+            val deviceCount = rs2_get_device_count(deviceList, error)
+            error.check()
+            return if (deviceCount == 0) {
+                emptyList()
+            } else {
+                (0 until deviceCount).map {
+                    RS2SensorDescription(deviceList, it)
+                }
+            }
+        }
+        /**
+         * open the first available sensor or a dummy sensor if no real sensors are available
+         */
+        fun openFirstOrDummy() : Sensor {
+            return listSensors().firstOrNull()?.open() ?: DummySensor()
+        }
+    }
+}

settings.gradle

@@ -33,6 +33,8 @@ include 'openrndr-demos',
         'orx-panel',
         'orx-poisson-fill',
         'orx-rabbit-control',
+        'orx-realsense2',
+        'orx-realsense2-natives-windows',
         'orx-runway',
         'orx-shader-phrases',
         'orx-shade-styles',