diff --git a/build.gradle b/build.gradle index cc20692a..78490a6b 100644 --- a/build.gradle +++ b/build.gradle @@ -18,10 +18,11 @@ def openrndrUseSnapshot = false apply plugin: 'org.jetbrains.dokka' project.ext { - openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.44" + openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.45.rc-1" kotlinVersion = "1.4.0" spekVersion = "2.0.12" libfreenectVersion = "0.5.7-1.5.4" + librealsense2Version = "2.29.0-1.5.4" gsonVersion = "2.8.6" antlrVersion = "4.8-1" tensorflowVersion = "0.2.0" @@ -200,7 +201,8 @@ task collectScreenshots { continue if (sub.name == "orx-video-profiles") continue - + if (sub.name == "orx-realsense2") + continue def set = sub.sourceSets.demo def ucl = new URLClassLoader(set.runtimeClasspath.collect { it.toURI().toURL() } as URL[]) diff --git a/orx-realsense2-natives-linux-x64/build.gradle b/orx-realsense2-natives-linux-x64/build.gradle new file mode 100644 index 00000000..4ac51b83 --- /dev/null +++ b/orx-realsense2-natives-linux-x64/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:linux-x86_64" +} diff --git a/orx-realsense2-natives-macos/build.gradle b/orx-realsense2-natives-macos/build.gradle new file mode 100644 index 00000000..efe34b4a --- /dev/null +++ b/orx-realsense2-natives-macos/build.gradle @@ -0,0 +1,4 @@ + +dependencies { + runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:macosx-x86_64" +} diff --git a/orx-realsense2-natives-windows/build.gradle b/orx-realsense2-natives-windows/build.gradle new file mode 100644 index 00000000..b94d9ac0 --- /dev/null +++ b/orx-realsense2-natives-windows/build.gradle @@ -0,0 +1,4 @@ + +dependencies { + runtimeOnly "org.bytedeco:librealsense2:$librealsense2Version:windows-x86_64" +} diff --git a/orx-realsense2/build.gradle b/orx-realsense2/build.gradle new file mode 100644 index 00000000..2919e113 --- /dev/null +++ b/orx-realsense2/build.gradle @@ -0,0 +1,24 @@ +sourceSets { + demo { + java { + srcDirs = ["src/demo/kotlin"] + compileClasspath += main.getCompileClasspath() + runtimeClasspath += main.getRuntimeClasspath() + } + } +} + + + +dependencies { + implementation "io.github.microutils:kotlin-logging:1.7.2" + api "org.bytedeco:librealsense2:$librealsense2Version" + + demoImplementation("org.openrndr:openrndr-core:$openrndrVersion") + demoImplementation("org.openrndr:openrndr-extensions:$openrndrVersion") + demoRuntimeOnly(project(":orx-realsense2-natives-$openrndrOS")) + demoRuntimeOnly("org.openrndr:openrndr-gl3:$openrndrVersion") + demoRuntimeOnly("org.openrndr:openrndr-gl3-natives-$openrndrOS:$openrndrVersion") + demoImplementation(sourceSets.getByName("main").output) + +} diff --git a/orx-realsense2/src/demo/kotlin/DemoRS201.kt b/orx-realsense2/src/demo/kotlin/DemoRS201.kt new file mode 100644 index 00000000..9ed9ce4f --- /dev/null +++ b/orx-realsense2/src/demo/kotlin/DemoRS201.kt @@ -0,0 +1,26 @@ +import org.openrndr.application +import org.openrndr.draw.ColorFormat +import org.openrndr.draw.ColorType +import org.openrndr.draw.colorBuffer +import org.openrndr.extra.realsense2.RS2Sensor + +fun main() { + application { + program { + val sensors = RS2Sensor.listSensors() + val depthFrame = colorBuffer(640, 480, format = ColorFormat.R, type = ColorType.UINT16) + for (sensor in sensors) { + println(sensor) + } + val sensor = RS2Sensor.openFirstOrDummy() + println(sensor) + sensor.depthFrameReceived.listen { + it.copyTo(depthFrame) + } + extend { + sensor.waitForFrames() + drawer.image(depthFrame) + } + } + } +} \ No newline at end of file diff --git a/orx-realsense2/src/main/kotlin/RS2Sensor.kt b/orx-realsense2/src/main/kotlin/RS2Sensor.kt new file mode 100644 index 00000000..8270334b --- /dev/null +++ b/orx-realsense2/src/main/kotlin/RS2Sensor.kt @@ -0,0 +1,163 @@ +package org.openrndr.extra.realsense2 + +import org.bytedeco.javacpp.BytePointer +import org.bytedeco.librealsense2.* +import org.bytedeco.librealsense2.global.realsense2.* +import org.openrndr.draw.ColorBuffer +import org.openrndr.events.Event +import java.nio.ByteBuffer + +private val rs2Context by lazy { + val error = rs2_error() + val ctx = rs2_create_context(RS2_API_VERSION, error) + ctx +} + +private fun rs2_error.check() { + if (!this.isNull) { + val function = rs2_get_failed_function(this) + val args = rs2_get_failed_args(this) + val errorMessage = rs2_get_error_message(this) + error("$errorMessage in $function with $args") + } +} + +enum class RS2DepthFormat { + UINT16 +} +class RS2SensorDescription(private val deviceList: rs2_device_list, private val deviceIndex: Int) { + + /** + * open realsense sensor from description entry + */ + fun open( + depthWidth: Int = 640, + depthHeight: Int = 480, + depthFps: Int = 30, + depthFormat: RS2DepthFormat = RS2DepthFormat.UINT16 + + ): RS2Sensor { + val error = rs2_error() + val device = rs2_create_device(deviceList, deviceIndex, error); + error.check() + + val pipeline = rs2_create_pipeline(rs2Context, error) + error.check() + + val config = rs2_create_config(error) + error.check() + + rs2_config_enable_stream(config, RS2_STREAM_DEPTH, 0, depthWidth, depthHeight, RS2_FORMAT_Z16, depthFps, error) + error.check() + + val pipelineProfile = rs2_pipeline_start_with_config(pipeline, config, error) + error.check() + return RS2Sensor(device, pipeline, pipelineProfile) + } +} + +class RS2FrameEvent(private val frame: rs2_frame, val frameWidth: Int, val frameHeight: Int) { + val frameData: ByteBuffer + get() { + val error = rs2_error() + val pointer = rs2_get_frame_data(frame, error) + val buffer = BytePointer(pointer).capacity(frameWidth * frameHeight * 2L).asByteBuffer() + error.check() + error.close() + pointer.close() + return buffer + } + + fun copyTo(target : ColorBuffer) { + target.write(frameData) + } +} + +abstract class Sensor { + /** + * depth frame received event, triggered from [waitForFrames] + */ + val depthFrameReceived = Event() + + /** + * wait for frames to arrives + */ + abstract fun waitForFrames() + + /** + * destroy the sensor + */ + abstract fun destroy() +} + +class DummySensor : Sensor() { + override fun waitForFrames() { + } + + override fun destroy() { + } +} + +class RS2Sensor( + private val device: rs2_device, + private val pipeline: rs2_pipeline, + private val pipelineProfile: rs2_pipeline_profile + +) : Sensor() { + override fun waitForFrames() { + val error = rs2_error() + val frames = rs2_pipeline_wait_for_frames(pipeline, RS2_DEFAULT_TIMEOUT, error) + error.check() + + val frameCount = rs2_embedded_frames_count(frames, error) + error.check() + + for (i in 0 until frameCount) { + val frame = rs2_extract_frame(frames, i, error) + error.check() + val cmp = rs2_is_frame_extendable_to(frame, RS2_EXTENSION_DEPTH_FRAME, error) + if (cmp != 0) { + val width = rs2_get_frame_width(frame, error) + error.check() + val height = rs2_get_frame_height(frame, error) + error.check() + val eventMessage = RS2FrameEvent(frame, width, height) + depthFrameReceived.trigger(eventMessage) + rs2_release_frame(frame) + } + } + rs2_release_frame(frames) + } + + override fun destroy() { + val error = rs2_error() + rs2_pipeline_stop(pipeline, error) + error.check() + } + + companion object { + /** + * list all connected Realsense devices + */ + fun listSensors(): List { + val error = rs2_error() + val deviceList = rs2_query_devices(rs2Context, error) + error.check() + val deviceCount = rs2_get_device_count(deviceList, error) + error.check() + return if (deviceCount == 0) { + emptyList() + } else { + (0 until deviceCount).map { + RS2SensorDescription(deviceList, it) + } + } + } + /** + * open the first available sensor or a dummy sensor if no real sensors are available + */ + fun openFirstOrDummy() : Sensor { + return listSensors().firstOrNull()?.open() ?: DummySensor() + } + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 96dcf8c6..e9b82bb4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -33,6 +33,8 @@ include 'openrndr-demos', 'orx-panel', 'orx-poisson-fill', 'orx-rabbit-control', + 'orx-realsense2', + 'orx-realsense2-natives-windows', 'orx-runway', 'orx-shader-phrases', 'orx-shade-styles',