diff --git a/build.gradle b/build.gradle index d0799c72..1784c08d 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,7 @@ plugins { project.ext { openrndrVersion = "0.3.35-rc1" + libfreenectVersion = "0.5.7-1.5.1" } allprojects { diff --git a/orx-kinect-common/src/main/kotlin/Kinect.kt b/orx-kinect-common/src/main/kotlin/Kinect.kt new file mode 100644 index 00000000..22a63309 --- /dev/null +++ b/orx-kinect-common/src/main/kotlin/Kinect.kt @@ -0,0 +1,125 @@ +package org.openrndr.extra.kinect + +import org.openrndr.Extension +import org.openrndr.Program +import org.openrndr.draw.* +import org.openrndr.math.Vector2 +import org.openrndr.resourceUrl +import java.lang.RuntimeException +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicReference +import kotlin.concurrent.thread + +interface Kinects { + fun countDevices(): Int + + /** + * Starts kinect device of a given number. + * + * @throws KinectException if device of such a number does not exist. + */ + fun startDevice(num: Int = 0): KinectDevice +} + +class DefaultKinects(private val manager: KinectsManager) : Kinects { + init { + manager.init() + // as we don't have explicit shutdown mechanism in OPENRNDR + // we need to install a shutdown hook for now + Runtime.getRuntime().addShutdownHook( + thread( + name = "${manager.javaClass.simpleName}-closer", + start = false, + isDaemon = false + ) { + manager.shutdown() + } + ) + } + override fun countDevices(): Int { + return manager.countDevices() + } + override fun startDevice(num: Int): KinectDevice { + return manager.startDevice(num) + } +} + +interface KinectsManager { + fun init() + fun countDevices(): Int + fun startDevice(num: Int): KinectDevice + fun shutdown() +} + +abstract class KinectDevice : Extension { + override var enabled: Boolean = true + abstract val depthCamera: KinectDepthCamera + override fun beforeDraw(drawer: Drawer, program: Program) { + if (depthCamera.enabled) { + depthCamera.update() + } + } +} + +interface KinectCamera { + var enabled: Boolean + val width: Int + val height: Int + var mirror: Boolean + val currentFrame: ColorBuffer + fun getLatestFrame(): ColorBuffer? + fun update() +} + +interface KinectDepthCamera : KinectCamera { + val depthScale: Double +} + +abstract class AbstractKinectDepthCamera( + final override val width: Int, + final override val height: Int, + final override val depthScale: Double +) : KinectDepthCamera { + protected val byteBufferRef: AtomicReference = AtomicReference() + private val rawBuffer: ColorBuffer = colorBuffer( + width, + height, + format = ColorFormat.R, + type = ColorType.UINT16 // it would be perfect if we could use isampler in the shader + ) + override val currentFrame: ColorBuffer = colorBuffer( + width, + height, + format = ColorFormat.R, + type = ColorType.FLOAT16 // in the future we might want to choose the precision here + ) + private val depthMapper = KinectRawDataToDepthMapper() + init { + depthMapper.depthScale = depthScale + depthMapper.mirror = false + depthMapper.resolution = Vector2(width.toDouble(), height.toDouble()) + } + private val newFrameRef = AtomicReference() + override fun getLatestFrame(): ColorBuffer? { + return newFrameRef.getAndSet(null) + } + override fun update() { + byteBufferRef.getAndSet(null)?.let { bytes -> + rawBuffer.write(bytes) + depthMapper.apply(rawBuffer, currentFrame) + newFrameRef.set(currentFrame) + } + } + override var mirror: Boolean + get() = depthMapper.mirror + set(value) { depthMapper.mirror = value } +} + +class KinectRawDataToDepthMapper : + Filter(filterShaderFromUrl(resourceUrl("kinect-raw-to-depth.frag", Kinects::class.java))) { + var depthScale: Double by parameters + var mirror: Boolean by parameters + var resolution: Vector2 by parameters +} + +class KinectException(msg: String) : RuntimeException(msg) diff --git a/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/kinect-raw-to-depth.frag b/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/kinect-raw-to-depth.frag new file mode 100644 index 00000000..804078af --- /dev/null +++ b/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/kinect-raw-to-depth.frag @@ -0,0 +1,16 @@ +#version 330 + +uniform sampler2D tex0; // kinect raw +uniform vec2 resolution; // kinect resolution +uniform float depthScale; // 32 for kinect1, 64 for kinect2 +uniform bool mirror; + +out float depth; + +void main() { + ivec2 uv = ivec2( + mirror ? int(resolution.x) - 1 - int(gl_FragCoord.x) : int(gl_FragCoord.x), + int(resolution.y) - 1 - int(gl_FragCoord.y) + ); + depth = texelFetch(tex0, uv, 0).r * depthScale; +} diff --git a/orx-kinect-v1-natives-linux-x64/build.gradle b/orx-kinect-v1-natives-linux-x64/build.gradle new file mode 100644 index 00000000..0f456c2f --- /dev/null +++ b/orx-kinect-v1-natives-linux-x64/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtime "org.bytedeco:libfreenect:$libfreenectVersion:linux-x86_64" +} diff --git a/orx-kinect-v1-natives-macos/build.gradle b/orx-kinect-v1-natives-macos/build.gradle new file mode 100644 index 00000000..711dae42 --- /dev/null +++ b/orx-kinect-v1-natives-macos/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtime "org.bytedeco:libfreenect:$libfreenectVersion:macosx-x86_64" +} diff --git a/orx-kinect-v1-natives-windows/build.gradle b/orx-kinect-v1-natives-windows/build.gradle new file mode 100644 index 00000000..15c9ecd2 --- /dev/null +++ b/orx-kinect-v1-natives-windows/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtime "org.bytedeco:libfreenect:$libfreenectVersion:windows-x86_64" +} diff --git a/orx-kinect-v1/build.gradle b/orx-kinect-v1/build.gradle new file mode 100644 index 00000000..dcbf0ad5 --- /dev/null +++ b/orx-kinect-v1/build.gradle @@ -0,0 +1,5 @@ +dependencies { + compile project(":orx-kinect-common") + compile "io.github.microutils:kotlin-logging:1.7.2" + compile "org.bytedeco:libfreenect:$libfreenectVersion" +} diff --git a/orx-kinect-v1/src/main/kotlin/KinectV1.kt b/orx-kinect-v1/src/main/kotlin/KinectV1.kt new file mode 100644 index 00000000..5d2ad830 --- /dev/null +++ b/orx-kinect-v1/src/main/kotlin/KinectV1.kt @@ -0,0 +1,149 @@ +package org.openrndr.extra.kinect + +import mu.KotlinLogging +import org.bytedeco.javacpp.Pointer +import org.bytedeco.libfreenect.freenect_context +import org.bytedeco.libfreenect.freenect_depth_cb +import org.bytedeco.libfreenect.freenect_device +import org.bytedeco.libfreenect.freenect_usb_context +import org.bytedeco.libfreenect.global.freenect.* +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.* +import kotlin.concurrent.thread + +fun getKinectsV1() : Kinects { + return DefaultKinects(KinectsV1Manager()) +} + +class KinectsV1Manager : KinectsManager { + + private val logger = KotlinLogging.logger {} + + private val fnCtx = freenect_context() + + private val fnUsbCtx = freenect_usb_context() + + private var running: Boolean = true + + private val devices: LinkedList = LinkedList() + + private val poller: Thread = thread(name = "Kinect1-poll", start = false, isDaemon = true) { + while (running && freenect_process_events(fnCtx) >= 0) {} + } + + inner class KinectV1Device(private val num: Int) : KinectDevice() { + override val depthCamera = KinectV1DepthCamera() + val fnDev = freenect_device() + init { + logger.info { "Opening Kinect1 device num: $num" } + verify(freenect_open_device(fnCtx, fnDev, num)) + } + fun shutdown() { + logger.info { "Shutting down Kinect1 device num: $num" } + if (!fnDev.isNull) { + verifyOnShutdown(freenect_stop_depth(fnDev)) + verifyOnShutdown(freenect_close_device(fnDev)) + } + } + inner class KinectV1DepthCamera : AbstractKinectDepthCamera( + width = 640, + height = 480, + depthScale = 32.0 + ) { + private val logger = KotlinLogging.logger {} + private val bytes = ByteBuffer.allocateDirect(width * height * 2) + private val freenectDepthCb = object : freenect_depth_cb() { + override fun call(dev: freenect_device?, depth: Pointer?, timestamp: Int) { + logger.trace { "depth frame received for Kinect1 device: $num, at: $timestamp" } + byteBufferRef.set(bytes) + } + } + init { + bytes.order(ByteOrder.nativeOrder()) + } + override var enabled: Boolean = false + set(value) { + field = if (value) { + start() + true + } else { + stop() + false + } + } + private fun start() { + logger.info { "Enabling Kinect1 depth camera, device num: $num" } + verify(freenect_set_depth_mode( + fnDev, freenect_find_depth_mode(FREENECT_RESOLUTION_MEDIUM, FREENECT_DEPTH_11BIT)) + ) + verify(freenect_set_depth_buffer(fnDev, Pointer(depthCamera.bytes))) + freenect_set_depth_callback(fnDev, depthCamera.freenectDepthCb) + verify(freenect_start_depth(fnDev)) + } + private fun stop() { + logger.info { "Disabling Kinect1 depth camera, device num: $num" } + verify(freenect_stop_depth(fnDev)) + } + } + } + + override fun init() { + logger.info("Initializing Kinect1 support, set log level to TRACE to see received frames") + verify(freenect_init(fnCtx, fnUsbCtx)) + freenect_set_log_level(fnCtx, FREENECT_LOG_DEBUG) + freenect_select_subdevices(fnCtx, FREENECT_DEVICE_CAMERA) + val num = verify(freenect_num_devices(fnCtx)) + if (num == 0) { + logger.warn { "Could not find any Kinect1 device, calling startDevice() will throw exception" } + } + logger.debug("Initializing Kinect1 poller thread") + poller.start() + // it seems that we have to wait a bit until kinect is actually initialized + Thread.sleep(100) + } + + override fun countDevices(): Int { + return verify(freenect_num_devices(fnCtx)) + } + + override fun startDevice(num: Int): KinectDevice { + val count = countDevices() + if (num >= count) { + throw KinectException("Non-existent Kinect1 device, num: $num") + } + val device = KinectV1Device(num) + devices.add(device) + return device + } + + override fun shutdown() { + logger.info("Shutting down Kinect1 support") + running = false + poller.join() + devices.forEach { + device -> device.shutdown() + } + if (!fnCtx.isNull) { + verifyOnShutdown(freenect_shutdown(fnCtx)) + } + } + + private fun verifyOnShutdown(ret: Int) { + if (ret != 0) { + logger.error("Unexpected return value while shutting down kinect support: {}", ret) + } + } + + private fun verify(ret: Int): Int { + if (ret < 0) { + fail("ret=$ret") + } + return ret + } + + private fun fail(message: String) { + throw KinectException("Kinect1 error: $message") + } + +} diff --git a/settings.gradle b/settings.gradle index e9def1be..420d5f80 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,4 +14,9 @@ include 'orx-camera', 'orx-no-clear', 'orx-noise', 'orx-obj-loader', - 'orx-olive' \ No newline at end of file + 'orx-olive', + 'orx-kinect-common', + 'orx-kinect-v1', + 'orx-kinect-v1-natives-linux-x64', + 'orx-kinect-v1-natives-macos' + 'orx-kinect-v1-natives-windows'