initial kinect1 support

build.gradle

@@ -38,6 +38,7 @@ plugins {
 
 project.ext {
     openrndrVersion = "0.3.35-rc1"
+    libfreenectVersion = "0.5.7-1.5.1"
 }
 
 allprojects {

orx-kinect-common/src/main/kotlin/Kinect.kt

@@ -0,0 +1,125 @@
+package org.openrndr.extra.kinect
+
+import org.openrndr.Extension
+import org.openrndr.Program
+import org.openrndr.draw.*
+import org.openrndr.math.Vector2
+import org.openrndr.resourceUrl
+import java.lang.RuntimeException
+import java.nio.ByteBuffer
+import java.util.concurrent.atomic.AtomicReference
+import kotlin.concurrent.thread
+
+interface Kinects {
+    fun countDevices(): Int
+
+    /**
+     * Starts kinect device of a given number.
+     *
+     * @throws KinectException if device of such a number does not exist.
+     */
+    fun startDevice(num: Int = 0): KinectDevice
+}
+
+class DefaultKinects(private val manager: KinectsManager) : Kinects {
+    init {
+        manager.init()
+        // as we don't have explicit shutdown mechanism in OPENRNDR
+        // we need to install a shutdown hook for now
+        Runtime.getRuntime().addShutdownHook(
+            thread(
+                name = "${manager.javaClass.simpleName}-closer",
+                start = false,
+                isDaemon = false
+            ) {
+                manager.shutdown()
+            }
+        )
+    }
+    override fun countDevices(): Int {
+        return manager.countDevices()
+    }
+    override fun startDevice(num: Int): KinectDevice {
+        return manager.startDevice(num)
+    }
+}
+
+interface KinectsManager {
+    fun init()
+    fun countDevices(): Int
+    fun startDevice(num: Int): KinectDevice
+    fun shutdown()
+}
+
+abstract class KinectDevice : Extension {
+    override var enabled: Boolean = true
+    abstract val depthCamera: KinectDepthCamera
+    override fun beforeDraw(drawer: Drawer, program: Program) {
+        if (depthCamera.enabled) {
+            depthCamera.update()
+        }
+    }
+}
+
+interface KinectCamera {
+    var enabled: Boolean
+    val width: Int
+    val height: Int
+    var mirror: Boolean
+    val currentFrame: ColorBuffer
+    fun getLatestFrame(): ColorBuffer?
+    fun update()
+}
+
+interface KinectDepthCamera : KinectCamera {
+    val depthScale: Double
+}
+
+abstract class AbstractKinectDepthCamera(
+    final override val width: Int,
+    final override val height: Int,
+    final override val depthScale: Double
+) : KinectDepthCamera {
+    protected val byteBufferRef: AtomicReference<ByteBuffer> = AtomicReference()
+    private val rawBuffer: ColorBuffer = colorBuffer(
+        width,
+        height,
+        format = ColorFormat.R,
+        type = ColorType.UINT16 // it would be perfect if we could use isampler in the shader
+    )
+    override val currentFrame: ColorBuffer = colorBuffer(
+        width,
+        height,
+        format = ColorFormat.R,
+        type = ColorType.FLOAT16 // in the future we might want to choose the precision here
+    )
+    private val depthMapper = KinectRawDataToDepthMapper()
+    init {
+        depthMapper.depthScale = depthScale
+        depthMapper.mirror = false
+        depthMapper.resolution = Vector2(width.toDouble(), height.toDouble())
+    }
+    private val newFrameRef = AtomicReference<ColorBuffer>()
+    override fun getLatestFrame(): ColorBuffer? {
+        return newFrameRef.getAndSet(null)
+    }
+    override fun update() {
+        byteBufferRef.getAndSet(null)?.let { bytes ->
+            rawBuffer.write(bytes)
+            depthMapper.apply(rawBuffer, currentFrame)
+            newFrameRef.set(currentFrame)
+        }
+    }
+    override var mirror: Boolean
+        get() = depthMapper.mirror
+        set(value) { depthMapper.mirror = value }
+}
+
+class KinectRawDataToDepthMapper :
+        Filter(filterShaderFromUrl(resourceUrl("kinect-raw-to-depth.frag", Kinects::class.java))) {
+    var depthScale: Double by parameters
+    var mirror: Boolean by parameters
+    var resolution: Vector2 by parameters
+}
+
+class KinectException(msg: String) : RuntimeException(msg)

orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/kinect-raw-to-depth.frag

@@ -0,0 +1,16 @@
+#version 330
+
+uniform sampler2D tex0;             // kinect raw
+uniform vec2      resolution;       // kinect resolution
+uniform float     depthScale;       // 32 for kinect1, 64 for kinect2
+uniform bool      mirror;
+
+out     float depth;
+
+void main() {
+    ivec2 uv = ivec2(
+        mirror ? int(resolution.x) - 1 - int(gl_FragCoord.x) : int(gl_FragCoord.x),
+        int(resolution.y) - 1 - int(gl_FragCoord.y)
+    );
+    depth = texelFetch(tex0, uv, 0).r * depthScale;
+}

orx-kinect-v1-natives-linux-x64/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    runtime "org.bytedeco:libfreenect:$libfreenectVersion:linux-x86_64"
+}

orx-kinect-v1-natives-macos/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    runtime "org.bytedeco:libfreenect:$libfreenectVersion:macosx-x86_64"
+}

orx-kinect-v1-natives-windows/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    runtime "org.bytedeco:libfreenect:$libfreenectVersion:windows-x86_64"
+}

orx-kinect-v1/build.gradle

@@ -0,0 +1,5 @@
+dependencies {
+    compile project(":orx-kinect-common")
+    compile "io.github.microutils:kotlin-logging:1.7.2"
+    compile "org.bytedeco:libfreenect:$libfreenectVersion"
+}

orx-kinect-v1/src/main/kotlin/KinectV1.kt

@@ -0,0 +1,149 @@
+package org.openrndr.extra.kinect
+
+import mu.KotlinLogging
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.libfreenect.freenect_context
+import org.bytedeco.libfreenect.freenect_depth_cb
+import org.bytedeco.libfreenect.freenect_device
+import org.bytedeco.libfreenect.freenect_usb_context
+import org.bytedeco.libfreenect.global.freenect.*
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.*
+import kotlin.concurrent.thread
+
+fun getKinectsV1() : Kinects {
+    return DefaultKinects(KinectsV1Manager())
+}
+
+class KinectsV1Manager : KinectsManager {
+
+    private val logger = KotlinLogging.logger {}
+
+    private val fnCtx = freenect_context()
+
+    private val fnUsbCtx = freenect_usb_context()
+
+    private var running: Boolean = true
+
+    private val devices: LinkedList<KinectV1Device> = LinkedList()
+
+    private val poller: Thread = thread(name = "Kinect1-poll", start = false, isDaemon = true) {
+        while (running && freenect_process_events(fnCtx) >= 0) {}
+    }
+
+    inner class KinectV1Device(private val num: Int) : KinectDevice() {
+        override val depthCamera = KinectV1DepthCamera()
+        val fnDev = freenect_device()
+        init {
+            logger.info { "Opening Kinect1 device num: $num" }
+            verify(freenect_open_device(fnCtx, fnDev, num))
+        }
+        fun shutdown() {
+            logger.info { "Shutting down Kinect1 device num: $num" }
+            if (!fnDev.isNull) {
+                verifyOnShutdown(freenect_stop_depth(fnDev))
+                verifyOnShutdown(freenect_close_device(fnDev))
+            }
+        }
+        inner class KinectV1DepthCamera : AbstractKinectDepthCamera(
+                width = 640,
+                height = 480,
+                depthScale = 32.0
+        ) {
+            private val logger = KotlinLogging.logger {}
+            private val bytes = ByteBuffer.allocateDirect(width * height * 2)
+            private val freenectDepthCb = object : freenect_depth_cb() {
+                override fun call(dev: freenect_device?, depth: Pointer?, timestamp: Int) {
+                    logger.trace { "depth frame received for Kinect1 device: $num, at: $timestamp" }
+                    byteBufferRef.set(bytes)
+                }
+            }
+            init {
+                bytes.order(ByteOrder.nativeOrder())
+            }
+            override var enabled: Boolean = false
+                set(value) {
+                    field = if (value) {
+                        start()
+                        true
+                    } else {
+                        stop()
+                        false
+                    }
+                }
+            private fun start() {
+                logger.info { "Enabling Kinect1 depth camera, device num: $num" }
+                verify(freenect_set_depth_mode(
+                        fnDev, freenect_find_depth_mode(FREENECT_RESOLUTION_MEDIUM, FREENECT_DEPTH_11BIT))
+                )
+                verify(freenect_set_depth_buffer(fnDev, Pointer(depthCamera.bytes)))
+                freenect_set_depth_callback(fnDev, depthCamera.freenectDepthCb)
+                verify(freenect_start_depth(fnDev))
+            }
+            private fun stop() {
+                logger.info { "Disabling Kinect1 depth camera, device num: $num" }
+                verify(freenect_stop_depth(fnDev))
+            }
+        }
+    }
+
+    override fun init() {
+        logger.info("Initializing Kinect1 support, set log level to TRACE to see received frames")
+        verify(freenect_init(fnCtx, fnUsbCtx))
+        freenect_set_log_level(fnCtx, FREENECT_LOG_DEBUG)
+        freenect_select_subdevices(fnCtx, FREENECT_DEVICE_CAMERA)
+        val num = verify(freenect_num_devices(fnCtx))
+        if (num == 0) {
+            logger.warn { "Could not find any Kinect1 device, calling startDevice() will throw exception" }
+        }
+        logger.debug("Initializing Kinect1 poller thread")
+        poller.start()
+        // it seems that we have to wait a bit until kinect is actually initialized
+        Thread.sleep(100)
+    }
+
+    override fun countDevices(): Int {
+        return verify(freenect_num_devices(fnCtx))
+    }
+
+    override fun startDevice(num: Int): KinectDevice {
+        val count = countDevices()
+        if (num >= count) {
+            throw KinectException("Non-existent Kinect1 device, num: $num")
+        }
+        val device = KinectV1Device(num)
+        devices.add(device)
+        return device
+    }
+
+    override fun shutdown() {
+        logger.info("Shutting down Kinect1 support")
+        running = false
+        poller.join()
+        devices.forEach {
+            device -> device.shutdown()
+        }
+        if (!fnCtx.isNull) {
+            verifyOnShutdown(freenect_shutdown(fnCtx))
+        }
+    }
+
+    private fun verifyOnShutdown(ret: Int) {
+        if (ret != 0) {
+            logger.error("Unexpected return value while shutting down kinect support: {}", ret)
+        }
+    }
+
+    private fun verify(ret: Int): Int {
+        if (ret < 0) {
+            fail("ret=$ret")
+        }
+        return ret
+    }
+
+    private fun fail(message: String) {
+        throw KinectException("Kinect1 error: $message")
+    }
+
+}

settings.gradle

@@ -14,4 +14,9 @@ include 'orx-camera',
         'orx-no-clear',
         'orx-noise',
         'orx-obj-loader',
-        'orx-olive'
+        'orx-olive',
+        'orx-kinect-common',
+        'orx-kinect-v1',
+        'orx-kinect-v1-natives-linux-x64',
+        'orx-kinect-v1-natives-macos'
+        'orx-kinect-v1-natives-windows'