Merge branch 'feature/7/kinect1Support' of https://github.com/morisil/orx into morisil-feature/7/kinect1Support

.gitignore

@@ -6,3 +6,4 @@ build/
 *.iml/
 .idea/
 gradle.properties
+/ShaderError.txt

build.gradle

@@ -10,7 +10,7 @@ buildscript {
 
 plugins {
     // plugin dependencies, load without applying them
-    id 'nebula.kotlin' version '1.3.41' apply false
+    id 'nebula.kotlin' version '1.3.50' apply false
     id 'com.jfrog.artifactory' version '4.6.2' apply false
 
     id 'nebula.contacts' version '4.1.1' apply false
@@ -38,7 +38,7 @@ plugins {
 
 project.ext {
     openrndrVersion = "0.3.35"
-    kotlinVersion = "1.3.41"
+    kotlinVersion = "1.3.50"
     spekVersion = "2.0.6"
     libfreenectVersion = "0.5.7-1.5.1"
 }
@@ -66,16 +66,14 @@ allprojects {
 
     repositories {
         mavenCentral()
+        jcenter()
         maven {
             url = "https://dl.bintray.com/openrndr/openrndr"
         }
 
-
         maven {
             url "https://dl.bintray.com/spekframework/spek"
         }
-
-
     }
 
     dependencies {
@@ -83,11 +81,12 @@ allprojects {
         compile "org.openrndr:openrndr-filter:$openrndrVersion"
         compile "org.openrndr:openrndr-shape:$openrndrVersion"
         compile group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '1.3.0-RC2'
-
         testImplementation "org.spekframework.spek2:spek-dsl-jvm:$spekVersion"
+        testImplementation "org.amshove.kluent:kluent:1.53"
         testImplementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion"
         testRuntimeOnly "org.spekframework.spek2:spek-runner-junit5:$spekVersion"
         testRuntimeOnly "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
+        runtime "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
     }
 
     contacts {

orx-kinect-common/src/main/kotlin/Kinect.kt

@@ -2,6 +2,7 @@ package org.openrndr.extra.kinect
 
 import org.openrndr.Extension
 import org.openrndr.draw.*
+import org.openrndr.resourceUrl
 import java.lang.RuntimeException
 
 /**
@@ -10,13 +11,16 @@ import java.lang.RuntimeException
  * @param <CTX> data needed to make low level kinect support calls.
  */
 interface Kinects<CTX> {
+
     fun countDevices(): Int
 
     /**
      * Starts kinect device of a given number.
      *
-     * @throws KinectException if device of such a number does not exist,
+     * @param num the kinect device index (starts with 0). If no value specified,
-     *          better to count them first.
+     *          it will default to 0.
+     * @throws KinectException if device of such a number does not exist
+     *          (better to count them first), or it was already started.
      * @see countDevices
      */
     fun startDevice(num: Int = 0): KinectDevice<CTX>
@@ -24,21 +28,24 @@ interface Kinects<CTX> {
     /**
      * Executes low level Kinect commands in the kinect thread.
      */
-    fun execute(commands: (CTX) -> Any) : Any
+    fun <T> execute(commands: (CTX) -> T) : T
+
 }
 
 /**
  * Represents specific device.
  *
- * @param <CTX> data needed to make low level kinect support calls.
+ * @param CTX type of data needed to make low level kinect support calls (e.g. freenect contexts).
  */
 interface KinectDevice<CTX> : Extension {
+
     val depthCamera: KinectDepthCamera
 
     /**
      * Executes low level Kinect commands in the kinect thread in the context of this device.
      */
-    fun execute(commands: (CTX) -> Any): Any
+    fun <T> execute(commands: (CTX) -> T): T
+
 }
 
 interface KinectCamera {
@@ -47,6 +54,19 @@ interface KinectCamera {
     val height: Int
     var mirror: Boolean
     val currentFrame: ColorBuffer
+    /**
+     * Returns the latest frame, but only once. Useful for the scenarios
+     * where each new frame triggers extra computation. Therefore the same
+     * expensive operation might happen only once, especially when the refresh
+     * rate of the target screen is higher than kinect's 30 fps.
+     * <p>
+     * Example usage:
+     * <pre>
+     * kinect.depthCamera.getLatestFrame()?.let { frame ->
+     *     grayscaleFilter.apply(frame, grayscaleBuffer)
+     * }
+     * </pre>
+     */
     fun getLatestFrame(): ColorBuffer?
 }
 
@@ -55,3 +75,31 @@ interface KinectDepthCamera : KinectCamera {
 }
 
 class KinectException(msg: String) : RuntimeException(msg)
+
+/**
+ * Maps depth values to grayscale.
+ */
+class DepthToGrayscaleMapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-grayscale.frag", Kinects::class.java))
+)
+
+/**
+ * Maps depth values to color map according to natural light dispersion as described
+ * by Alan Zucconi in the
+ * <a href="https://www.alanzucconi.com/2017/07/15/improving-the-rainbow/">Improving the Rainbow</a>
+ * article.
+ */
+class DepthToColorsZucconi6Mapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-colors-zucconi6.frag", Kinects::class.java))
+)
+
+/**
+ * Maps depth values to color map according to
+ * <a href="https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html">
+ *     Turbo, An Improved Rainbow Colormap for Visualization
+ * </a>
+ * by Google.
+ */
+class DepthToColorsTurboMapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-colors-turbo.frag", Kinects::class.java))
+)

orx-kinect-common/src/main/kotlin/org/openrndr/extra/kinect/impl/KinectImpl.kt

@@ -1,7 +1,10 @@
-package org.openrndr.extra.kinect
+package org.openrndr.extra.kinect.impl
 
 import org.openrndr.Program
 import org.openrndr.draw.*
+import org.openrndr.extra.kinect.KinectDepthCamera
+import org.openrndr.extra.kinect.KinectDevice
+import org.openrndr.extra.kinect.Kinects
 import org.openrndr.math.Vector2
 import org.openrndr.resourceUrl
 import java.nio.ByteBuffer
@@ -9,8 +12,11 @@ import java.util.concurrent.atomic.AtomicReference
 import java.util.function.Supplier
 import kotlin.concurrent.thread
 
+class DefaultKinects<CTX>(
+        private val program: Program,
+        private val manager: KinectsManager<CTX>
+) : Kinects<CTX> {
 
-class DefaultKinects<CTX>(private val manager: KinectsManager<CTX>) : Kinects<CTX> {
     init {
         manager.initialize()
         // as we don't have explicit shutdown mechanism in OPENRNDR
@@ -31,10 +37,12 @@ class DefaultKinects<CTX>(private val manager: KinectsManager<CTX>) : Kinects<CT
     }
 
     override fun startDevice(num: Int): KinectDevice<CTX> {
-        return manager.startDevice(num)
+        val device = manager.startDevice(num)
+        program.extend(device)
+        return device
     }
 
-    override fun execute(commands: (CTX) -> Any): Any {
+    override fun <T> execute(commands: (CTX) -> T): T {
         return manager.execute(commands)
     }
 
@@ -44,7 +52,7 @@ interface KinectsManager<CTX> {
     fun initialize()
     fun countDevices(): Int
     fun startDevice(num: Int): KinectDevice<CTX>
-    fun execute(commands: (CTX) -> Any): Any
+    fun <T> execute(commands: (CTX) -> T): T
     fun shutdown()
 }
 
@@ -53,7 +61,7 @@ interface KinectFeatureEnabler {
 }
 
 interface KinectCommandsExecutor<CTX> {
-    fun execute(commands: (CTX) -> Any): Any
+    fun <T> execute(commands: (CTX) -> T): T
 }
 
 class DefaultKinectDevice<CTX>(
@@ -64,7 +72,8 @@ class DefaultKinectDevice<CTX>(
     override fun beforeDraw(drawer: Drawer, program: Program) {
         depthCamera.update()
     }
-    override fun execute(commands: (CTX) -> Any): Any {
+
+    override fun <T> execute(commands: (CTX) -> T): T {
         return commandsExecutor.execute(commands)
     }
 }
@@ -128,7 +137,10 @@ class DefaultKinectDepthCamera(
 }
 
 private class KinectRawDataToDepthMapper :
-        Filter(filterShaderFromUrl(resourceUrl("kinect-raw-to-depth.frag", Kinects::class.java))) {
+        Filter(filterShaderFromUrl(
+                resourceUrl("kinect-raw-to-depth.frag",
+                DefaultKinects::class.java))
+        ) {
     var depthScale: Double by parameters
     var mirror: Boolean by parameters
     var resolution: Vector2 by parameters

orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-turbo.frag

@@ -0,0 +1,41 @@
+#version 330
+
+uniform sampler2D tex0;
+out     vec3 color;
+
+float saturate(in float x) {
+    return max(0, min(1, x));
+}
+
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+// Polynomial approximation in GLSL for the Turbo colormap
+// Original LUT: https://gist.github.com/mikhailov-work/ee72ba4191942acecc03fe6da94fc73f
+
+// Authors:
+//   Colormap Design: Anton Mikhailov (mikhailov@google.com)
+//   GLSL Approximation: Ruofei Du (ruofei@google.com)
+
+vec3 TurboColormap(in float x) {
+    const vec4 kRedVec4 = vec4(0.13572138, 4.61539260, -42.66032258, 132.13108234);
+    const vec4 kGreenVec4 = vec4(0.09140261, 2.19418839, 4.84296658, -14.18503333);
+    const vec4 kBlueVec4 = vec4(0.10667330, 12.64194608, -60.58204836, 110.36276771);
+    const vec2 kRedVec2 = vec2(-152.94239396, 59.28637943);
+    const vec2 kGreenVec2 = vec2(4.27729857, 2.82956604);
+    const vec2 kBlueVec2 = vec2(-89.90310912, 27.34824973);
+
+    x = saturate(x);
+    vec4 v4 = vec4( 1.0, x, x * x, x * x * x);
+    vec2 v2 = v4.zw * v4.z;
+    return vec3(
+    dot(v4, kRedVec4)   + dot(v2, kRedVec2),
+    dot(v4, kGreenVec4) + dot(v2, kGreenVec2),
+    dot(v4, kBlueVec4)  + dot(v2, kBlueVec2)
+    );
+}
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : TurboColormap(depth);
+}

orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-zucconi6.frag

@@ -0,0 +1,68 @@
+#version 330
+
+uniform sampler2D tex0;             // kinect raw
+out     vec3 color;
+
+// Spectral Colour Schemes
+// By Alan Zucconi
+// Website: www.alanzucconi.com
+// Twitter: @AlanZucconi
+
+// Example of different spectral colour schemes
+// to convert visible wavelengths of light (400-700 nm) to RGB colours.
+
+// The function "spectral_zucconi6" provides the best approximation
+// without including any branching.
+// Its faster version, "spectral_zucconi", is advised for mobile applications.
+
+
+// Read "Improving the Rainbow" for more information
+// http://www.alanzucconi.com/?p=6703
+
+
+
+float saturate (float x)
+{
+    return min(1.0, max(0.0,x));
+}
+vec3 saturate (vec3 x)
+{
+    return min(vec3(1.,1.,1.), max(vec3(0.,0.,0.),x));
+}
+
+// --- Spectral Zucconi --------------------------------------------
+// By Alan Zucconi
+// Based on GPU Gems: https://developer.nvidia.com/sites/all/modules/custom/gpugems/books/GPUGems/gpugems_ch08.html
+// But with values optimised to match as close as possible the visible spectrum
+// Fits this: https://commons.wikimedia.org/wiki/File:Linear_visible_spectrum.svg
+// With weighter MSE (RGB weights: 0.3, 0.59, 0.11)
+vec3 bump3y (vec3 x, vec3 yoffset)
+{
+    vec3 y = vec3(1.,1.,1.) - x * x;
+    y = saturate(y-yoffset);
+    return y;
+}
+
+// --- Spectral Zucconi 6 --------------------------------------------
+
+// Based on GPU Gems
+// Optimised by Alan Zucconi
+vec3 spectral_zucconi6 (float x)
+{
+    const vec3 c1 = vec3(3.54585104, 2.93225262, 2.41593945);
+    const vec3 x1 = vec3(0.69549072, 0.49228336, 0.27699880);
+    const vec3 y1 = vec3(0.02312639, 0.15225084, 0.52607955);
+
+    const vec3 c2 = vec3(3.90307140, 3.21182957, 3.96587128);
+    const vec3 x2 = vec3(0.11748627, 0.86755042, 0.66077860);
+    const vec3 y2 = vec3(0.84897130, 0.88445281, 0.73949448);
+
+    return
+    bump3y(c1 * (x - x1), y1) +
+    bump3y(c2 * (x - x2), y2) ;
+}
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : spectral_zucconi6(depth);
+}

orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-grayscale.frag

@@ -0,0 +1,9 @@
+#version 330
+
+uniform sampler2D tex0;
+out     vec3 color;
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : vec3(depth);
+}

orx-kinect-v1-demo/build.gradle

@@ -0,0 +1,13 @@
+def os = org.gradle.internal.os.OperatingSystem.current()
+def openrndrOs
+if      (os.windows) { openrndrOs = "windows" }
+else if (os.macOsX)  { openrndrOs = "macos" }
+else if (os.linux)   { openrndrOs = "linux-x64" }
+
+dependencies {
+    compile project(":orx-kinect-v1")
+    runtime project(":orx-kinect-v1-natives-$openrndrOs")
+    runtime "org.openrndr:openrndr-gl3:$openrndrVersion"
+    runtime "org.openrndr:openrndr-gl3-natives-$openrndrOs:$openrndrVersion"
+    runtime "ch.qos.logback:logback-classic:1.2.3"
+}

orx-kinect-v1-demo/src/main/kotlin/BasicUseCaseDemo.kt

@@ -0,0 +1,26 @@
+package org.openrndr.extra.kinect.v1.demo
+
+import org.openrndr.application
+import org.openrndr.extra.kinect.v1.getKinectsV1
+
+/**
+ * Basic kinect use case showing continuous stream from the depth camera.
+ *
+ * Note: kinect depth map is stored only on the RED color channel to save
+ *       space. Therefore depth map is displayed only in the red tones.
+ */
+fun main() = application {
+    configure { // default resolution of the Kinect v1 depth camera
+        width = 640
+        height = 480
+    }
+    program {
+        val kinects = getKinectsV1(this)
+        val kinect = kinects.startDevice()
+        kinect.depthCamera.enabled = true
+        kinect.depthCamera.mirror = true
+        extend {
+            drawer.image(kinect.depthCamera.currentFrame)
+        }
+    }
+}

orx-kinect-v1-demo/src/main/kotlin/DepthToColorMapsDemo.kt

@@ -0,0 +1,73 @@
+package org.openrndr.extra.kinect.v1.demo
+
+import org.openrndr.application
+import org.openrndr.draw.ColorBuffer
+import org.openrndr.draw.ColorFormat
+import org.openrndr.draw.colorBuffer
+import org.openrndr.extra.kinect.*
+import org.openrndr.extra.kinect.v1.getKinectsV1
+
+/**
+ * Shows 4 different representations of the depth map.
+ * <ol>
+ *     <li>the original depth map stored as RED channel values</li>
+ *     <li>the same values expressed as gray tones</li>
+ *     <li>
+ *         color map according to natural light dispersion as described
+ *         by Alan Zucconi in the
+ *         <a href="https://www.alanzucconi.com/2017/07/15/improving-the-rainbow/">Improving the Rainbow</a>
+ *         article.
+ *     </li>
+ *     <li>
+ *         color map according to
+ *         <a href="https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html">
+ *             Turbo, An Improved Rainbow Colormap for Visualization
+ *         </a>
+ *         by Google.
+ *     </li>
+ * </ol>
+ *
+ * @see DepthToGrayscaleMapper
+ * @see DepthToColorsZucconi6Mapper
+ * @see DepthToColorsTurboMapper
+ */
+fun main() = application {
+    configure {
+        width =  2 * 640
+        height = 2 * 480
+    }
+    program {
+        val kinects = getKinectsV1(this)
+        val kinect = kinects.startDevice()
+        kinect.depthCamera.enabled = true
+        kinect.depthCamera.mirror = true
+        val camera = kinect.depthCamera
+        val grayscaleFilter = DepthToGrayscaleMapper()
+        val zucconiFilter = DepthToColorsZucconi6Mapper()
+        val turboFilter = DepthToColorsTurboMapper()
+        val grayscaleBuffer = kinectColorBuffer(camera)
+        val zucconiBuffer = kinectColorBuffer(camera)
+        val turboBuffer = kinectColorBuffer(camera)
+        extend {
+            /*
+             * Note: getting the latest frame this way will guarantee
+             * that filters are being applied only if the actual new frame
+             * from kinect was received. Kinect has different refresh rate
+             * than usual screen (30 fps).
+             */
+            kinect.depthCamera.getLatestFrame()?.let { frame ->
+                grayscaleFilter.apply(frame, grayscaleBuffer)
+                zucconiFilter.apply(frame, zucconiBuffer)
+                turboFilter.apply(frame, turboBuffer)
+            }
+            drawer.image(camera.currentFrame)
+            drawer.image(grayscaleBuffer, camera.width.toDouble(), 0.0)
+            drawer.image(turboBuffer, 0.0, camera.height.toDouble())
+            drawer.image(zucconiBuffer, camera.width.toDouble(), camera.height.toDouble())
+        }
+    }
+}
+
+private fun kinectColorBuffer(camera: KinectCamera): ColorBuffer {
+    return colorBuffer(camera.width, camera.height, format = ColorFormat.RGB)
+}

orx-kinect-v1-demo/src/main/kotlin/MultipleKinectsDemo.kt

@@ -0,0 +1,33 @@
+package org.openrndr.extra.kinect.v1.demo
+
+import org.openrndr.application
+import org.openrndr.extra.kinect.v1.getKinectsV1
+
+/**
+ * Stream from 2 kinects side by side.
+ */
+fun main() = application {
+    configure {
+        width  = 640 * 2
+        height = 480
+    }
+    program {
+        val kinects = getKinectsV1(this)
+        val depthCamera1 = kinects.startDevice(0).depthCamera
+        val depthCamera2 = kinects.startDevice(1).depthCamera
+        depthCamera1.enabled = true
+        depthCamera1.mirror = true
+        depthCamera2.enabled = true
+        depthCamera2.mirror = true
+        extend {
+            drawer.image(depthCamera1.currentFrame)
+            drawer.image(depthCamera2.currentFrame, depthCamera1.width.toDouble(), 0.0)
+        }
+        keyboard.keyDown.listen { keyEvent ->
+            if (keyEvent.name == "1") {depthCamera1.enabled = !depthCamera1.enabled }
+            if (keyEvent.name == "2") {depthCamera2.enabled = !depthCamera2.enabled }
+            if (keyEvent.name == "q") {depthCamera1.mirror = !depthCamera1.mirror }
+            if (keyEvent.name == "w") {depthCamera2.mirror = !depthCamera2.mirror }
+        }
+    }
+}

orx-kinect-v1-demo/src/main/kotlin/NativeFreenectCommandsDemo.kt

@@ -0,0 +1,43 @@
+package org.openrndr.extra.kinect.v1.demo
+
+import org.bytedeco.libfreenect.global.freenect
+import org.bytedeco.libfreenect.global.freenect.*
+import org.openrndr.application
+import org.openrndr.extra.kinect.v1.getKinectsV1
+import java.lang.RuntimeException
+
+/**
+ * Even though this library is abstracting freenect access, it is still
+ * possible to call any low level kinect API through execute methods.
+ * The calls are executed in separate kinect runner thread but they will
+ * block the calling thread until the result is returned.
+ */
+fun main() = application {
+    program {
+        val kinects = getKinectsV1(this)
+        // the same as calling kinects.countDevices(), here to show that any value might be returned from execute
+        val num = kinects.execute { ctx -> freenect_num_devices(ctx.fnCtx) }
+        if (num == 0) { throw RuntimeException("no kinect detected") }
+        kinects.execute { ctx ->
+            freenect_set_log_level(ctx.fnCtx, freenect.FREENECT_LOG_FLOOD) // lots of logs
+        }
+        kinects.execute { ctx ->
+            // extra FREENECT_DEVICE_MOTOR gives control over tilt and LEDs
+            freenect_select_subdevices(ctx.fnCtx, FREENECT_DEVICE_CAMERA xor FREENECT_DEVICE_MOTOR)
+        }
+        val kinect = kinects.startDevice()
+        var tilt = 90.0
+        extend {
+            kinect.execute { ctx ->
+                freenect_set_led(ctx.fnDev, (seconds * 10).toInt() % 7) // disco
+            }
+            val currentTilt = if ((seconds % 10) < 5) -90.0 else 90.0
+            if (currentTilt != tilt) {
+                kinect.execute { ctx ->
+                    freenect_set_tilt_degs(ctx.fnDev, currentTilt)
+                }
+                tilt = currentTilt
+            }
+        }
+    }
+}

orx-kinect-v1-demo/src/main/resources/logback.xml

@@ -0,0 +1,13 @@
+<configuration debug="false" scan="false" scanPeriod="10 seconds">
+  <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+    <resetJUL>true</resetJUL>
+  </contextListener>
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+  <root level="DEBUG">
+    <appender-ref ref="STDOUT"/>
+  </root>
+</configuration>

orx-kinect-v1/src/main/kotlin/KinectV1.kt

@@ -5,7 +5,9 @@ import org.bytedeco.javacpp.Pointer
 import org.bytedeco.libfreenect.*
 import org.bytedeco.libfreenect.global.freenect.*
 import org.bytedeco.libfreenect.presets.freenect
+import org.openrndr.Program
 import org.openrndr.extra.kinect.*
+import org.openrndr.extra.kinect.impl.*
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
 import java.util.*
@@ -13,6 +15,7 @@ import java.util.concurrent.*
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 import java.util.function.Supplier
+import kotlin.concurrent.thread
 
 /**
  * Returns support for kinect version 1.
@@ -25,8 +28,8 @@ import java.util.function.Supplier
  *          kinect. Subsequent runs of the same program don't require
  *          this delay at all.
  */
-fun getKinectsV1(depthCameraInitializationDelay: Long = 100) : Kinects<Freenect> {
+fun getKinectsV1(program: Program, depthCameraInitializationDelay: Long = 100) : Kinects<Freenect> {
-    return DefaultKinects(KinectsV1Manager(depthCameraInitializationDelay))
+    return DefaultKinects(program, KinectsV1Manager(depthCameraInitializationDelay))
 }
 
 /** Provides low level freenect context for calling native freenect methods. */
@@ -46,23 +49,33 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
 
     private val ctx = Freenect(fnCtx, fnUsbCtx)
 
+    private var taskQueue = LinkedBlockingDeque<FutureTask<*>>()
+
     private var running = true
 
+    private val runner = thread(
+            name = "Kinect1-runner",
+            start = false,
+            isDaemon = true
+    ) {
+        initializeFreenect()
+        while (running) { mainLoop() }
+        shutdownFreenect()
+    }
+
+    private var expectingEvents = false
+
     private val devices: LinkedList<FreenectDevice> = LinkedList()
 
     private val timeout = freenect.timeval()
     init { timeout.tv_sec(1) }
 
-    private val executor = Executors.newSingleThreadExecutor{
+    private inner class KinectV1CommandsExecutor(val context: Freenect): KinectCommandsExecutor<Freenect> {
-        runnable -> Thread(runnable, "Kinect1-runner")
+        override fun <T> execute(commands: (Freenect) -> T): T {
-    }
+            return callSync {
-
+                logger.trace { "executing native freenect commands" }
-    private inner class KinectV1CommandsExecutor(val context: Freenect) : KinectCommandsExecutor<Freenect> {
-        override fun execute(commands: (Freenect) -> Any): Any {
-            return executor.submit(Callable {
-                logger.debug { "Executing native freenect commands" }
                 commands(context)
-            }).get()
+            }
         }
     }
 
@@ -70,49 +83,65 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
 
     override fun initialize() {
         logger.info("Initializing Kinect1 support, set log level to TRACE to see received frames")
-        executor.execute {
+        runner.start()
-            logger.debug("Initializing freenect")
+    }
-            verify(freenect_init(fnCtx, fnUsbCtx))
+
-            freenect_set_log_level(fnCtx, FREENECT_LOG_INFO)
+    private fun initializeFreenect() {
-            freenect_select_subdevices(fnCtx, FREENECT_DEVICE_CAMERA)
+        logger.debug("initializing freenect")
-            val num = verify(freenect_num_devices(fnCtx))
+        verify(freenect_init(fnCtx, fnUsbCtx))
-            if (num == 0) {
+        freenect_set_log_level(fnCtx, FREENECT_LOG_INFO)
-                logger.warn { "Could not find any Kinect1 device, calling startDevice() will throw exception" }
+        freenect_select_subdevices(fnCtx, FREENECT_DEVICE_CAMERA)
-            }
+        val num = verify(freenect_num_devices(fnCtx))
+        if (num == 0) {
+            logger.warn { "Could not find any Kinect1 device, calling startDevice() will throw exception" }
         }
-        executor.execute(object : Runnable {
+    }
-            override fun run() {
+
-                if (!running) { return }
+    private fun mainLoop() {
-                val ret = freenect_process_events_timeout(fnCtx, timeout)
+        if (expectingEvents) {
-                if (ret != 0) {
+            val ret = freenect_process_events(fnCtx)
-                    logger.error { "freenect_process_events_timeout returned non-zero value: $ret" }
+            if (ret != 0) { logger.error { "freenect_process_events returned non-zero value: $ret" } }
-                }
+            val tasks = taskQueue.iterator()
-                executor.execute(this) // loop
+            for (task in tasks) {
+                tasks.remove()
+                task.run()
             }
-        })
+        } else {
+            taskQueue.poll(100, TimeUnit.MILLISECONDS)?.run()
+        }
+    }
+
+    private fun shutdownFreenect() {
+        logger.debug("shutting down freenect")
+        if (!fnCtx.isNull) {
+            devices.forEach { device -> device.shutdown() }
+            devices.clear()
+            verifyOnShutdown(freenect_shutdown(fnCtx))
+        }
     }
 
     override fun countDevices(): Int {
-        return executor.submit(
+        return callSync { verify(freenect_num_devices(fnCtx)) }
-            Callable { verify(freenect_num_devices(fnCtx)) }
-        ).get()
     }
 
-    // FIXME we should prevent from starting the same device multiple times
     override fun startDevice(num: Int): KinectDevice<Freenect> {
+        callSync {
+            devices.find { device -> device.num == num }
+        }?.let {
+            throw KinectException("Kinect1 device already started, num: $num")
+        }
         val count = countDevices()
         if (num >= count) {
             throw KinectException(
-                    "Trying to start non-existent Kinect1 device, device count: $count, num: $num"
+                    "Trying to start non-existent Kinect1 device, " +
+                            "device count: $count, num: $num (index starts with 0)"
             )
         }
-        val device = executor.submit(
+        val device = callSync {
-            Callable {
+            val device = FreenectDevice(num)
-                val device = FreenectDevice(num)
+            devices.add(device)
-                devices.add(device)
+            device
-                device
+        }
-            }
-        ).get()
         return DefaultKinectDevice(
                 DefaultKinectDepthCamera(
                         device.depthCamera.width,
@@ -125,32 +154,38 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
         )
     }
 
-    override fun execute(commands: (Freenect) -> Any): Any {
+    override fun <T> execute(commands: (Freenect) -> T): T {
         return commandsExecutor.execute(commands)
     }
 
     override fun shutdown() {
         logger.info("Shutting down Kinect1 support")
-        executor.submit { running = false }.get()
+        callSync { running = false }
-        executor.submit {
+        runner.join()
-            if (!fnCtx.isNull) {
-                devices.forEach { device -> device.shutdown() }
-                devices.clear()
-            }
-        }.get() // wait to finish
-        executor.shutdown()
-        executor.awaitTermination(1100, TimeUnit.MILLISECONDS)
-        // value slightly higher than 1sec polling timeout, just in case
     }
 
-    private inner class FreenectDevice(private val num: Int) {
+    private inline fun <T> callSync(crossinline block: () -> T): T {
+        val task = FutureTask<T>(Callable { block() })
+        taskQueue.add(task)
+        return task.get()
+    }
+
+    private inner class FreenectDevice(val num: Int) {
+
         val depthCamera = FreenectDepthCamera()
+
         val fnDev = freenect_device()
+
         val devCtx = Freenect(fnCtx, fnUsbCtx, fnDev)
+
         init {
             logger.info { "Opening Kinect1 device num: $num" }
             verify(freenect_open_device(fnCtx, fnDev, num))
         }
+
+        val expectingEvents: Boolean
+            get() = depthCamera.expectingEvents // or other device in the future
+
         fun shutdown() {
             logger.info { "Shutting down Kinect1 device num: $num" }
             if (!fnDev.isNull) {
@@ -158,23 +193,32 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
                 verifyOnShutdown(freenect_close_device(fnDev))
             }
         }
+
         inner class FreenectDepthCamera {
+
             val width: Int = 640
             val height: Int = 480
+
             private val bytes = ByteBuffer.allocateDirect(width * height * 2)
-            private val currentBytesRef = AtomicReference<ByteBuffer?>()
             init { bytes.order(ByteOrder.nativeOrder()) }
+
+            private val currentBytesRef = AtomicReference<ByteBuffer?>()
+
             private val freenectDepthCb = object : freenect_depth_cb() {
                 override fun call(dev: freenect_device?, depth: Pointer?, timestamp: Int) {
                     logger.trace { "depth frame received for Kinect1 device: $num, at: $timestamp" }
                     currentBytesRef.set(bytes)
                 }
             }
+
             val bytesSupplier = Supplier<ByteBuffer?> { currentBytesRef.getAndSet(null) }
+
             val enabler = object : KinectFeatureEnabler {
+
                 private val atomicEnabled = AtomicBoolean(false)
                 private val inProgress = AtomicBoolean(false)
-                override var enabled
+
+                override var enabled // usually called from rendering thread
                     get() = atomicEnabled.get()
                     set(value) {
                         if (atomicEnabled.get() == value) {
@@ -183,29 +227,31 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
                         }
                         if (!inProgress.getAndSet(true)) {
                             if (value) {
-                                executor.execute {
+                                callSync {
                                     try {
                                         start()
-                                    } finally {
+                                        atomicEnabled.set(true)
-                                        inProgress.set(false)
+                                        updateExpectingEvents()
-                                    }
+                                    } finally { inProgress.set(false) }
                                 }
-                                atomicEnabled.set(true)
                             } else {
-                                executor.execute {
+                                callSync {
                                     try {
                                         stop()
-                                    } finally {
+                                        atomicEnabled.set(false)
-                                        inProgress.set(false)
+                                        updateExpectingEvents()
-                                    }
+                                    } finally { inProgress.set(false) }
                                 }
-                                atomicEnabled.set(false)
                             }
                         } else {
                             logger.warn { "Operation in progress, Kinect1 device: $num, requested enabled=$value" }
                         }
                     }
             }
+
+            val expectingEvents: Boolean
+                get() = depthCamera.enabler.enabled
+
             private fun start() {
                 logger.info { "Enabling Kinect1 depth camera, device num: $num" }
                 verify(freenect_set_depth_mode(
@@ -216,6 +262,7 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
                 Thread.sleep(depthCameraInitializationDelay) // here is the hack
                 freenect_set_depth_callback(fnDev, freenectDepthCb)
             }
+
             private fun stop() {
                 logger.info { "Disabling Kinect1 depth camera, device num: $num" }
                 verify(freenect_stop_depth(fnDev))
@@ -223,6 +270,10 @@ private class KinectsV1Manager(val depthCameraInitializationDelay: Long) : Kinec
         }
     }
 
+    private fun updateExpectingEvents() {
+        expectingEvents = devices.any { device -> device.expectingEvents }
+    }
+
     private fun verifyOnShutdown(ret: Int) {
         if (ret != 0) {
             logger.error { "Unexpected return value while shutting down Kinect1 support: $ret" }

orx-noise/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    implementation project(":orx-shader-phrases")
+}

orx-noise/src/main/kotlin/phrases/NoisePhrases.kt

@@ -0,0 +1,32 @@
+@file:ShaderPhrases(exports = ["hash22","hash21","valueNoise21"])
+package org.openrndr.extra.noise.phrases
+
+import org.openrndr.extra.shaderphrases.annotations.ShaderPhrase
+import org.openrndr.extra.shaderphrases.annotations.ShaderPhrases
+
+@ShaderPhrase(exports = ["hash22"])
+val phraseHash22 = """vec2 hash22(vec2 p) {
+    float n = sin(dot(p, vec2(41, 289)));
+    return fract(vec2(262144, 32768)*n);
+}    
+"""
+
+@ShaderPhrase(exports = ["hash21"])
+val phraseHash21 = "float hash21(vec2 p) { return fract(1e4 * sin(17.0 * p.x + p.y * 0.1) * (0.1 + abs(sin(p.y * 13.0 + p.x)))); }"
+
+@ShaderPhrase(exports = ["valueNoise21"], imports = ["hash21"])
+val phraseValueNoise21 = """
+    
+float noise(vec2 x) {
+    vec2 i = floor(x);
+    vec2 f = fract(x);
+
+    float a = hash21(i);
+    float b = hash21(i + vec2(1.0, 0.0));
+    float c = hash21(i + vec2(0.0, 1.0));
+    float d = hash21(i + vec2(1.0, 1.0));
+
+    vec2 u = f * f * (3.0 - 2.0 * f);
+    return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y;
+}   
+""".trimIndent()

orx-shader-phrases/src/main/kotlin/ShaderPreprocessor.kt

@@ -0,0 +1,33 @@
+package org.openrndr.extra.shaderphrases
+
+import org.openrndr.extra.shaderphrases.annotations.ShaderPhrases
+
+fun preprocessShader(shader: String): String {
+    val lines = shader.split("\n")
+    val processed = lines.map {
+        if (it.startsWith("import ")) {
+            val tokens = it.split(" ")
+            val full = tokens[1]
+            val fullTokens = full.split(".")
+            val fieldName = fullTokens.last().replace(";","")
+            val packageClassTokens = fullTokens.dropLast(1)
+            val packageClass = packageClassTokens.joinToString(".")
+
+            val c = Class.forName(packageClass)
+            if (c.annotations.any { it.annotationClass == ShaderPhrases::class }) {
+                if (fieldName == "*") {
+                    c.declaredFields.filter { println(it.type); it.type.name =="java.lang.String" }.map {
+                        it.get(null)
+                    }.joinToString("\n")
+                } else {
+                    c.getDeclaredField(fieldName).get(null)
+                }
+            } else {
+                throw IllegalArgumentException("class $packageClass has no ShaderPhrases annotation")
+            }
+        } else {
+            it
+        }
+    }
+    return processed.joinToString("\n")
+}

orx-shader-phrases/src/main/kotlin/annotations/ShaderPhrase.kt

@@ -0,0 +1,15 @@
+package org.openrndr.extra.shaderphrases.annotations
+
+enum class ShaderPhraseLanguage {
+    GLSL
+}
+
+@Retention(AnnotationRetention.RUNTIME)
+@Target(AnnotationTarget.FILE, AnnotationTarget.CLASS, AnnotationTarget.FIELD)
+annotation class ShaderPhrase(val exports: Array<String>,
+                              val imports: Array<String> = emptyArray(),
+                              val language: ShaderPhraseLanguage = ShaderPhraseLanguage.GLSL)
+
+
+@Target(AnnotationTarget.FILE)
+annotation class ShaderPhrases(val exports: Array<String>)

orx-shader-phrases/src/main/kotlin/phrases/Dummy.kt

@@ -0,0 +1,24 @@
+@file:JvmName("Dummy")
+@file:ShaderPhrases(["dummy"])
+
+package org.openrndr.extra.shaderphrases.phrases
+import org.openrndr.extra.shaderphrases.annotations.ShaderPhrase
+import org.openrndr.extra.shaderphrases.annotations.ShaderPhrases
+
+@ShaderPhrase(["dummy"])
+const val phraseDummy = """
+float dummy() {
+    return 0.0;    
+}
+"""
+
+
+fun main() {
+    val c = Class.forName("org.openrndr.extra.shaderphrases.phrases.Dummy")
+
+    if (c.annotations.any { it.annotationClass == ShaderPhrases::class }) {
+        println(c.getDeclaredField("phraseDummy").get(null))
+
+
+    }
+}

orx-shader-phrases/src/test/kotlin/TestPreprocessShader.kt

@@ -0,0 +1,19 @@
+import org.openrndr.extra.shaderphrases.preprocessShader
+import org.spekframework.spek2.Spek
+import org.spekframework.spek2.style.specification.describe
+
+object TestPreprocessShader:Spek({
+
+    describe("A shader with import statements") {
+        val shader = """
+#version 330
+import org.openrndr.extra.shaderphrases.phrases.Dummy.*
+
+
+""".trimIndent()
+        describe("when preprocessed") {
+            val processed = preprocessShader(shader)
+            println(processed)
+        }
+    }
+})

settings.gradle

@@ -15,8 +15,11 @@ include 'orx-camera',
         'orx-noise',
         'orx-obj-loader',
         'orx-olive',
+        'orx-shader-phrases',
+        'orx-shader-phrases-processor',
         'orx-kinect-common',
         'orx-kinect-v1',
         'orx-kinect-v1-natives-linux-x64',
-        'orx-kinect-v1-natives-macos'
+        'orx-kinect-v1-natives-macos',
-        'orx-kinect-v1-natives-windows'
+        'orx-kinect-v1-natives-windows',
+        'orx-kinect-v1-demo'