Upgrade to OPENRNDR 0.4 snapshot

orx-jvm/orx-kinect-common/src/main/kotlin/Kinect.kt

@@ -0,0 +1,105 @@
+package org.openrndr.extra.kinect
+
+import org.openrndr.Extension
+import org.openrndr.draw.*
+import org.openrndr.resourceUrl
+import java.lang.RuntimeException
+
+/**
+ * Represents all the accessible kinects handled by a specific driver (V1, V2).
+ *
+ * @param <CTX> data needed to make low level kinect support calls.
+ */
+interface Kinects<CTX> {
+
+    fun countDevices(): Int
+
+    /**
+     * Starts kinect device of a given number.
+     *
+     * @param num the kinect device index (starts with 0). If no value specified,
+     *          it will default to 0.
+     * @throws KinectException if device of such a number does not exist
+     *          (better to count them first), or it was already started.
+     * @see countDevices
+     */
+    fun startDevice(num: Int = 0): KinectDevice<CTX>
+
+    /**
+     * Executes low level Kinect commands in the kinect thread.
+     */
+    fun <T> execute(commands: (CTX) -> T) : T
+
+}
+
+/**
+ * Represents specific device.
+ *
+ * @param CTX type of data needed to make low level kinect support calls (e.g. freenect contexts).
+ */
+interface KinectDevice<CTX> : Extension {
+
+    val depthCamera: KinectDepthCamera
+
+    /**
+     * Executes low level Kinect commands in the kinect thread in the context of this device.
+     */
+    fun <T> execute(commands: (CTX) -> T): T
+
+}
+
+interface KinectCamera {
+    var enabled: Boolean
+    val width: Int
+    val height: Int
+    var mirror: Boolean
+    val currentFrame: ColorBuffer
+    /**
+     * Returns the latest frame, but only once. Useful for the scenarios
+     * where each new frame triggers extra computation. Therefore the same
+     * expensive operation might happen only once, especially when the refresh
+     * rate of the target screen is higher than kinect's 30 fps.
+     * <p>
+     * Example usage:
+     * <pre>
+     * kinect.depthCamera.getLatestFrame()?.let { frame ->
+     *     grayscaleFilter.apply(frame, grayscaleBuffer)
+     * }
+     * </pre>
+     */
+    fun getLatestFrame(): ColorBuffer?
+}
+
+interface KinectDepthCamera : KinectCamera {
+    /* no special attributes at the moment */
+}
+
+class KinectException(msg: String) : RuntimeException(msg)
+
+/**
+ * Maps depth values to grayscale.
+ */
+class DepthToGrayscaleMapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-grayscale.frag", Kinects::class))
+)
+
+/**
+ * Maps depth values to color map according to natural light dispersion as described
+ * by Alan Zucconi in the
+ * <a href="https://www.alanzucconi.com/2017/07/15/improving-the-rainbow/">Improving the Rainbow</a>
+ * article.
+ */
+class DepthToColorsZucconi6Mapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-colors-zucconi6.frag", Kinects::class))
+)
+
+/**
+ * Maps depth values to color map according to
+ * <a href="https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html">
+ *     Turbo, An Improved Rainbow Colormap for Visualization
+ * </a>
+ * by Google.
+ */
+class DepthToColorsTurboMapper : Filter(
+        filterShaderFromUrl(resourceUrl("depth-to-colors-turbo.frag", Kinects::class))
+)

orx-jvm/orx-kinect-common/src/main/kotlin/org/openrndr/extra/kinect/impl/KinectImpl.kt

@@ -0,0 +1,155 @@
+package org.openrndr.extra.kinect.org.openrndr.extra.kinect.impl
+
+import org.openrndr.Program
+import org.openrndr.draw.*
+import org.openrndr.extra.kinect.KinectDepthCamera
+import org.openrndr.extra.kinect.KinectDevice
+import org.openrndr.extra.kinect.Kinects
+import org.openrndr.math.Vector2
+import org.openrndr.resourceUrl
+import java.nio.ByteBuffer
+import java.util.concurrent.atomic.AtomicReference
+import java.util.function.Supplier
+import kotlin.concurrent.thread
+
+class DefaultKinects<CTX>(
+    private val program: Program,
+    private val manager: KinectsManager<CTX>
+) : Kinects<CTX> {
+
+    private inner class Destroyer : Thread() {
+        override fun run() {
+            manager.shutdown()
+        }
+    }
+
+    init {
+        manager.initialize()
+        // as we don't have explicit shutdown mechanism in OPENRNDR
+        // we need to install a shutdown hook for now
+        Runtime.getRuntime().addShutdownHook(Destroyer())
+    }
+
+    override fun countDevices(): Int {
+        return manager.countDevices()
+    }
+
+    override fun startDevice(num: Int): KinectDevice<CTX> {
+        val device = manager.startDevice(num)
+        program.extend(device)
+        return device
+    }
+
+    override fun <T> execute(commands: (CTX) -> T): T {
+        return manager.execute(commands)
+    }
+
+}
+
+interface KinectsManager<CTX> {
+    fun initialize()
+    fun countDevices(): Int
+    fun startDevice(num: Int): KinectDevice<CTX>
+    fun <T> execute(commands: (CTX) -> T): T
+    fun shutdown()
+}
+
+interface KinectFeatureEnabler {
+    var enabled: Boolean
+}
+
+interface KinectCommandsExecutor<CTX> {
+    fun <T> execute(commands: (CTX) -> T): T
+}
+
+class DefaultKinectDevice<CTX>(
+    override val depthCamera: DefaultKinectDepthCamera,
+    private val commandsExecutor: KinectCommandsExecutor<CTX>
+) : KinectDevice<CTX> {
+    override var enabled: Boolean = true
+    override fun beforeDraw(drawer: Drawer, program: Program) {
+        depthCamera.update()
+    }
+
+    override fun <T> execute(commands: (CTX) -> T): T {
+        return commandsExecutor.execute(commands)
+    }
+}
+
+class DefaultKinectDepthCamera(
+    override val width: Int,
+    override val height: Int,
+    depthScale: Double,
+    private val enabler: KinectFeatureEnabler,
+    private val bytesSupplier: Supplier<ByteBuffer?>
+) :
+    KinectDepthCamera, UpdatableKinectCamera {
+
+    override var enabled: Boolean
+        get() = enabler.enabled
+        set(value) {
+            enabler.enabled = value
+        }
+
+    private val rawBuffer: ColorBuffer = colorBuffer(
+        width,
+        height,
+        format = ColorFormat.R,
+        type = ColorType.UINT16 // it would be perfect if we could use isampler in the shader
+    )
+    override val currentFrame: ColorBuffer = colorBuffer(
+        width,
+        height,
+        format = ColorFormat.R,
+        type = ColorType.FLOAT16 // in the future we might want to choose the precision here
+    )
+
+    private val depthMapper = KinectRawDataToDepthMapper()
+
+    init {
+        depthMapper.depthScale = depthScale
+        depthMapper.mirror = false
+        depthMapper.resolution = Vector2(width.toDouble(), height.toDouble())
+    }
+
+    private val newFrameRef = AtomicReference<ColorBuffer>()
+
+    override fun getLatestFrame(): ColorBuffer? {
+        return newFrameRef.getAndSet(null)
+    }
+
+    override fun update() {
+        if (enabled) {
+            bytesSupplier.get()?.let { bytes ->
+                rawBuffer.write(bytes)
+                depthMapper.apply(rawBuffer, currentFrame)
+                newFrameRef.set(currentFrame)
+            }
+        }
+    }
+
+    override var mirror: Boolean
+        get() = depthMapper.mirror
+        set(value) {
+            depthMapper.mirror = value
+        }
+
+}
+
+private class KinectRawDataToDepthMapper :
+    Filter(
+        filterShaderFromUrl(
+            resourceUrl(
+                "kinect-raw-to-depth.frag",
+                DefaultKinects::class
+            )
+        )
+    ) {
+    var depthScale: Double by parameters
+    var mirror: Boolean by parameters
+    var resolution: Vector2 by parameters
+}
+
+private interface UpdatableKinectCamera {
+    fun update()
+}

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-turbo.frag

@@ -0,0 +1,41 @@
+#version 330
+
+uniform sampler2D tex0;
+out     vec3 color;
+
+float saturate(in float x) {
+    return max(0, min(1, x));
+}
+
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+// Polynomial approximation in GLSL for the Turbo colormap
+// Original LUT: https://gist.github.com/mikhailov-work/ee72ba4191942acecc03fe6da94fc73f
+
+// Authors:
+//   Colormap Design: Anton Mikhailov (mikhailov@google.com)
+//   GLSL Approximation: Ruofei Du (ruofei@google.com)
+
+vec3 TurboColormap(in float x) {
+    const vec4 kRedVec4 = vec4(0.13572138, 4.61539260, -42.66032258, 132.13108234);
+    const vec4 kGreenVec4 = vec4(0.09140261, 2.19418839, 4.84296658, -14.18503333);
+    const vec4 kBlueVec4 = vec4(0.10667330, 12.64194608, -60.58204836, 110.36276771);
+    const vec2 kRedVec2 = vec2(-152.94239396, 59.28637943);
+    const vec2 kGreenVec2 = vec2(4.27729857, 2.82956604);
+    const vec2 kBlueVec2 = vec2(-89.90310912, 27.34824973);
+
+    x = saturate(x);
+    vec4 v4 = vec4( 1.0, x, x * x, x * x * x);
+    vec2 v2 = v4.zw * v4.z;
+    return vec3(
+    dot(v4, kRedVec4)   + dot(v2, kRedVec2),
+    dot(v4, kGreenVec4) + dot(v2, kGreenVec2),
+    dot(v4, kBlueVec4)  + dot(v2, kBlueVec2)
+    );
+}
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : TurboColormap(depth);
+}

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-zucconi6.frag

@@ -0,0 +1,68 @@
+#version 330
+
+uniform sampler2D tex0;             // kinect raw
+out     vec3 color;
+
+// Spectral Colour Schemes
+// By Alan Zucconi
+// Website: www.alanzucconi.com
+// Twitter: @AlanZucconi
+
+// Example of different spectral colour schemes
+// to convert visible wavelengths of light (400-700 nm) to RGB colours.
+
+// The function "spectral_zucconi6" provides the best approximation
+// without including any branching.
+// Its faster version, "spectral_zucconi", is advised for mobile applications.
+
+
+// Read "Improving the Rainbow" for more information
+// http://www.alanzucconi.com/?p=6703
+
+
+
+float saturate (float x)
+{
+    return min(1.0, max(0.0,x));
+}
+vec3 saturate (vec3 x)
+{
+    return min(vec3(1.,1.,1.), max(vec3(0.,0.,0.),x));
+}
+
+// --- Spectral Zucconi --------------------------------------------
+// By Alan Zucconi
+// Based on GPU Gems: https://developer.nvidia.com/sites/all/modules/custom/gpugems/books/GPUGems/gpugems_ch08.html
+// But with values optimised to match as close as possible the visible spectrum
+// Fits this: https://commons.wikimedia.org/wiki/File:Linear_visible_spectrum.svg
+// With weighter MSE (RGB weights: 0.3, 0.59, 0.11)
+vec3 bump3y (vec3 x, vec3 yoffset)
+{
+    vec3 y = vec3(1.,1.,1.) - x * x;
+    y = saturate(y-yoffset);
+    return y;
+}
+
+// --- Spectral Zucconi 6 --------------------------------------------
+
+// Based on GPU Gems
+// Optimised by Alan Zucconi
+vec3 spectral_zucconi6 (float x)
+{
+    const vec3 c1 = vec3(3.54585104, 2.93225262, 2.41593945);
+    const vec3 x1 = vec3(0.69549072, 0.49228336, 0.27699880);
+    const vec3 y1 = vec3(0.02312639, 0.15225084, 0.52607955);
+
+    const vec3 c2 = vec3(3.90307140, 3.21182957, 3.96587128);
+    const vec3 x2 = vec3(0.11748627, 0.86755042, 0.66077860);
+    const vec3 y2 = vec3(0.84897130, 0.88445281, 0.73949448);
+
+    return
+    bump3y(c1 * (x - x1), y1) +
+    bump3y(c2 * (x - x2), y2) ;
+}
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : spectral_zucconi6(depth);
+}

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-grayscale.frag

@@ -0,0 +1,9 @@
+#version 330
+
+uniform sampler2D tex0;
+out     vec3 color;
+
+void main() {
+    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
+    color = (depth >= .999) ? vec3(0) : vec3(depth);
+}

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/impl/kinect-raw-to-depth.frag

@@ -0,0 +1,16 @@
+#version 330
+
+uniform sampler2D tex0;             // kinect raw
+uniform vec2      resolution;       // kinect resolution
+uniform float     depthScale;       // 32 for kinect1, 64 for kinect2
+uniform bool      mirror;
+
+out     float depth;
+
+void main() {
+    ivec2 uv = ivec2(
+        mirror ? int(resolution.x) - 1 - int(gl_FragCoord.x) : int(gl_FragCoord.x),
+        int(resolution.y) - 1 - int(gl_FragCoord.y)
+    );
+    depth = texelFetch(tex0, uv, 0).r * depthScale;
+}