Add skinning support to orx-dnk3a

build.gradle

@@ -14,7 +14,7 @@ buildscript {
 apply plugin: 'org.jetbrains.dokka'
 
 project.ext {
-    openrndrVersion = "0.3.43-rc.1"
+    openrndrVersion = "0.3.43-rc.2"
     kotlinVersion = "1.3.72"
     spekVersion = "2.0.10"
     libfreenectVersion = "0.5.7-1.5.3"

demo-data/gltf-models/box/Box.gltf

@@ -1,142 +0,0 @@
-{
-  "asset": {
-    "generator": "COLLADA2GLTF",
-    "version": "2.0"
-  },
-  "scene": 0,
-  "scenes": [
-    {
-      "nodes": [
-        0
-      ]
-    }
-  ],
-  "nodes": [
-    {
-      "children": [
-        1
-      ],
-      "matrix": [
-        1.0,
-        0.0,
-        0.0,
-        0.0,
-        0.0,
-        0.0,
-        -1.0,
-        0.0,
-        0.0,
-        1.0,
-        0.0,
-        0.0,
-        0.0,
-        0.0,
-        0.0,
-        1.0
-      ]
-    },
-    {
-      "mesh": 0
-    }
-  ],
-  "meshes": [
-    {
-      "primitives": [
-        {
-          "attributes": {
-            "NORMAL": 1,
-            "POSITION": 2
-          },
-          "indices": 0,
-          "mode": 5,
-          "material": 0
-        }
-      ],
-      "name": "Mesh"
-    }
-  ],
-  "accessors": [
-    {
-      "bufferView": 0,
-      "byteOffset": 0,
-      "componentType": 5123,
-      "count": 36,
-      "max": [
-        23
-      ],
-      "min": [
-        0
-      ],
-      "type": "SCALAR"
-    },
-    {
-      "bufferView": 1,
-      "byteOffset": 0,
-      "componentType": 5126,
-      "count": 24,
-      "max": [
-        1.0,
-        1.0,
-        1.0
-      ],
-      "min": [
-        -1.0,
-        -1.0,
-        -1.0
-      ],
-      "type": "VEC3"
-    },
-    {
-      "bufferView": 1,
-      "byteOffset": 288,
-      "componentType": 5126,
-      "count": 24,
-      "max": [
-        0.5,
-        0.5,
-        0.5
-      ],
-      "min": [
-        -0.5,
-        -0.5,
-        -0.5
-      ],
-      "type": "VEC3"
-    }
-  ],
-  "materials": [
-    {
-      "pbrMetallicRoughness": {
-        "baseColorFactor": [
-          0.800000011920929,
-          0.0,
-          0.0,
-          1.0
-        ],
-        "metallicFactor": 0.0
-      },
-      "name": "Red"
-    }
-  ],
-  "bufferViews": [
-    {
-      "buffer": 0,
-      "byteOffset": 576,
-      "byteLength": 72,
-      "target": 34963
-    },
-    {
-      "buffer": 0,
-      "byteOffset": 0,
-      "byteLength": 576,
-      "byteStride": 12,
-      "target": 34962
-    }
-  ],
-  "buffers": [
-    {
-      "byteLength": 648,
-      "uri": "Box0.bin"
-    }
-  ]
-}

orx-dnk3/README.md

@@ -12,10 +12,10 @@ Supported Gltf features
   - [x] Basic materials
   - [x] Normal maps
   - [x] Metallic/roughness maps
-  - [ ] Skinning
+  - [x] Skinning
   - [x] Double-sided materials
   - [ ] Transparency
-- [ ] Animations 
+- [x] Animations 
 - [ ] Cameras
 - [ ] Lights
 <!-- __demos__ -->

orx-dnk3/build.gradle

@@ -12,9 +12,11 @@ dependencies {
     implementation "com.google.code.gson:gson:$gsonVersion"
     implementation(project(":orx-fx"))
     implementation(project(":orx-keyframer"))
+
     demoImplementation(project(":orx-camera"))
     demoImplementation("org.openrndr:openrndr-core:$openrndrVersion")
     demoImplementation("org.openrndr:openrndr-extensions:$openrndrVersion")
+    demoImplementation("org.openrndr:openrndr-ffmpeg:$openrndrVersion")
     demoRuntimeOnly("org.openrndr:openrndr-gl3:$openrndrVersion")
     demoRuntimeOnly("org.openrndr:openrndr-gl3-natives-$openrndrOS:$openrndrVersion")
     demoImplementation(sourceSets.getByName("main").output)

orx-dnk3/src/demo/kotlin/DemoScene02.kt

@@ -25,7 +25,7 @@ fun main() = application {
             }
         }
 
-        val gltf = loadGltfFromFile(File("demo-data/gltf-models/complex02/scene.gltf"))
+        val gltf = loadGltfFromFile(File("demo-data/gltf-models/duck/Duck.gltf"))
 //        val gltf = loadGltfFromGlbFile(File("demo-data/gltf-models/splash-sss.glb"))
         val scene = Scene(SceneNode())
 
@@ -48,8 +48,8 @@ fun main() = application {
         val renderer = dryRenderer()
         extend(Orbital()) {
             far = 500.0
-            lookAt = Vector3(0.0, 0.7, 0.0)
+            lookAt = Vector3(0.0, 0.8, 0.0)
-            eye = Vector3(3.0, 0.7, -2.0)
+            eye = Vector3(3.0, 0.8, -2.0)
             fov = 30.0
         }
         extend {

orx-dnk3/src/demo/kotlin/DemoSkinning01.kt

@@ -0,0 +1,52 @@
+import org.openrndr.application
+import org.openrndr.color.ColorRGBa
+import org.openrndr.extensions.SingleScreenshot
+import org.openrndr.extra.dnk3.*
+import org.openrndr.extra.dnk3.gltf.buildSceneNodes
+import org.openrndr.extra.dnk3.gltf.loadGltfFromFile
+import org.openrndr.extras.camera.Orbital
+import org.openrndr.math.Vector3
+import org.openrndr.math.mod_
+import java.io.File
+
+fun main() = application {
+    configure {
+        width = 1280
+        height = 720
+        //multisample = WindowMultisample.SampleCount(8)
+    }
+
+    program {
+        if (System.getProperty("takeScreenshot") == "true") {
+            extend(SingleScreenshot()) {
+                this.outputFile = System.getProperty("screenshotPath")
+            }
+        }
+
+        val gltf = loadGltfFromFile(File("demo-data/gltf-models/fox/Fox.glb"))
+        val scene = Scene(SceneNode())
+
+        scene.root.entities.add(HemisphereLight().apply {
+            upColor = ColorRGBa.WHITE.shade(0.4)
+            downColor = ColorRGBa.GRAY.shade(0.1)
+        })
+        val sceneData = gltf.buildSceneNodes()
+        scene.root.children.addAll(sceneData.scenes.first())
+
+
+        // -- create a renderer
+        val renderer = dryRenderer()
+        extend(Orbital()) {
+            far = 500.0
+            lookAt = Vector3(0.0, 40.0, 0.0)
+            eye = Vector3(150.0, 40.0, 200.0)
+            fov = 40.0
+        }
+
+        extend {
+            sceneData.animations[2].applyToTargets(seconds.mod_(sceneData.animations[2].duration))
+            drawer.clear(ColorRGBa.PINK)
+            renderer.draw(drawer, scene)
+        }
+    }
+}

orx-dnk3/src/main/kotlin/Entity.kt

@@ -2,24 +2,47 @@ package org.openrndr.extra.dnk3
 
 import org.openrndr.color.ColorRGBa
 import org.openrndr.draw.*
+import org.openrndr.math.Matrix44
 
 
 class Geometry(val vertexBuffers: List<VertexBuffer>,
                val indexBuffer: IndexBuffer?,
                val primitive: DrawPrimitive,
                val offset: Int,
-               val vertexCount: Int)
+               val vertexCount: Int) {
+
+    override fun toString(): String {
+        return "Geometry(vertexBuffers: $vertexBuffers, indexBuffers: $indexBuffer, primitive: $primitive, offset: $offset, vertexCount: $vertexCount)"
+    }
+
+}
 
 val DummyGeometry = Geometry(emptyList(), null, DrawPrimitive.TRIANGLES, 0, 0)
 
 sealed class Entity
 
-class MeshPrimitive(var geometry: Geometry, var material: Material)
+class MeshPrimitive(var geometry: Geometry, var material: Material) {
+
+    override fun toString(): String {
+        return "MeshPrimitive(geometry: $geometry, material: $material)"
+    }
+
+}
 
 class MeshPrimitiveInstance(val primitive: MeshPrimitive, val instances: Int, val attributes: List<VertexBuffer>)
 
 abstract class MeshBase(var primitives: List<MeshPrimitive>) : Entity()
-class Mesh(primitives: List<MeshPrimitive>) : MeshBase(primitives)
+class Mesh(primitives: List<MeshPrimitive>) : MeshBase(primitives) {
+    override fun toString(): String {
+        return "Mesh(primitives: $primitives)"
+    }
+}
+
+class SkinnedMesh(primitives: List<MeshPrimitive>,
+                  val joints: List<SceneNode>,
+                  val skeleton: SceneNode,
+                  val inverseBindMatrices: List<Matrix44>
+) : MeshBase(primitives)
 
 class InstancedMesh(primitives: List<MeshPrimitive>,
                     var instances: Int,

orx-dnk3/src/main/kotlin/Facet.kt

@@ -113,10 +113,9 @@ class ClipPositionFacet : ColorBufferFacetCombiner(setOf(FacetType.CLIP_POSITION
 
 class LDRColorFacet : ColorBufferFacetCombiner(setOf(FacetType.DIFFUSE, FacetType.SPECULAR), "color", ColorFormat.RGBa, ColorType.UINT8) {
     override fun generateShader() = """
-    vec3 oofinalColor =  (f_diffuse.rgb + f_specular.rgb + f_emission.rgb) * (1.0 - f_fog.a) + f_fog.rgb * f_fog.a;
+    vec3 finalColor =  (max(vec3(0.0), f_diffuse.rgb) + max(vec3(0.0),f_specular.rgb) + max(vec3(0.0), f_emission.rgb)) * (1.0 - f_fog.a) + f_fog.rgb * f_fog.a;
-    o_$targetOutput = pow(vec4(oofinalColor.rgb, 1.0), vec4(1.0/2.2));
+    o_$targetOutput = pow(vec4(finalColor.rgb, 1.0), vec4(1.0/2.2));
-    o_$targetOutput *= m_color.a; 
+    o_$targetOutput *= m_color.a;
-    
     
     """
 }

orx-dnk3/src/main/kotlin/Material.kt

@@ -8,7 +8,7 @@ import org.openrndr.draw.shadeStyle
 interface Material {
     var doubleSided: Boolean
     var transparent: Boolean
-    fun generateShadeStyle(context: MaterialContext): ShadeStyle
+    fun generateShadeStyle(context: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle
     fun applyToShadeStyle(context: MaterialContext, shadeStyle: ShadeStyle)
 }
 
@@ -17,7 +17,7 @@ class DummyMaterial : Material {
     override var transparent: Boolean = false
 
 
-    override fun generateShadeStyle(context: MaterialContext): ShadeStyle {
+    override fun generateShadeStyle(context: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle {
         return shadeStyle {
             fragmentTransform = """
                 x_fill.rgb = vec3(normalize(v_viewNormal).z);
@@ -38,3 +38,7 @@ data class MaterialContext(val pass: RenderPass,
                            val meshCubemaps: Map<Mesh, Cubemap>
 )
 
+data class PrimitiveContext(val hasNormalAttribute: Boolean, val hasSkinning: Boolean)
+
+
+data class ContextKey(val materialContext: MaterialContext, val primitiveContext: PrimitiveContext)

orx-dnk3/src/main/kotlin/PBRMaterial.kt

@@ -7,6 +7,7 @@ import org.openrndr.math.Vector3
 import org.openrndr.math.Vector4
 import org.openrndr.math.transforms.normalMatrix
 import java.nio.ByteBuffer
+import javax.naming.Context
 import kotlin.math.cos
 
 
@@ -29,7 +30,7 @@ private val noise128 by lazy {
     cb
 }
 
-private fun PointLight.fs(index: Int): String = """
+private fun PointLight.fs(index: Int, hasNormalAttribute: Boolean): String = """
 |{
 |   vec3 Lr = p_lightPosition$index - v_worldPosition;
 |   float distance = length(Lr);
@@ -37,7 +38,7 @@ private fun PointLight.fs(index: Int): String = """
 |   p_lightLinearAttenuation$index * distance + p_lightQuadraticAttenuation$index * distance * distance);
 |   vec3 L = normalize(Lr);
 |
-|   float side = dot(L, N) ;
+|   float side = ${if (hasNormalAttribute) "dot(L, N)" else "3.1415"};
 |   f_diffuse += attenuation * max(0, side / 3.1415) * p_lightColor$index.rgb * m_color.rgb;
 |   f_specular += attenuation * ggx(N, V, L, m_roughness, m_f0) * p_lightColor$index.rgb * m_color.rgb;
 }
@@ -45,14 +46,14 @@ private fun PointLight.fs(index: Int): String = """
 
 private fun AmbientLight.fs(index: Int): String = "f_ambient += p_lightColor$index.rgb * ((1.0 - m_metalness) * m_color.rgb);"
 
-private fun DirectionalLight.fs(index: Int) = """
+private fun DirectionalLight.fs(index: Int, hasNormalAttribute: Boolean) = """
 |{
 |    vec3 L = normalize(-p_lightDirection$index);
 |    float attenuation = 1.0;
 |    vec3 H = normalize(V + L);
-|    float NoL = clamp(dot(N, L), 0.0, 1.0);
+|    float NoL = ${if (hasNormalAttribute) "clamp(dot(N, L), 0.0, 1.0)" else "1"};
 |    float LoH = clamp(dot(L, H), 0.0, 1.0);
-|    float NoH = clamp(dot(N, H), 0.0, 1.0);
+|    float NoH = ${if (hasNormalAttribute) "clamp(dot(N, H), 0.0, 1.0)" else "1"};
 |    vec3 Lr = (p_lightPosition$index - v_worldPosition);
 //|    vec3 L = normalize(Lr);
 |    ${shadows.fs(index)}
@@ -66,15 +67,15 @@ private fun DirectionalLight.fs(index: Int) = """
 |}
 """.trimMargin()
 
-private fun HemisphereLight.fs(index: Int): String = """
+private fun HemisphereLight.fs(index: Int, hasNormalAttribute: Boolean): String = """
 |{
-|   float f = dot(N, p_lightDirection$index) * 0.5 + 0.5;
+|   float f = ${if (hasNormalAttribute) "dot(N, p_lightDirection$index) * 0.5 + 0.5" else "1"};
 |   vec3 irr = ${irradianceMap?.let { "texture(p_lightIrradianceMap$index, N).rgb" } ?: "vec3(1.0)"};
 |   f_diffuse += mix(p_lightDownColor$index.rgb, p_lightUpColor$index.rgb, f) * irr * ((1.0 - m_metalness) * m_color.rgb) * m_ambientOcclusion;
 |}
 """.trimMargin()
 
-private fun SpotLight.fs(index: Int): String {
+private fun SpotLight.fs(index: Int, hasNormalAttribute: Boolean): String {
     val shadows = shadows
     return """
 |{
@@ -85,7 +86,7 @@ private fun SpotLight.fs(index: Int): String {
 |   attenuation = 1.0;
 |   vec3 L = normalize(Lr);
 
-|   float NoL = clamp(dot(N, L), 0.0, 1.0);
+|   float NoL = ${if (hasNormalAttribute) "clamp(dot(N, L), 0.0, 1.0)" else "1"};
 |   float side = dot(L, N);
 |   float hit = max(dot(-L, p_lightDirection$index), 0.0);
 |   float falloff = clamp((hit - p_lightOuterCos$index) / (p_lightInnerCos$index - p_lightOuterCos$index), 0.0, 1.0);
@@ -94,7 +95,7 @@ private fun SpotLight.fs(index: Int): String {
 |   {
 |       vec3 H = normalize(V + L);
 |       float LoH = clamp(dot(L, H), 0.0, 1.0);
-|       float NoH = clamp(dot(N, H), 0.0, 1.0);
+|       float NoH = ${if (hasNormalAttribute) "clamp(dot(N, H), 0.0, 1.0)" else 1.0};
 |       f_diffuse += NoL * (0.1+0.9*attenuation) * Fd_Burley(m_roughness * m_roughness, NoV, NoL, LoH) * p_lightColor$index.rgb * m_color.rgb ;
 |       float Dg = D_GGX(m_roughness * m_roughness, NoH, H);
 |       float Vs = V_SmithGGXCorrelated(m_roughness * m_roughness, NoV, NoL);
@@ -114,7 +115,12 @@ private fun Fog.fs(index: Int): String = """
 """.trimMargin()
 
 sealed class TextureSource
-object DummySource : TextureSource()
+object DummySource : TextureSource() {
+    override fun toString(): String {
+        return "DummySource()"
+    }
+}
+
 abstract class TextureFromColorBuffer(var texture: ColorBuffer, var textureFunction: TextureFunction) : TextureSource()
 
 class TextureFromCode(val code: String) : TextureSource()
@@ -142,10 +148,14 @@ enum class TextureFunction(val function: (String, String) -> String) {
  */
 class ModelCoordinates(texture: ColorBuffer,
                        var input: String = "va_texCoord0.xy",
-                       var tangentInput : String? = null,
+                       var tangentInput: String? = null,
                        textureFunction: TextureFunction = TextureFunction.TILING,
                        var pre: String? = null,
-                       var post: String? = null) : TextureFromColorBuffer(texture, textureFunction)
+                       var post: String? = null) : TextureFromColorBuffer(texture, textureFunction) {
+    override fun toString(): String {
+        return "ModelCoordinates(texture: $texture, input: $input, $tangentInput: $tangentInput, textureFunction: $textureFunction, pre: $pre, post: $post)"
+    }
+}
 
 class Triplanar(texture: ColorBuffer,
                 var scale: Double = 1.0,
@@ -171,7 +181,8 @@ private fun ModelCoordinates.fs(index: Int) = """
 |   ${if (pre != null) "{ $pre } " else ""}
 |   x_texture = ${textureFunction.function("p_texture$index", "x_texCoord")};
 |   ${if (post != null) "{ $post } " else ""}
-|   ${if (tangentInput != null) { """
+|   ${if (tangentInput != null) {
+    """
 |        vec3 normal = normalize(va_normal.xyz);
 |        vec3 tangent = normalize(${tangentInput}.xyz);
 |        vec3 bitangent = cross(normal, tangent) * ${tangentInput}.w;
@@ -179,7 +190,7 @@ private fun ModelCoordinates.fs(index: Int) = """
 |        x_texture.rgb = tbn * normalize(x_texture.rgb - vec3(0.5, 0.5, 0.)) ;
 |    
 """.trimMargin()
-    
+
 } else ""}   
 |   tex$index = x_texture;
 |}
@@ -225,16 +236,20 @@ private fun Triplanar.fs(index: Int, target: TextureTarget) = """
     """.trimIndent() else ""}
 """.trimMargin()
 
-sealed class TextureTarget {
+sealed class TextureTarget(val name: String) {
-    object NONE : TextureTarget()
+    object NONE : TextureTarget("NONE")
-    object COLOR : TextureTarget()
+    object COLOR : TextureTarget("COLOR")
-    object ROUGHNESS : TextureTarget()
+    object ROUGHNESS : TextureTarget("ROUGHNESS")
-    object METALNESS : TextureTarget()
+    object METALNESS : TextureTarget("METALNESS")
-    object METALNESS_ROUGHNESS : TextureTarget()
+    object METALNESS_ROUGHNESS : TextureTarget("METALNESS_ROUGHNESS")
-    object EMISSION : TextureTarget()
+    object EMISSION : TextureTarget("EMISSION")
-    object NORMAL : TextureTarget()
+    object NORMAL : TextureTarget("NORMAL")
-    object AMBIENT_OCCLUSION : TextureTarget()
+    object AMBIENT_OCCLUSION : TextureTarget("AMBIENT_OCCLUSION")
-    class Height(var scale: Double = 1.0) : TextureTarget()
+    class Height(var scale: Double = 1.0) : TextureTarget("Height")
+
+    override fun toString(): String {
+        return "TextureTarget(name: $name)"
+    }
 }
 
 class Texture(var source: TextureSource,
@@ -243,9 +258,17 @@ class Texture(var source: TextureSource,
         val copied = Texture(source, target)
         return copied
     }
+
+    override fun toString(): String {
+        return "Texture(source: $source, target: $target)"
+    }
 }
 
 class PBRMaterial : Material {
+    override fun toString(): String {
+        return "PBRMaterial(textures: $textures, color: $color, metalness: $metalness, roughness: $roughness, emissive: $emission))"
+    }
+
     override var doubleSided: Boolean = false
     override var transparent: Boolean = false
     var environmentMap = false
@@ -259,7 +282,7 @@ class PBRMaterial : Material {
     var parameters = mutableMapOf<String, Any>()
     var textures = mutableListOf<Texture>()
 
-    val shadeStyles = mutableMapOf<MaterialContext, ShadeStyle>()
+    val shadeStyles = mutableMapOf<ContextKey, ShadeStyle>()
 
 //    fun copy(): PBRMaterial {
 //        val copied = PBRMaterial()
@@ -276,9 +299,9 @@ class PBRMaterial : Material {
 //        return copied
 //    }
 
-    override fun generateShadeStyle(context: MaterialContext): ShadeStyle {
+    override fun generateShadeStyle(materialContext: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle {
-        val cached = shadeStyles.getOrPut(context) {
+        val cached = shadeStyles.getOrPut(ContextKey(materialContext, primitiveContext)) {
-            val needLight = needLight(context)
+            val needLight = needLight(materialContext)
             val preambleFS = """
             vec4 m_color = p_color;
             float m_f0 = 0.5;
@@ -317,6 +340,20 @@ class PBRMaterial : Material {
 
             val displacers = textures.filter { it.target is TextureTarget.Height }
 
+            val skinVS = if (primitiveContext.hasSkinning) """
+                    uvec4 j = a_joints;
+                    mat4 skinTransform = p_jointTransforms[j.x] * a_weights.x 
+                    + p_jointTransforms[j.y] * a_weights.y 
+                    + p_jointTransforms[j.z] * a_weights.z 
+                    + p_jointTransforms[j.w] * a_weights.w;                    
+
+                    ${if (primitiveContext.hasNormalAttribute) """
+                    x_normal = normalize(mat3(skinTransform) * x_normal);                        
+                    """.trimIndent() else ""}
+                    
+                    x_position = (skinTransform * vec4(x_position,1)).xyz;
+            """.trimIndent() else ""
+
             val textureVS = if (displacers.isNotEmpty()) textures.mapIndexed { index, it ->
                 if (it.target is TextureTarget.Height) {
                     when (val source = it.source) {
@@ -331,7 +368,7 @@ class PBRMaterial : Material {
                 } else ""
             }.joinToString("\n") else ""
 
-            val lights = context.lights
+            val lights = materialContext.lights
             val lightFS = if (needLight) """
         vec3 f_diffuse = vec3(0.0);
         vec3 f_specular = vec3(0.0);
@@ -342,9 +379,9 @@ class PBRMaterial : Material {
         vec3 ep = (p_viewMatrixInverse * vec4(0.0, 0.0, 0.0, 1.0)).xyz;
         vec3 Vr = ep - v_worldPosition;
         vec3 V = normalize(Vr);
-        float NoV = abs(dot(N, V)) + 1e-5;
+        float NoV = ${if (primitiveContext.hasNormalAttribute) "abs(dot(N, V)) + 1e-5" else "1"};
 
-        ${if (environmentMap && context.meshCubemaps.isNotEmpty()) """
+        ${if (environmentMap && materialContext.meshCubemaps.isNotEmpty() && primitiveContext.hasNormalAttribute) """
            {
                 vec2 dfg = PrefilteredDFG_Karis(m_roughness, NoV);
                 vec3 sc = m_metalness * m_color.rgb + (1.0-m_metalness) * vec3(0.04);
@@ -356,32 +393,32 @@ class PBRMaterial : Material {
         ${lights.mapIndexed { index, (node, light) ->
                 when (light) {
                     is AmbientLight -> light.fs(index)
-                    is PointLight -> light.fs(index)
+                    is PointLight -> light.fs(index, primitiveContext.hasNormalAttribute)
-                    is SpotLight -> light.fs(index)
+                    is SpotLight -> light.fs(index, primitiveContext.hasNormalAttribute)
-                    is DirectionalLight -> light.fs(index)
+                    is DirectionalLight -> light.fs(index, primitiveContext.hasNormalAttribute)
-                    is HemisphereLight -> light.fs(index)
+                    is HemisphereLight -> light.fs(index, primitiveContext.hasNormalAttribute)
                     else -> TODO()
                 }
             }.joinToString("\n")}
 
-        ${context.fogs.mapIndexed { index, (node, fog) ->
+        ${materialContext.fogs.mapIndexed { index, (node, fog) ->
                 fog.fs(index)
             }.joinToString("\n")}
 
     """.trimIndent() else ""
             val rt = RenderTarget.active
 
-            val combinerFS = context.pass.combiners.map {
+            val combinerFS = materialContext.pass.combiners.map {
                 it.generateShader()
             }.joinToString("\n")
 
             val fs = preambleFS + textureFs + lightFS + combinerFS
-            val vs = (this@PBRMaterial.vertexTransform ?: "") + textureVS
+            val vs = (this@PBRMaterial.vertexTransform ?: "") + textureVS + skinVS
 
             shadeStyle {
                 vertexPreamble = """
                     $shaderNoRepetitionVert
-                     ${(this@PBRMaterial.vertexPreamble)?:""}
+                     ${(this@PBRMaterial.vertexPreamble) ?: ""}
                 """.trimIndent()
                 fragmentPreamble = """
             |$shaderLinePlaneIntersect
@@ -394,7 +431,7 @@ class PBRMaterial : Material {
                 this.suppressDefaultOutput = true
                 this.vertexTransform = vs
                 fragmentTransform = fs
-                context.pass.combiners.map {
+                materialContext.pass.combiners.map {
                     if (rt.colorBuffers.size <= 1) {
                         this.output(it.targetOutput, 0)
                     } else

orx-dnk3/src/main/kotlin/SceneRenderer.kt

@@ -5,6 +5,7 @@ import org.openrndr.draw.*
 import org.openrndr.draw.depthBuffer
 import org.openrndr.extra.fx.blur.ApproximateGaussianBlur
 import org.openrndr.math.Matrix44
+import org.openrndr.math.transforms.normalMatrix
 
 class SceneRenderer {
 
@@ -48,10 +49,11 @@ class SceneRenderer {
 
         val lights = scene.root.findContent { this as? Light }
         val meshes = scene.root.findContent { this as? Mesh }
+        val skinnedMeshes = scene.root.findContent { this as? SkinnedMesh }
+
         val fogs = scene.root.findContent { this as? Fog }
         val instancedMeshes = scene.root.findContent { this as? InstancedMesh }
 
-
         run {
             lights.filter { it.content is ShadowLight && (it.content as ShadowLight).shadows is Shadows.MappedShadows }.forEach {
                 val shadowLight = it.content as ShadowLight
@@ -80,7 +82,7 @@ class SceneRenderer {
 
                     drawer.clear(ColorRGBa.BLACK)
                     drawer.cullTestPass = CullTestPass.FRONT
-                    drawPass(drawer, pass, materialContext, meshes, instancedMeshes)
+                    drawPass(drawer, pass, materialContext, meshes, instancedMeshes, skinnedMeshes)
                 }
                 when (shadowLight.shadows) {
                     is Shadows.VSM -> {
@@ -99,7 +101,6 @@ class SceneRenderer {
             for (pass in outputPasses) {
                 val materialContext = MaterialContext(pass, lights, fogs, shadowLightTargets, meshCubemaps)
 
-
                 val defaultPasses = setOf(DefaultTransparentPass, DefaultOpaquePass)
 
                 if ((pass !in defaultPasses || postSteps.isNotEmpty()) && outputPassTarget == null) {
@@ -109,7 +110,7 @@ class SceneRenderer {
                 if (pass == outputPasses[0]) {
                     outputPassTarget?.let {
                         drawer.withTarget(it) {
-                            background(ColorRGBa.PINK)
+                            clear(ColorRGBa.PINK)
                         }
                     }
                 }
@@ -122,7 +123,7 @@ class SceneRenderer {
                     }
                 }
                 outputPassTarget?.bind()
-                drawPass(drawer, pass, materialContext, meshes, instancedMeshes)
+                drawPass(drawer, pass, materialContext, meshes, instancedMeshes, skinnedMeshes)
                 outputPassTarget?.unbind()
 
                 outputPassTarget?.let { output ->
@@ -167,7 +168,9 @@ class SceneRenderer {
 
     private fun drawPass(drawer: Drawer, pass: RenderPass, materialContext: MaterialContext,
                          meshes: List<NodeContent<Mesh>>,
-                         instancedMeshes: List<NodeContent<InstancedMesh>>) {
+                         instancedMeshes: List<NodeContent<InstancedMesh>>,
+                         skinnedMeshes: List<NodeContent<SkinnedMesh>>
+    ) {
 
         drawer.depthWrite = pass.depthWrite
         val primitives = meshes.flatMap { mesh ->
@@ -185,8 +188,9 @@ class SceneRenderer {
                         if (primitive.material.doubleSided) {
                             drawer.drawStyle.cullTestPass = CullTestPass.ALWAYS
                         }
-
+                        val hasNormalAttribute = primitive.geometry.vertexBuffers.any { it.vertexFormat.hasAttribute("normal") }
-                        val shadeStyle = primitive.material.generateShadeStyle(materialContext)
+                        val primitiveContext = PrimitiveContext(hasNormalAttribute, false)
+                        val shadeStyle = primitive.material.generateShadeStyle(materialContext, primitiveContext)
                         shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed)
                         primitive.material.applyToShadeStyle(materialContext, shadeStyle)
                         drawer.shadeStyle = shadeStyle
@@ -207,6 +211,62 @@ class SceneRenderer {
                     }
                 }
 
+
+        val skinnedPrimitives = skinnedMeshes.flatMap { mesh ->
+            mesh.content.primitives.map { primitive ->
+                NodeContent(mesh.node, Pair(primitive, mesh))
+            }
+        }
+
+        skinnedPrimitives
+                .filter {
+                    (it.content.first.material.transparent && pass.renderTransparent) ||
+                            (!it.content.first.material.transparent && pass.renderOpaque)
+                }
+                .forEach {
+                    val primitive = it.content.first
+                    val skinnedMesh = it.content.second.content
+                    drawer.isolated {
+                        if (primitive.material.doubleSided) {
+                            drawer.drawStyle.cullTestPass = CullTestPass.ALWAYS
+                        }
+                        val hasNormalAttribute = primitive.geometry.vertexBuffers.any { it.vertexFormat.hasAttribute("normal") }
+                        val primitiveContext = PrimitiveContext(hasNormalAttribute, true)
+
+                        val nodeInverse = it.node.worldTransform.inversed
+
+
+
+                        val jointTransforms = (skinnedMesh.joints zip skinnedMesh.inverseBindMatrices)
+                                .map{ (nodeInverse * it.first.worldTransform * it.second) }
+//                        val jointNormalTransforms = jointTransforms.map { Matrix44.IDENTITY }
+
+                        val shadeStyle = primitive.material.generateShadeStyle(materialContext, primitiveContext)
+
+                        shadeStyle.parameter("jointTransforms", jointTransforms.toTypedArray())
+//                        shadeStyle.parameter("jointNormalTransforms", jointNormalTransforms.toTypedArray())
+
+                        shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed)
+                        primitive.material.applyToShadeStyle(materialContext, shadeStyle)
+                        drawer.shadeStyle = shadeStyle
+                        drawer.model = it.node.worldTransform
+
+                        if (primitive.geometry.indexBuffer == null) {
+                            drawer.vertexBuffer(primitive.geometry.vertexBuffers,
+                                    primitive.geometry.primitive,
+                                    primitive.geometry.offset,
+                                    primitive.geometry.vertexCount)
+                        } else {
+                            drawer.vertexBuffer(primitive.geometry.indexBuffer!!,
+                                    primitive.geometry.vertexBuffers,
+                                    primitive.geometry.primitive,
+                                    primitive.geometry.offset,
+                                    primitive.geometry.vertexCount)
+                        }
+                    }
+                }
+
+
         val instancedPrimitives = instancedMeshes.flatMap { mesh ->
             mesh.content.primitives.map { primitive ->
                 NodeContent(mesh.node, MeshPrimitiveInstance(primitive, mesh.content.instances, mesh.content.attributes))
@@ -219,7 +279,8 @@ class SceneRenderer {
                 .forEach {
                     val primitive = it.content
                     drawer.isolated {
-                        val shadeStyle = primitive.primitive.material.generateShadeStyle(materialContext)
+                        val primitiveContext = PrimitiveContext(true,  false)
+                        val shadeStyle = primitive.primitive.material.generateShadeStyle(materialContext, primitiveContext)
                         shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed)
                         primitive.primitive.material.applyToShadeStyle(materialContext, shadeStyle)
                         if (primitive.primitive.material.doubleSided) {

orx-dnk3/src/main/kotlin/gltf/Glb.kt

@@ -28,6 +28,7 @@ fun loadGltfFromGlbFile(file: File): GltfFile {
                 if (chunkType == 0x004E4942) ByteBuffer.allocateDirect(chunkLength) else ByteBuffer.allocate(chunkLength)
         (chunkBuffer as ByteBuffer)
         channel.read(chunkBuffer)
+        chunkBuffer.order(ByteOrder.nativeOrder())
         return chunkBuffer
     }
 

orx-dnk3/src/main/kotlin/gltf/Gltf.kt

@@ -33,7 +33,8 @@ class GltfNode(val children: IntArray?,
                val scale: DoubleArray?,
                val rotation: DoubleArray?,
                val translation: DoubleArray?,
-               val mesh: Int?)
+               val mesh: Int?,
+               val skin: Int?)
 
 class GltfPrimitive(val attributes: LinkedHashMap<String, Int>, val indices: Int?, val mode: Int?, val material: Int) {
     fun createDrawCommand(gltfFile: GltfFile): GltfDrawCommand {
@@ -87,6 +88,23 @@ class GltfPrimitive(val attributes: LinkedHashMap<String, Int>, val indices: Int
                         textureCoordinate(dimensions, 0)
                         accessors.add(accessor)
                     }
+                    "JOINTS_0" -> {
+                        val type = when (Pair(accessor.type, accessor.componentType)) {
+                            Pair("VEC4", GLTF_UNSIGNED_BYTE) -> VertexElementType.VECTOR4_UINT8
+                            Pair("VEC4", GLTF_UNSIGNED_SHORT) -> VertexElementType.VECTOR4_UINT16
+                            else -> error("not supported ${accessor.type} / ${accessor.componentType}")
+                        }
+                        attribute("joints", type)
+                        accessors.add(accessor)
+                    }
+                    "WEIGHTS_0" -> {
+                        val type = when (Pair(accessor.type, accessor.componentType)) {
+                            Pair("VEC4", GLTF_FLOAT) -> VertexElementType.VECTOR4_FLOAT32
+                            else -> error("not supported ${accessor.type} / ${accessor.componentType}")
+                        }
+                        attribute("weights", type)
+                        accessors.add(accessor)
+                    }
                 }
             }
         }
@@ -181,7 +199,6 @@ class GltfBufferView(val buffer: Int,
 
 class GltfBuffer(val byteLength: Int, val uri: String?) {
     fun contents(gltfFile: GltfFile): ByteBuffer = if (uri != null) {
-
         if (uri.startsWith("data:")) {
             val base64 = uri.substring(uri.indexOf(",") + 1)
             val decoded = Base64.getDecoder().decode(base64)
@@ -223,6 +240,9 @@ class GltfChannelTarget(val node: Int?, val path: String?)
 
 class GltfChannel(val sampler: Int, val target: GltfChannelTarget)
 
+
+class GltfSkin(val inverseBindMatrices: Int, val joints: IntArray, val skeleton: Int)
+
 class GltfFile(
         val asset: GltfAsset?,
         val scene: Int?,
@@ -236,7 +256,9 @@ class GltfFile(
         val images: List<GltfImage>?,
         val textures: List<GltfTexture>?,
         val samplers: List<GltfSampler>?,
-        val animations: List<GltfAnimation>?) {
+        val animations: List<GltfAnimation>?,
+        val skins: List<GltfSkin>?
+) {
     @Transient
     lateinit var file: File
 

orx-dnk3/src/main/kotlin/gltf/GltfScene.kt

@@ -10,10 +10,18 @@ import org.openrndr.math.Quaternion
 import org.openrndr.math.Vector3
 import org.openrndr.math.transforms.transform
 import java.io.File
+import java.nio.Buffer
 import java.nio.ByteOrder
 import kotlin.reflect.KMutableProperty0
 
 class SceneAnimation(var channels: List<AnimationChannel>) {
+
+    val duration: Double
+    get() {
+        return channels.maxBy { it.duration }?.duration ?:0.0
+    }
+
+
     fun applyToTargets(input: Double) {
         for (channel in channels) {
             channel.applyToTarget(input)
@@ -22,6 +30,7 @@ class SceneAnimation(var channels: List<AnimationChannel>) {
 }
 
 sealed class AnimationChannel {
+    abstract val duration: Double
     abstract fun applyToTarget(input: Double)
 }
 
@@ -30,6 +39,9 @@ class QuaternionChannel(val target: KMutableProperty0<Quaternion>,
     override fun applyToTarget(input: Double) {
         target.set(keyframer.value(input) ?: Quaternion.IDENTITY)
     }
+
+    override val duration: Double
+        get() = keyframer.duration()
 }
 
 class Vector3Channel(val target: KMutableProperty0<Vector3>,
@@ -37,6 +49,8 @@ class Vector3Channel(val target: KMutableProperty0<Vector3>,
     override fun applyToTarget(input: Double) {
         target.set(keyframer.value(input) ?: default)
     }
+    override val duration: Double
+        get() = keyframer.duration()
 }
 
 class GltfSceneNode : SceneNode() {
@@ -44,6 +58,11 @@ class GltfSceneNode : SceneNode() {
     var scale = Vector3.ONE
     var rotation = Quaternion.IDENTITY
 
+    override fun toString(): String {
+        return "translation: $translation, scale: $scale, rotation: $rotation, children: ${children.size}, entities: ${entities} "
+    }
+
+
     override var transform: Matrix44 = Matrix44.IDENTITY
         get() = transform {
             translate(translation)
@@ -93,6 +112,12 @@ fun GltfFile.buildSceneNodes(): GltfSceneData {
         pbrMetallicRoughness?.let { pbr ->
             material.roughness = pbr.roughnessFactor ?: 1.0
             material.metalness = pbr.metallicFactor ?: 1.0
+
+            material.color = ColorRGBa.WHITE
+            pbr.baseColorFactor?.let {
+                material.color = ColorRGBa(it[0], it[1], it[2], it[3])
+            }
+
             pbr.baseColorTexture?.let { texture ->
                 val cb = images!![textures!![texture.index].source].createSceneImage()
                 cb.filter(MinifyingFilter.LINEAR_MIPMAP_LINEAR, MagnifyingFilter.LINEAR)
@@ -181,32 +206,10 @@ fun GltfFile.buildSceneNodes(): GltfSceneData {
         return MeshPrimitive(geometry, material)
     }
 
-    val sceneMeshes = mutableMapOf<GltfMesh, Mesh>()
-    fun GltfMesh.createSceneMesh(): Mesh = sceneMeshes.getOrPut(this) {
-        Mesh(primitives.map {
-            it.createScenePrimitive()
-        })
-    }
 
     val sceneNodes = mutableMapOf<GltfNode, SceneNode>()
     fun GltfNode.createSceneNode(): SceneNode = sceneNodes.getOrPut(this) {
         val node = GltfSceneNode()
-        mesh?.let {
-            node.entities.add(meshes[it].createSceneMesh())
-        }
-//        val localTransform = transform {
-//            translation?.let {
-//                translate(it[0], it[1], it[2])
-//            }
-//            rotation?.let {
-//                val q = Quaternion(it[0], it[1], it[2], it[3])
-//                multiply(q.matrix.matrix44)
-//            }
-//            scale?.let {
-//                scale(it[0], it[1], it[2])
-//            }
-//        }
-
         node.translation = translation?.let { Vector3(it[0], it[1], it[2]) } ?: Vector3.ZERO
         node.scale = scale?.let { Vector3(it[0], it[1], it[2]) } ?: Vector3.ONE
         node.rotation = rotation?.let { Quaternion(it[0], it[1], it[2], it[3]) } ?: Quaternion.IDENTITY
@@ -214,20 +217,66 @@ fun GltfFile.buildSceneNodes(): GltfSceneData {
         matrix?.let {
             node.transform = Matrix44.fromDoubleArray(it).transposed
         }
-
-
-//        node.transform = this.matrix?.let {
-//            Matrix44.fromDoubleArray(it).transposed
-//        } ?: localTransform
         for (child in children.orEmpty) {
             node.children.add(nodes[child].createSceneNode())
         }
         node
     }
 
+    val sceneMeshes = mutableMapOf<GltfMesh, MeshBase>()
+    fun GltfMesh.createSceneMesh(skin: GltfSkin?): MeshBase = sceneMeshes.getOrPut(this) {
+        if (skin == null) {
+            Mesh(primitives.map {
+                it.createScenePrimitive()
+            })
+        } else {
+            val joints = skin.joints.map { nodes[it].createSceneNode() }
+            val skeleton = nodes[skin.skeleton].createSceneNode()
+            val ibmAccessor = accessors[skin.inverseBindMatrices]
+            val ibmBufferView = bufferViews[ibmAccessor.bufferView]
+            val ibmBuffer = buffers[ibmBufferView.buffer]
+
+            val ibmData = ibmBuffer.contents(this@buildSceneNodes)
+            ibmData.order(ByteOrder.nativeOrder())
+            (ibmData as Buffer).position(ibmAccessor.byteOffset + (ibmBufferView.byteOffset ?: 0))
+
+            require(ibmAccessor.type == "MAT4")
+            require(ibmAccessor.componentType == GLTF_FLOAT)
+            require(ibmAccessor.count == joints.size)
+            val ibms = (0 until ibmAccessor.count).map {
+                val array = DoubleArray(16)
+                for (i in 0 until 16) {
+                    array[i] = ibmData.float.toDouble()
+                }
+
+                val m = Matrix44.fromDoubleArray(array).transposed
+                println("--")
+                println(m)
+                println(array.joinToString(","))
+
+                m
+            }
+
+            SkinnedMesh(primitives.map {
+                it.createScenePrimitive()
+            }, joints, skeleton, ibms)
+        }
+    }
+
     val scenes = scenes.map { scene ->
+        println(scene.nodes.size)
         scene.nodes.map { node ->
-            nodes[node].createSceneNode()
+            println("node: $node")
+            val gltfNode = nodes[node]
+            val sceneNode = gltfNode.createSceneNode()
+            sceneNode
+        }
+    }
+    for ((gltfNode, sceneNode) in sceneNodes) {
+        gltfNode.mesh?.let {
+            val skin = gltfNode.skin?.let { (skins!!)[it] }
+            println("adding mesh")
+            sceneNode.entities.add(meshes[it].createSceneMesh(skin))
         }
     }
 
@@ -276,7 +325,7 @@ fun GltfFile.buildSceneNodes(): GltfSceneData {
                         }
                         val keyframer = KeyframerChannelQuaternion()
                         val inputOffset = (inputBufferView.byteOffset ?: 0) + (inputAccessor.byteOffset ?: 0)
-                        val outputOffset = (outputBufferView.byteOffset ?: 0) + (inputAccessor.byteOffset ?: 0)
+                        val outputOffset = (outputBufferView.byteOffset ?: 0) + (outputAccessor.byteOffset ?: 0)
                         val inputStride = (inputBufferView.byteStride ?: 4)
                         val outputStride = (outputBufferView.byteStride ?: 16)
                         for (i in 0 until outputAccessor.count) {