diff --git a/build.gradle b/build.gradle index f98fc092..52d551da 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ buildscript { apply plugin: 'org.jetbrains.dokka' project.ext { - openrndrVersion = "0.3.43-rc.1" + openrndrVersion = "0.3.43-rc.2" kotlinVersion = "1.3.72" spekVersion = "2.0.10" libfreenectVersion = "0.5.7-1.5.3" diff --git a/demo-data/gltf-models/box/Box.glb b/demo-data/gltf-models/box/Box.glb new file mode 100644 index 00000000..95ec886b Binary files /dev/null and b/demo-data/gltf-models/box/Box.glb differ diff --git a/demo-data/gltf-models/box/Box.gltf b/demo-data/gltf-models/box/Box.gltf deleted file mode 100644 index f434ca23..00000000 --- a/demo-data/gltf-models/box/Box.gltf +++ /dev/null @@ -1,142 +0,0 @@ -{ - "asset": { - "generator": "COLLADA2GLTF", - "version": "2.0" - }, - "scene": 0, - "scenes": [ - { - "nodes": [ - 0 - ] - } - ], - "nodes": [ - { - "children": [ - 1 - ], - "matrix": [ - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - -1.0, - 0.0, - 0.0, - 1.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 1.0 - ] - }, - { - "mesh": 0 - } - ], - "meshes": [ - { - "primitives": [ - { - "attributes": { - "NORMAL": 1, - "POSITION": 2 - }, - "indices": 0, - "mode": 5, - "material": 0 - } - ], - "name": "Mesh" - } - ], - "accessors": [ - { - "bufferView": 0, - "byteOffset": 0, - "componentType": 5123, - "count": 36, - "max": [ - 23 - ], - "min": [ - 0 - ], - "type": "SCALAR" - }, - { - "bufferView": 1, - "byteOffset": 0, - "componentType": 5126, - "count": 24, - "max": [ - 1.0, - 1.0, - 1.0 - ], - "min": [ - -1.0, - -1.0, - -1.0 - ], - "type": "VEC3" - }, - { - "bufferView": 1, - "byteOffset": 288, - "componentType": 5126, - "count": 24, - "max": [ - 0.5, - 0.5, - 0.5 - ], - "min": [ - -0.5, - -0.5, - -0.5 - ], - "type": "VEC3" - } - ], - "materials": [ - { - "pbrMetallicRoughness": { - "baseColorFactor": [ - 0.800000011920929, - 0.0, - 0.0, - 1.0 - ], - "metallicFactor": 0.0 - }, - "name": "Red" - } - ], - "bufferViews": [ - { - "buffer": 0, - "byteOffset": 576, - "byteLength": 72, - "target": 34963 - }, - { - "buffer": 0, - "byteOffset": 0, - "byteLength": 576, - "byteStride": 12, - "target": 34962 - } - ], - "buffers": [ - { - "byteLength": 648, - "uri": "Box0.bin" - } - ] -} diff --git a/demo-data/gltf-models/box/Box0.bin b/demo-data/gltf-models/box/Box0.bin deleted file mode 100644 index d7798abb..00000000 Binary files a/demo-data/gltf-models/box/Box0.bin and /dev/null differ diff --git a/demo-data/gltf-models/fox/Fox.glb b/demo-data/gltf-models/fox/Fox.glb new file mode 100644 index 00000000..a541b6f4 Binary files /dev/null and b/demo-data/gltf-models/fox/Fox.glb differ diff --git a/orx-dnk3/README.md b/orx-dnk3/README.md index 06e793a5..56f7da49 100644 --- a/orx-dnk3/README.md +++ b/orx-dnk3/README.md @@ -12,10 +12,10 @@ Supported Gltf features - [x] Basic materials - [x] Normal maps - [x] Metallic/roughness maps - - [ ] Skinning + - [x] Skinning - [x] Double-sided materials - [ ] Transparency -- [ ] Animations +- [x] Animations - [ ] Cameras - [ ] Lights diff --git a/orx-dnk3/build.gradle b/orx-dnk3/build.gradle index ba8a852a..82a9d5a5 100644 --- a/orx-dnk3/build.gradle +++ b/orx-dnk3/build.gradle @@ -12,9 +12,11 @@ dependencies { implementation "com.google.code.gson:gson:$gsonVersion" implementation(project(":orx-fx")) implementation(project(":orx-keyframer")) + demoImplementation(project(":orx-camera")) demoImplementation("org.openrndr:openrndr-core:$openrndrVersion") demoImplementation("org.openrndr:openrndr-extensions:$openrndrVersion") + demoImplementation("org.openrndr:openrndr-ffmpeg:$openrndrVersion") demoRuntimeOnly("org.openrndr:openrndr-gl3:$openrndrVersion") demoRuntimeOnly("org.openrndr:openrndr-gl3-natives-$openrndrOS:$openrndrVersion") demoImplementation(sourceSets.getByName("main").output) diff --git a/orx-dnk3/src/demo/kotlin/DemoScene02.kt b/orx-dnk3/src/demo/kotlin/DemoScene02.kt index a589b042..73659c8f 100644 --- a/orx-dnk3/src/demo/kotlin/DemoScene02.kt +++ b/orx-dnk3/src/demo/kotlin/DemoScene02.kt @@ -25,7 +25,7 @@ fun main() = application { } } - val gltf = loadGltfFromFile(File("demo-data/gltf-models/complex02/scene.gltf")) + val gltf = loadGltfFromFile(File("demo-data/gltf-models/duck/Duck.gltf")) // val gltf = loadGltfFromGlbFile(File("demo-data/gltf-models/splash-sss.glb")) val scene = Scene(SceneNode()) @@ -48,8 +48,8 @@ fun main() = application { val renderer = dryRenderer() extend(Orbital()) { far = 500.0 - lookAt = Vector3(0.0, 0.7, 0.0) - eye = Vector3(3.0, 0.7, -2.0) + lookAt = Vector3(0.0, 0.8, 0.0) + eye = Vector3(3.0, 0.8, -2.0) fov = 30.0 } extend { diff --git a/orx-dnk3/src/demo/kotlin/DemoSkinning01.kt b/orx-dnk3/src/demo/kotlin/DemoSkinning01.kt new file mode 100644 index 00000000..d45e9169 --- /dev/null +++ b/orx-dnk3/src/demo/kotlin/DemoSkinning01.kt @@ -0,0 +1,52 @@ +import org.openrndr.application +import org.openrndr.color.ColorRGBa +import org.openrndr.extensions.SingleScreenshot +import org.openrndr.extra.dnk3.* +import org.openrndr.extra.dnk3.gltf.buildSceneNodes +import org.openrndr.extra.dnk3.gltf.loadGltfFromFile +import org.openrndr.extras.camera.Orbital +import org.openrndr.math.Vector3 +import org.openrndr.math.mod_ +import java.io.File + +fun main() = application { + configure { + width = 1280 + height = 720 + //multisample = WindowMultisample.SampleCount(8) + } + + program { + if (System.getProperty("takeScreenshot") == "true") { + extend(SingleScreenshot()) { + this.outputFile = System.getProperty("screenshotPath") + } + } + + val gltf = loadGltfFromFile(File("demo-data/gltf-models/fox/Fox.glb")) + val scene = Scene(SceneNode()) + + scene.root.entities.add(HemisphereLight().apply { + upColor = ColorRGBa.WHITE.shade(0.4) + downColor = ColorRGBa.GRAY.shade(0.1) + }) + val sceneData = gltf.buildSceneNodes() + scene.root.children.addAll(sceneData.scenes.first()) + + + // -- create a renderer + val renderer = dryRenderer() + extend(Orbital()) { + far = 500.0 + lookAt = Vector3(0.0, 40.0, 0.0) + eye = Vector3(150.0, 40.0, 200.0) + fov = 40.0 + } + + extend { + sceneData.animations[2].applyToTargets(seconds.mod_(sceneData.animations[2].duration)) + drawer.clear(ColorRGBa.PINK) + renderer.draw(drawer, scene) + } + } +} \ No newline at end of file diff --git a/orx-dnk3/src/main/kotlin/Entity.kt b/orx-dnk3/src/main/kotlin/Entity.kt index f8a4d1cf..11cdbecd 100644 --- a/orx-dnk3/src/main/kotlin/Entity.kt +++ b/orx-dnk3/src/main/kotlin/Entity.kt @@ -2,24 +2,47 @@ package org.openrndr.extra.dnk3 import org.openrndr.color.ColorRGBa import org.openrndr.draw.* +import org.openrndr.math.Matrix44 class Geometry(val vertexBuffers: List, val indexBuffer: IndexBuffer?, val primitive: DrawPrimitive, val offset: Int, - val vertexCount: Int) + val vertexCount: Int) { + + override fun toString(): String { + return "Geometry(vertexBuffers: $vertexBuffers, indexBuffers: $indexBuffer, primitive: $primitive, offset: $offset, vertexCount: $vertexCount)" + } + +} val DummyGeometry = Geometry(emptyList(), null, DrawPrimitive.TRIANGLES, 0, 0) sealed class Entity -class MeshPrimitive(var geometry: Geometry, var material: Material) +class MeshPrimitive(var geometry: Geometry, var material: Material) { + + override fun toString(): String { + return "MeshPrimitive(geometry: $geometry, material: $material)" + } + +} class MeshPrimitiveInstance(val primitive: MeshPrimitive, val instances: Int, val attributes: List) abstract class MeshBase(var primitives: List) : Entity() -class Mesh(primitives: List) : MeshBase(primitives) +class Mesh(primitives: List) : MeshBase(primitives) { + override fun toString(): String { + return "Mesh(primitives: $primitives)" + } +} + +class SkinnedMesh(primitives: List, + val joints: List, + val skeleton: SceneNode, + val inverseBindMatrices: List +) : MeshBase(primitives) class InstancedMesh(primitives: List, var instances: Int, diff --git a/orx-dnk3/src/main/kotlin/Facet.kt b/orx-dnk3/src/main/kotlin/Facet.kt index f4519eab..197cb717 100644 --- a/orx-dnk3/src/main/kotlin/Facet.kt +++ b/orx-dnk3/src/main/kotlin/Facet.kt @@ -113,10 +113,9 @@ class ClipPositionFacet : ColorBufferFacetCombiner(setOf(FacetType.CLIP_POSITION class LDRColorFacet : ColorBufferFacetCombiner(setOf(FacetType.DIFFUSE, FacetType.SPECULAR), "color", ColorFormat.RGBa, ColorType.UINT8) { override fun generateShader() = """ - vec3 oofinalColor = (f_diffuse.rgb + f_specular.rgb + f_emission.rgb) * (1.0 - f_fog.a) + f_fog.rgb * f_fog.a; - o_$targetOutput = pow(vec4(oofinalColor.rgb, 1.0), vec4(1.0/2.2)); - o_$targetOutput *= m_color.a; - + vec3 finalColor = (max(vec3(0.0), f_diffuse.rgb) + max(vec3(0.0),f_specular.rgb) + max(vec3(0.0), f_emission.rgb)) * (1.0 - f_fog.a) + f_fog.rgb * f_fog.a; + o_$targetOutput = pow(vec4(finalColor.rgb, 1.0), vec4(1.0/2.2)); + o_$targetOutput *= m_color.a; """ } \ No newline at end of file diff --git a/orx-dnk3/src/main/kotlin/Material.kt b/orx-dnk3/src/main/kotlin/Material.kt index 9e488538..a9f79013 100644 --- a/orx-dnk3/src/main/kotlin/Material.kt +++ b/orx-dnk3/src/main/kotlin/Material.kt @@ -8,7 +8,7 @@ import org.openrndr.draw.shadeStyle interface Material { var doubleSided: Boolean var transparent: Boolean - fun generateShadeStyle(context: MaterialContext): ShadeStyle + fun generateShadeStyle(context: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle fun applyToShadeStyle(context: MaterialContext, shadeStyle: ShadeStyle) } @@ -17,7 +17,7 @@ class DummyMaterial : Material { override var transparent: Boolean = false - override fun generateShadeStyle(context: MaterialContext): ShadeStyle { + override fun generateShadeStyle(context: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle { return shadeStyle { fragmentTransform = """ x_fill.rgb = vec3(normalize(v_viewNormal).z); @@ -38,3 +38,7 @@ data class MaterialContext(val pass: RenderPass, val meshCubemaps: Map ) +data class PrimitiveContext(val hasNormalAttribute: Boolean, val hasSkinning: Boolean) + + +data class ContextKey(val materialContext: MaterialContext, val primitiveContext: PrimitiveContext) diff --git a/orx-dnk3/src/main/kotlin/PBRMaterial.kt b/orx-dnk3/src/main/kotlin/PBRMaterial.kt index 835d2503..c1d1dd62 100644 --- a/orx-dnk3/src/main/kotlin/PBRMaterial.kt +++ b/orx-dnk3/src/main/kotlin/PBRMaterial.kt @@ -7,6 +7,7 @@ import org.openrndr.math.Vector3 import org.openrndr.math.Vector4 import org.openrndr.math.transforms.normalMatrix import java.nio.ByteBuffer +import javax.naming.Context import kotlin.math.cos @@ -29,7 +30,7 @@ private val noise128 by lazy { cb } -private fun PointLight.fs(index: Int): String = """ +private fun PointLight.fs(index: Int, hasNormalAttribute: Boolean): String = """ |{ | vec3 Lr = p_lightPosition$index - v_worldPosition; | float distance = length(Lr); @@ -37,7 +38,7 @@ private fun PointLight.fs(index: Int): String = """ | p_lightLinearAttenuation$index * distance + p_lightQuadraticAttenuation$index * distance * distance); | vec3 L = normalize(Lr); | -| float side = dot(L, N) ; +| float side = ${if (hasNormalAttribute) "dot(L, N)" else "3.1415"}; | f_diffuse += attenuation * max(0, side / 3.1415) * p_lightColor$index.rgb * m_color.rgb; | f_specular += attenuation * ggx(N, V, L, m_roughness, m_f0) * p_lightColor$index.rgb * m_color.rgb; } @@ -45,14 +46,14 @@ private fun PointLight.fs(index: Int): String = """ private fun AmbientLight.fs(index: Int): String = "f_ambient += p_lightColor$index.rgb * ((1.0 - m_metalness) * m_color.rgb);" -private fun DirectionalLight.fs(index: Int) = """ +private fun DirectionalLight.fs(index: Int, hasNormalAttribute: Boolean) = """ |{ | vec3 L = normalize(-p_lightDirection$index); | float attenuation = 1.0; | vec3 H = normalize(V + L); -| float NoL = clamp(dot(N, L), 0.0, 1.0); +| float NoL = ${if (hasNormalAttribute) "clamp(dot(N, L), 0.0, 1.0)" else "1"}; | float LoH = clamp(dot(L, H), 0.0, 1.0); -| float NoH = clamp(dot(N, H), 0.0, 1.0); +| float NoH = ${if (hasNormalAttribute) "clamp(dot(N, H), 0.0, 1.0)" else "1"}; | vec3 Lr = (p_lightPosition$index - v_worldPosition); //| vec3 L = normalize(Lr); | ${shadows.fs(index)} @@ -66,15 +67,15 @@ private fun DirectionalLight.fs(index: Int) = """ |} """.trimMargin() -private fun HemisphereLight.fs(index: Int): String = """ +private fun HemisphereLight.fs(index: Int, hasNormalAttribute: Boolean): String = """ |{ -| float f = dot(N, p_lightDirection$index) * 0.5 + 0.5; +| float f = ${if (hasNormalAttribute) "dot(N, p_lightDirection$index) * 0.5 + 0.5" else "1"}; | vec3 irr = ${irradianceMap?.let { "texture(p_lightIrradianceMap$index, N).rgb" } ?: "vec3(1.0)"}; | f_diffuse += mix(p_lightDownColor$index.rgb, p_lightUpColor$index.rgb, f) * irr * ((1.0 - m_metalness) * m_color.rgb) * m_ambientOcclusion; |} """.trimMargin() -private fun SpotLight.fs(index: Int): String { +private fun SpotLight.fs(index: Int, hasNormalAttribute: Boolean): String { val shadows = shadows return """ |{ @@ -85,7 +86,7 @@ private fun SpotLight.fs(index: Int): String { | attenuation = 1.0; | vec3 L = normalize(Lr); -| float NoL = clamp(dot(N, L), 0.0, 1.0); +| float NoL = ${if (hasNormalAttribute) "clamp(dot(N, L), 0.0, 1.0)" else "1"}; | float side = dot(L, N); | float hit = max(dot(-L, p_lightDirection$index), 0.0); | float falloff = clamp((hit - p_lightOuterCos$index) / (p_lightInnerCos$index - p_lightOuterCos$index), 0.0, 1.0); @@ -94,7 +95,7 @@ private fun SpotLight.fs(index: Int): String { | { | vec3 H = normalize(V + L); | float LoH = clamp(dot(L, H), 0.0, 1.0); -| float NoH = clamp(dot(N, H), 0.0, 1.0); +| float NoH = ${if (hasNormalAttribute) "clamp(dot(N, H), 0.0, 1.0)" else 1.0}; | f_diffuse += NoL * (0.1+0.9*attenuation) * Fd_Burley(m_roughness * m_roughness, NoV, NoL, LoH) * p_lightColor$index.rgb * m_color.rgb ; | float Dg = D_GGX(m_roughness * m_roughness, NoH, H); | float Vs = V_SmithGGXCorrelated(m_roughness * m_roughness, NoV, NoL); @@ -114,7 +115,12 @@ private fun Fog.fs(index: Int): String = """ """.trimMargin() sealed class TextureSource -object DummySource : TextureSource() +object DummySource : TextureSource() { + override fun toString(): String { + return "DummySource()" + } +} + abstract class TextureFromColorBuffer(var texture: ColorBuffer, var textureFunction: TextureFunction) : TextureSource() class TextureFromCode(val code: String) : TextureSource() @@ -142,10 +148,14 @@ enum class TextureFunction(val function: (String, String) -> String) { */ class ModelCoordinates(texture: ColorBuffer, var input: String = "va_texCoord0.xy", - var tangentInput : String? = null, + var tangentInput: String? = null, textureFunction: TextureFunction = TextureFunction.TILING, var pre: String? = null, - var post: String? = null) : TextureFromColorBuffer(texture, textureFunction) + var post: String? = null) : TextureFromColorBuffer(texture, textureFunction) { + override fun toString(): String { + return "ModelCoordinates(texture: $texture, input: $input, $tangentInput: $tangentInput, textureFunction: $textureFunction, pre: $pre, post: $post)" + } +} class Triplanar(texture: ColorBuffer, var scale: Double = 1.0, @@ -171,7 +181,8 @@ private fun ModelCoordinates.fs(index: Int) = """ | ${if (pre != null) "{ $pre } " else ""} | x_texture = ${textureFunction.function("p_texture$index", "x_texCoord")}; | ${if (post != null) "{ $post } " else ""} -| ${if (tangentInput != null) { """ +| ${if (tangentInput != null) { + """ | vec3 normal = normalize(va_normal.xyz); | vec3 tangent = normalize(${tangentInput}.xyz); | vec3 bitangent = cross(normal, tangent) * ${tangentInput}.w; @@ -179,7 +190,7 @@ private fun ModelCoordinates.fs(index: Int) = """ | x_texture.rgb = tbn * normalize(x_texture.rgb - vec3(0.5, 0.5, 0.)) ; | """.trimMargin() - + } else ""} | tex$index = x_texture; |} @@ -225,16 +236,20 @@ private fun Triplanar.fs(index: Int, target: TextureTarget) = """ """.trimIndent() else ""} """.trimMargin() -sealed class TextureTarget { - object NONE : TextureTarget() - object COLOR : TextureTarget() - object ROUGHNESS : TextureTarget() - object METALNESS : TextureTarget() - object METALNESS_ROUGHNESS : TextureTarget() - object EMISSION : TextureTarget() - object NORMAL : TextureTarget() - object AMBIENT_OCCLUSION : TextureTarget() - class Height(var scale: Double = 1.0) : TextureTarget() +sealed class TextureTarget(val name: String) { + object NONE : TextureTarget("NONE") + object COLOR : TextureTarget("COLOR") + object ROUGHNESS : TextureTarget("ROUGHNESS") + object METALNESS : TextureTarget("METALNESS") + object METALNESS_ROUGHNESS : TextureTarget("METALNESS_ROUGHNESS") + object EMISSION : TextureTarget("EMISSION") + object NORMAL : TextureTarget("NORMAL") + object AMBIENT_OCCLUSION : TextureTarget("AMBIENT_OCCLUSION") + class Height(var scale: Double = 1.0) : TextureTarget("Height") + + override fun toString(): String { + return "TextureTarget(name: $name)" + } } class Texture(var source: TextureSource, @@ -243,9 +258,17 @@ class Texture(var source: TextureSource, val copied = Texture(source, target) return copied } + + override fun toString(): String { + return "Texture(source: $source, target: $target)" + } } class PBRMaterial : Material { + override fun toString(): String { + return "PBRMaterial(textures: $textures, color: $color, metalness: $metalness, roughness: $roughness, emissive: $emission))" + } + override var doubleSided: Boolean = false override var transparent: Boolean = false var environmentMap = false @@ -259,7 +282,7 @@ class PBRMaterial : Material { var parameters = mutableMapOf() var textures = mutableListOf() - val shadeStyles = mutableMapOf() + val shadeStyles = mutableMapOf() // fun copy(): PBRMaterial { // val copied = PBRMaterial() @@ -276,9 +299,9 @@ class PBRMaterial : Material { // return copied // } - override fun generateShadeStyle(context: MaterialContext): ShadeStyle { - val cached = shadeStyles.getOrPut(context) { - val needLight = needLight(context) + override fun generateShadeStyle(materialContext: MaterialContext, primitiveContext: PrimitiveContext): ShadeStyle { + val cached = shadeStyles.getOrPut(ContextKey(materialContext, primitiveContext)) { + val needLight = needLight(materialContext) val preambleFS = """ vec4 m_color = p_color; float m_f0 = 0.5; @@ -317,6 +340,20 @@ class PBRMaterial : Material { val displacers = textures.filter { it.target is TextureTarget.Height } + val skinVS = if (primitiveContext.hasSkinning) """ + uvec4 j = a_joints; + mat4 skinTransform = p_jointTransforms[j.x] * a_weights.x + + p_jointTransforms[j.y] * a_weights.y + + p_jointTransforms[j.z] * a_weights.z + + p_jointTransforms[j.w] * a_weights.w; + + ${if (primitiveContext.hasNormalAttribute) """ + x_normal = normalize(mat3(skinTransform) * x_normal); + """.trimIndent() else ""} + + x_position = (skinTransform * vec4(x_position,1)).xyz; + """.trimIndent() else "" + val textureVS = if (displacers.isNotEmpty()) textures.mapIndexed { index, it -> if (it.target is TextureTarget.Height) { when (val source = it.source) { @@ -331,7 +368,7 @@ class PBRMaterial : Material { } else "" }.joinToString("\n") else "" - val lights = context.lights + val lights = materialContext.lights val lightFS = if (needLight) """ vec3 f_diffuse = vec3(0.0); vec3 f_specular = vec3(0.0); @@ -342,9 +379,9 @@ class PBRMaterial : Material { vec3 ep = (p_viewMatrixInverse * vec4(0.0, 0.0, 0.0, 1.0)).xyz; vec3 Vr = ep - v_worldPosition; vec3 V = normalize(Vr); - float NoV = abs(dot(N, V)) + 1e-5; + float NoV = ${if (primitiveContext.hasNormalAttribute) "abs(dot(N, V)) + 1e-5" else "1"}; - ${if (environmentMap && context.meshCubemaps.isNotEmpty()) """ + ${if (environmentMap && materialContext.meshCubemaps.isNotEmpty() && primitiveContext.hasNormalAttribute) """ { vec2 dfg = PrefilteredDFG_Karis(m_roughness, NoV); vec3 sc = m_metalness * m_color.rgb + (1.0-m_metalness) * vec3(0.04); @@ -356,32 +393,32 @@ class PBRMaterial : Material { ${lights.mapIndexed { index, (node, light) -> when (light) { is AmbientLight -> light.fs(index) - is PointLight -> light.fs(index) - is SpotLight -> light.fs(index) - is DirectionalLight -> light.fs(index) - is HemisphereLight -> light.fs(index) + is PointLight -> light.fs(index, primitiveContext.hasNormalAttribute) + is SpotLight -> light.fs(index, primitiveContext.hasNormalAttribute) + is DirectionalLight -> light.fs(index, primitiveContext.hasNormalAttribute) + is HemisphereLight -> light.fs(index, primitiveContext.hasNormalAttribute) else -> TODO() } }.joinToString("\n")} - ${context.fogs.mapIndexed { index, (node, fog) -> + ${materialContext.fogs.mapIndexed { index, (node, fog) -> fog.fs(index) }.joinToString("\n")} """.trimIndent() else "" val rt = RenderTarget.active - val combinerFS = context.pass.combiners.map { + val combinerFS = materialContext.pass.combiners.map { it.generateShader() }.joinToString("\n") val fs = preambleFS + textureFs + lightFS + combinerFS - val vs = (this@PBRMaterial.vertexTransform ?: "") + textureVS + val vs = (this@PBRMaterial.vertexTransform ?: "") + textureVS + skinVS shadeStyle { vertexPreamble = """ $shaderNoRepetitionVert - ${(this@PBRMaterial.vertexPreamble)?:""} + ${(this@PBRMaterial.vertexPreamble) ?: ""} """.trimIndent() fragmentPreamble = """ |$shaderLinePlaneIntersect @@ -394,7 +431,7 @@ class PBRMaterial : Material { this.suppressDefaultOutput = true this.vertexTransform = vs fragmentTransform = fs - context.pass.combiners.map { + materialContext.pass.combiners.map { if (rt.colorBuffers.size <= 1) { this.output(it.targetOutput, 0) } else diff --git a/orx-dnk3/src/main/kotlin/SceneRenderer.kt b/orx-dnk3/src/main/kotlin/SceneRenderer.kt index 870b401f..e9f2d18b 100644 --- a/orx-dnk3/src/main/kotlin/SceneRenderer.kt +++ b/orx-dnk3/src/main/kotlin/SceneRenderer.kt @@ -5,6 +5,7 @@ import org.openrndr.draw.* import org.openrndr.draw.depthBuffer import org.openrndr.extra.fx.blur.ApproximateGaussianBlur import org.openrndr.math.Matrix44 +import org.openrndr.math.transforms.normalMatrix class SceneRenderer { @@ -48,10 +49,11 @@ class SceneRenderer { val lights = scene.root.findContent { this as? Light } val meshes = scene.root.findContent { this as? Mesh } + val skinnedMeshes = scene.root.findContent { this as? SkinnedMesh } + val fogs = scene.root.findContent { this as? Fog } val instancedMeshes = scene.root.findContent { this as? InstancedMesh } - run { lights.filter { it.content is ShadowLight && (it.content as ShadowLight).shadows is Shadows.MappedShadows }.forEach { val shadowLight = it.content as ShadowLight @@ -80,7 +82,7 @@ class SceneRenderer { drawer.clear(ColorRGBa.BLACK) drawer.cullTestPass = CullTestPass.FRONT - drawPass(drawer, pass, materialContext, meshes, instancedMeshes) + drawPass(drawer, pass, materialContext, meshes, instancedMeshes, skinnedMeshes) } when (shadowLight.shadows) { is Shadows.VSM -> { @@ -99,7 +101,6 @@ class SceneRenderer { for (pass in outputPasses) { val materialContext = MaterialContext(pass, lights, fogs, shadowLightTargets, meshCubemaps) - val defaultPasses = setOf(DefaultTransparentPass, DefaultOpaquePass) if ((pass !in defaultPasses || postSteps.isNotEmpty()) && outputPassTarget == null) { @@ -109,7 +110,7 @@ class SceneRenderer { if (pass == outputPasses[0]) { outputPassTarget?.let { drawer.withTarget(it) { - background(ColorRGBa.PINK) + clear(ColorRGBa.PINK) } } } @@ -122,7 +123,7 @@ class SceneRenderer { } } outputPassTarget?.bind() - drawPass(drawer, pass, materialContext, meshes, instancedMeshes) + drawPass(drawer, pass, materialContext, meshes, instancedMeshes, skinnedMeshes) outputPassTarget?.unbind() outputPassTarget?.let { output -> @@ -167,7 +168,9 @@ class SceneRenderer { private fun drawPass(drawer: Drawer, pass: RenderPass, materialContext: MaterialContext, meshes: List>, - instancedMeshes: List>) { + instancedMeshes: List>, + skinnedMeshes: List> + ) { drawer.depthWrite = pass.depthWrite val primitives = meshes.flatMap { mesh -> @@ -185,8 +188,9 @@ class SceneRenderer { if (primitive.material.doubleSided) { drawer.drawStyle.cullTestPass = CullTestPass.ALWAYS } - - val shadeStyle = primitive.material.generateShadeStyle(materialContext) + val hasNormalAttribute = primitive.geometry.vertexBuffers.any { it.vertexFormat.hasAttribute("normal") } + val primitiveContext = PrimitiveContext(hasNormalAttribute, false) + val shadeStyle = primitive.material.generateShadeStyle(materialContext, primitiveContext) shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed) primitive.material.applyToShadeStyle(materialContext, shadeStyle) drawer.shadeStyle = shadeStyle @@ -207,6 +211,62 @@ class SceneRenderer { } } + + val skinnedPrimitives = skinnedMeshes.flatMap { mesh -> + mesh.content.primitives.map { primitive -> + NodeContent(mesh.node, Pair(primitive, mesh)) + } + } + + skinnedPrimitives + .filter { + (it.content.first.material.transparent && pass.renderTransparent) || + (!it.content.first.material.transparent && pass.renderOpaque) + } + .forEach { + val primitive = it.content.first + val skinnedMesh = it.content.second.content + drawer.isolated { + if (primitive.material.doubleSided) { + drawer.drawStyle.cullTestPass = CullTestPass.ALWAYS + } + val hasNormalAttribute = primitive.geometry.vertexBuffers.any { it.vertexFormat.hasAttribute("normal") } + val primitiveContext = PrimitiveContext(hasNormalAttribute, true) + + val nodeInverse = it.node.worldTransform.inversed + + + + val jointTransforms = (skinnedMesh.joints zip skinnedMesh.inverseBindMatrices) + .map{ (nodeInverse * it.first.worldTransform * it.second) } +// val jointNormalTransforms = jointTransforms.map { Matrix44.IDENTITY } + + val shadeStyle = primitive.material.generateShadeStyle(materialContext, primitiveContext) + + shadeStyle.parameter("jointTransforms", jointTransforms.toTypedArray()) +// shadeStyle.parameter("jointNormalTransforms", jointNormalTransforms.toTypedArray()) + + shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed) + primitive.material.applyToShadeStyle(materialContext, shadeStyle) + drawer.shadeStyle = shadeStyle + drawer.model = it.node.worldTransform + + if (primitive.geometry.indexBuffer == null) { + drawer.vertexBuffer(primitive.geometry.vertexBuffers, + primitive.geometry.primitive, + primitive.geometry.offset, + primitive.geometry.vertexCount) + } else { + drawer.vertexBuffer(primitive.geometry.indexBuffer!!, + primitive.geometry.vertexBuffers, + primitive.geometry.primitive, + primitive.geometry.offset, + primitive.geometry.vertexCount) + } + } + } + + val instancedPrimitives = instancedMeshes.flatMap { mesh -> mesh.content.primitives.map { primitive -> NodeContent(mesh.node, MeshPrimitiveInstance(primitive, mesh.content.instances, mesh.content.attributes)) @@ -219,7 +279,8 @@ class SceneRenderer { .forEach { val primitive = it.content drawer.isolated { - val shadeStyle = primitive.primitive.material.generateShadeStyle(materialContext) + val primitiveContext = PrimitiveContext(true, false) + val shadeStyle = primitive.primitive.material.generateShadeStyle(materialContext, primitiveContext) shadeStyle.parameter("viewMatrixInverse", drawer.view.inversed) primitive.primitive.material.applyToShadeStyle(materialContext, shadeStyle) if (primitive.primitive.material.doubleSided) { diff --git a/orx-dnk3/src/main/kotlin/gltf/Glb.kt b/orx-dnk3/src/main/kotlin/gltf/Glb.kt index bcc1cde3..87301f4c 100644 --- a/orx-dnk3/src/main/kotlin/gltf/Glb.kt +++ b/orx-dnk3/src/main/kotlin/gltf/Glb.kt @@ -28,6 +28,7 @@ fun loadGltfFromGlbFile(file: File): GltfFile { if (chunkType == 0x004E4942) ByteBuffer.allocateDirect(chunkLength) else ByteBuffer.allocate(chunkLength) (chunkBuffer as ByteBuffer) channel.read(chunkBuffer) + chunkBuffer.order(ByteOrder.nativeOrder()) return chunkBuffer } diff --git a/orx-dnk3/src/main/kotlin/gltf/Gltf.kt b/orx-dnk3/src/main/kotlin/gltf/Gltf.kt index 78f6cc21..a1ae60c2 100644 --- a/orx-dnk3/src/main/kotlin/gltf/Gltf.kt +++ b/orx-dnk3/src/main/kotlin/gltf/Gltf.kt @@ -33,7 +33,8 @@ class GltfNode(val children: IntArray?, val scale: DoubleArray?, val rotation: DoubleArray?, val translation: DoubleArray?, - val mesh: Int?) + val mesh: Int?, + val skin: Int?) class GltfPrimitive(val attributes: LinkedHashMap, val indices: Int?, val mode: Int?, val material: Int) { fun createDrawCommand(gltfFile: GltfFile): GltfDrawCommand { @@ -87,6 +88,23 @@ class GltfPrimitive(val attributes: LinkedHashMap, val indices: Int textureCoordinate(dimensions, 0) accessors.add(accessor) } + "JOINTS_0" -> { + val type = when (Pair(accessor.type, accessor.componentType)) { + Pair("VEC4", GLTF_UNSIGNED_BYTE) -> VertexElementType.VECTOR4_UINT8 + Pair("VEC4", GLTF_UNSIGNED_SHORT) -> VertexElementType.VECTOR4_UINT16 + else -> error("not supported ${accessor.type} / ${accessor.componentType}") + } + attribute("joints", type) + accessors.add(accessor) + } + "WEIGHTS_0" -> { + val type = when (Pair(accessor.type, accessor.componentType)) { + Pair("VEC4", GLTF_FLOAT) -> VertexElementType.VECTOR4_FLOAT32 + else -> error("not supported ${accessor.type} / ${accessor.componentType}") + } + attribute("weights", type) + accessors.add(accessor) + } } } } @@ -181,7 +199,6 @@ class GltfBufferView(val buffer: Int, class GltfBuffer(val byteLength: Int, val uri: String?) { fun contents(gltfFile: GltfFile): ByteBuffer = if (uri != null) { - if (uri.startsWith("data:")) { val base64 = uri.substring(uri.indexOf(",") + 1) val decoded = Base64.getDecoder().decode(base64) @@ -223,6 +240,9 @@ class GltfChannelTarget(val node: Int?, val path: String?) class GltfChannel(val sampler: Int, val target: GltfChannelTarget) + +class GltfSkin(val inverseBindMatrices: Int, val joints: IntArray, val skeleton: Int) + class GltfFile( val asset: GltfAsset?, val scene: Int?, @@ -236,7 +256,9 @@ class GltfFile( val images: List?, val textures: List?, val samplers: List?, - val animations: List?) { + val animations: List?, + val skins: List? +) { @Transient lateinit var file: File diff --git a/orx-dnk3/src/main/kotlin/gltf/GltfScene.kt b/orx-dnk3/src/main/kotlin/gltf/GltfScene.kt index 432381f4..cb1807ea 100644 --- a/orx-dnk3/src/main/kotlin/gltf/GltfScene.kt +++ b/orx-dnk3/src/main/kotlin/gltf/GltfScene.kt @@ -10,10 +10,18 @@ import org.openrndr.math.Quaternion import org.openrndr.math.Vector3 import org.openrndr.math.transforms.transform import java.io.File +import java.nio.Buffer import java.nio.ByteOrder import kotlin.reflect.KMutableProperty0 class SceneAnimation(var channels: List) { + + val duration: Double + get() { + return channels.maxBy { it.duration }?.duration ?:0.0 + } + + fun applyToTargets(input: Double) { for (channel in channels) { channel.applyToTarget(input) @@ -22,6 +30,7 @@ class SceneAnimation(var channels: List) { } sealed class AnimationChannel { + abstract val duration: Double abstract fun applyToTarget(input: Double) } @@ -30,6 +39,9 @@ class QuaternionChannel(val target: KMutableProperty0, override fun applyToTarget(input: Double) { target.set(keyframer.value(input) ?: Quaternion.IDENTITY) } + + override val duration: Double + get() = keyframer.duration() } class Vector3Channel(val target: KMutableProperty0, @@ -37,6 +49,8 @@ class Vector3Channel(val target: KMutableProperty0, override fun applyToTarget(input: Double) { target.set(keyframer.value(input) ?: default) } + override val duration: Double + get() = keyframer.duration() } class GltfSceneNode : SceneNode() { @@ -44,6 +58,11 @@ class GltfSceneNode : SceneNode() { var scale = Vector3.ONE var rotation = Quaternion.IDENTITY + override fun toString(): String { + return "translation: $translation, scale: $scale, rotation: $rotation, children: ${children.size}, entities: ${entities} " + } + + override var transform: Matrix44 = Matrix44.IDENTITY get() = transform { translate(translation) @@ -93,6 +112,12 @@ fun GltfFile.buildSceneNodes(): GltfSceneData { pbrMetallicRoughness?.let { pbr -> material.roughness = pbr.roughnessFactor ?: 1.0 material.metalness = pbr.metallicFactor ?: 1.0 + + material.color = ColorRGBa.WHITE + pbr.baseColorFactor?.let { + material.color = ColorRGBa(it[0], it[1], it[2], it[3]) + } + pbr.baseColorTexture?.let { texture -> val cb = images!![textures!![texture.index].source].createSceneImage() cb.filter(MinifyingFilter.LINEAR_MIPMAP_LINEAR, MagnifyingFilter.LINEAR) @@ -181,32 +206,10 @@ fun GltfFile.buildSceneNodes(): GltfSceneData { return MeshPrimitive(geometry, material) } - val sceneMeshes = mutableMapOf() - fun GltfMesh.createSceneMesh(): Mesh = sceneMeshes.getOrPut(this) { - Mesh(primitives.map { - it.createScenePrimitive() - }) - } val sceneNodes = mutableMapOf() fun GltfNode.createSceneNode(): SceneNode = sceneNodes.getOrPut(this) { val node = GltfSceneNode() - mesh?.let { - node.entities.add(meshes[it].createSceneMesh()) - } -// val localTransform = transform { -// translation?.let { -// translate(it[0], it[1], it[2]) -// } -// rotation?.let { -// val q = Quaternion(it[0], it[1], it[2], it[3]) -// multiply(q.matrix.matrix44) -// } -// scale?.let { -// scale(it[0], it[1], it[2]) -// } -// } - node.translation = translation?.let { Vector3(it[0], it[1], it[2]) } ?: Vector3.ZERO node.scale = scale?.let { Vector3(it[0], it[1], it[2]) } ?: Vector3.ONE node.rotation = rotation?.let { Quaternion(it[0], it[1], it[2], it[3]) } ?: Quaternion.IDENTITY @@ -214,20 +217,66 @@ fun GltfFile.buildSceneNodes(): GltfSceneData { matrix?.let { node.transform = Matrix44.fromDoubleArray(it).transposed } - - -// node.transform = this.matrix?.let { -// Matrix44.fromDoubleArray(it).transposed -// } ?: localTransform for (child in children.orEmpty) { node.children.add(nodes[child].createSceneNode()) } node } + val sceneMeshes = mutableMapOf() + fun GltfMesh.createSceneMesh(skin: GltfSkin?): MeshBase = sceneMeshes.getOrPut(this) { + if (skin == null) { + Mesh(primitives.map { + it.createScenePrimitive() + }) + } else { + val joints = skin.joints.map { nodes[it].createSceneNode() } + val skeleton = nodes[skin.skeleton].createSceneNode() + val ibmAccessor = accessors[skin.inverseBindMatrices] + val ibmBufferView = bufferViews[ibmAccessor.bufferView] + val ibmBuffer = buffers[ibmBufferView.buffer] + + val ibmData = ibmBuffer.contents(this@buildSceneNodes) + ibmData.order(ByteOrder.nativeOrder()) + (ibmData as Buffer).position(ibmAccessor.byteOffset + (ibmBufferView.byteOffset ?: 0)) + + require(ibmAccessor.type == "MAT4") + require(ibmAccessor.componentType == GLTF_FLOAT) + require(ibmAccessor.count == joints.size) + val ibms = (0 until ibmAccessor.count).map { + val array = DoubleArray(16) + for (i in 0 until 16) { + array[i] = ibmData.float.toDouble() + } + + val m = Matrix44.fromDoubleArray(array).transposed + println("--") + println(m) + println(array.joinToString(",")) + + m + } + + SkinnedMesh(primitives.map { + it.createScenePrimitive() + }, joints, skeleton, ibms) + } + } + val scenes = scenes.map { scene -> + println(scene.nodes.size) scene.nodes.map { node -> - nodes[node].createSceneNode() + println("node: $node") + val gltfNode = nodes[node] + val sceneNode = gltfNode.createSceneNode() + sceneNode + } + } + for ((gltfNode, sceneNode) in sceneNodes) { + gltfNode.mesh?.let { + val skin = gltfNode.skin?.let { (skins!!)[it] } + println("adding mesh") + sceneNode.entities.add(meshes[it].createSceneMesh(skin)) } } @@ -276,7 +325,7 @@ fun GltfFile.buildSceneNodes(): GltfSceneData { } val keyframer = KeyframerChannelQuaternion() val inputOffset = (inputBufferView.byteOffset ?: 0) + (inputAccessor.byteOffset ?: 0) - val outputOffset = (outputBufferView.byteOffset ?: 0) + (inputAccessor.byteOffset ?: 0) + val outputOffset = (outputBufferView.byteOffset ?: 0) + (outputAccessor.byteOffset ?: 0) val inputStride = (inputBufferView.byteStride ?: 4) val outputStride = (outputBufferView.byteStride ?: 16) for (i in 0 until outputAccessor.count) {