Merge pull request #37 from ricardomatias/orx-glslify-preprocessor

Swith orx-glslify to preprocessor usage

orx-glslify/README.md

@@ -4,9 +4,6 @@ Easily use glslify compatible shaders found on [npm](https://www.npmjs.com/searc
 
 ### Caveats
 
-Some glslify shaders have their own imports. When this happens we print a message to the console,
-so you can proceed to import them. These need to be imported in the shader file on top of the main import.
-
 There's also a mapping functionality that glslify provides that we don't support. This can be easily solved
 by doing as the following example (based on `glsl-raytrace` package):
 
@@ -14,26 +11,11 @@ by doing as the following example (based on `glsl-raytrace` package):
 const int steps = 50;
 vec2 map(vec3 p);
 
-#pragma import shaders.RayMarching.*
+#pragma glslify: raytrace = require(glsl-raytrace)
 ```
 
 ## Example
 
-Shader Phrases file:
-```kotlin
-@file:JvmName("Checkers")
-@file:ShaderPhrases
-
-package shaders
-
-import org.openrndr.extra.glslify.glslify
-import org.openrndr.extra.shaderphrases.annotations.ShaderPhrases
-
-val periodic by lazy { glslify("glsl-noise/classic/3d", "perlin") }
-val checker by lazy { glslify("glsl-checker") }
-val easings by lazy { glslify("glsl-easings/cubic-in-out", "easing")}
-```
-
 Shader file:
 
 ```glsl
@@ -46,7 +28,9 @@ uniform float uTime;
 
 out vec4 o_color;
 
-#pragma import shaders.Checkers.*
+#pragma glslify: checker = require(glsl-checker)
+#pragma glslify: perlin = require(glsl-noise/classic/3d)
+#pragma glslify: easing = require(glsl-easings/cubic-in-out)
 
 void main() {
     vec2 uv = v_texCoord0;
@@ -59,3 +43,15 @@ void main() {
     o_color = vec4(col, 1.0);
 }
 ```
+
+Then preprocess it with either: 
+
+```kotlin
+preprocessGlslifyFromUrl(resourceUrl("/shaders/ray-marching.glsl"))
+```
+
+or
+
+```kotlin
+preprocessGlslify("""version #330 ...""")
+```

orx-glslify/src/main/kotlin/Glslify.kt

@@ -1,8 +1,8 @@
 package org.openrndr.extra.glslify
 
 import mu.KotlinLogging
+import org.openrndr.draw.codeFromURL
 import java.io.ByteArrayInputStream
-import java.io.File
 import java.io.InputStream
 import java.net.URL
 import java.nio.file.Files
@@ -12,28 +12,48 @@ import javax.net.ssl.HttpsURLConnection
 import kotlin.streams.toList
 
 internal const val BASE_URL = "https://registry.npmjs.org"
-internal const val GLSLIFY_PATH = "src/main/resources/glslify"
-internal const val IMPORT_PATT = """#pragma\sglslify:\s*(.*)\s=\s*require\('?([\w\-\\/]+).*'?\)$"""
+internal const val IMPORT_PATT = """#pragma\sglslify:\s([\w]+).*\s=\s*require\('?([\w\-\\/]+).*'?\)$"""
 internal const val EXPORT_PATT = """#pragma\sglslify:\s*export\(([\w\-]+)\)"""
+internal const val PRAGMA_IDENTIFIER = "#pragma glslify"
 
 internal val shaderExtensions = arrayOf("glsl", "frag")
 
-private data class GlslifyImport(
-        val functionName: String,
-        val pkgName: String,
-        var exists: Boolean
-)
+internal val logger = KotlinLogging.logger {}
 
-private val logger = KotlinLogging.logger {}
+private var importTree = mutableSetOf<String>()
 
-fun glslify(module: String, renameFunctionTo: String? = null): String {
-    val (moduleName: String, shaderPath: Path) = parseModule(module)
-    val moduleDir = Paths.get("$GLSLIFY_PATH/$moduleName")
-
-    return glslifyImport(moduleName, moduleDir, shaderPath, renameFunctionTo)
+fun preprocessGlslifyFromUrl(url: String, glslifyPath: String = "src/main/resources/glslify"): String {
+    return preprocessGlslify(codeFromURL(url), glslifyPath)
 }
 
-private fun glslifyImport(moduleName: String, moduleDir: Path, shaderPath: Path, renameFunctionTo: String? = null): String {
+
+fun preprocessGlslify(source: String, glslifyPath: String = "src/main/resources/glslify"): String {
+    importTree = mutableSetOf()
+
+    return source.split("\n").map { line ->
+        if (line.startsWith(PRAGMA_IDENTIFIER)) {
+            Regex(IMPORT_PATT).find(line)?.let {
+                if (it.groupValues.size > 1) {
+                    val functionName = it.groupValues[1]
+                    val module = it.groupValues[2]
+
+                    val (moduleName, shaderPath) = parseModule(module)
+                    val importPath = Paths.get(glslifyPath).resolve(moduleName)
+
+                    if (importTree.contains(moduleName)) return@map ""
+
+                    importTree.add(moduleName)
+
+                    return@map glslifyImport(moduleName, importPath, shaderPath, functionName, glslifyPath)
+                }
+            }
+        }
+
+        line
+    }.joinToString("\n").trimMargin()
+}
+
+private fun glslifyImport(moduleName: String, moduleDir: Path, shaderPath: Path, renameFunctionTo: String? = null, glslifyPath: String): String {
     if (!Files.exists(moduleDir)) {
         fetchModule(moduleName, moduleDir)
     } else {
@@ -47,51 +67,44 @@ private fun glslifyImport(moduleName: String, moduleDir: Path, shaderPath: Path,
             moduleDir.resolve("$shaderPath.$it")
         }.first { Files.exists(it) }
     } catch (ex: NoSuchElementException) {
-        logger.trace("[glslify] $moduleName: $shaderPath doesn't lead to any shader file")
-
-        return ""
+        throw error("[glslify] $moduleName: $shaderPath doesn't lead to any shader file")
     }
 
-    val imports = mutableListOf<GlslifyImport>()
     var exportName: String? = null
 
-    var shader = Files.lines(shaderFile).filter { line ->
-        line.contains("#pragma").let { isPragma ->
-            if (isPragma) {
-                Regex(IMPORT_PATT).find(line)?.let {
-                    if (it.groupValues.size > 1) {
-                        val importPath = Paths.get(GLSLIFY_PATH).resolve(it.groupValues[2])
+    var shader = Files.lines(shaderFile).map { line ->
+        if (line.startsWith(PRAGMA_IDENTIFIER)) {
+            Regex(IMPORT_PATT).find(line)?.let {
+                val functionName = it.groupValues[1]
+                val module = it.groupValues[2]
 
-                        imports.add(GlslifyImport(it.groupValues[1], it.groupValues[2], Files.exists(importPath)))
-                    }
-                }
+                val (innerModuleName, innerShaderPath) = parseModule(module)
+                val importPath = Paths.get(glslifyPath).resolve(moduleName)
 
-                Regex(EXPORT_PATT).find(line)?.let {
-                    if (it.groupValues.size > 1) {
-                        exportName = it.groupValues[1]
-                    }
-                }
+                if (importTree.contains(innerModuleName)) return@map ""
+
+                importTree.add(innerModuleName)
+
+                return@map glslifyImport(innerModuleName, importPath, innerShaderPath, functionName, glslifyPath)
             }
 
-            !isPragma // we want to exclude any #pragma statements
+            Regex(EXPORT_PATT).find(line)?.let {
+                if (it.groupValues.size > 1) {
+                    exportName = it.groupValues[1]
+                }
+
+                return@map ""
+            }
         }
+
+        line
     }.toList().joinToString("\n")
 
-    val missingImports = imports.filter { !it.exists }
-
-    if (missingImports.isNotEmpty()) {
-        missingImports.forEach {
-            logger.info("Missing package: ${it.pkgName} - Import name: ${it.functionName}")
-        }
-
-        throw error("[glslify] Please declare the missing imports")
-    }
-
     if (renameFunctionTo != null && exportName != null) {
         shader = shader.replace( exportName!!, renameFunctionTo)
     }
 
-    return shader
+    return shader.trimMargin()
 }
 
 internal fun fetchModule(moduleName: String, moduleDir: Path) {
@@ -119,4 +132,4 @@ internal fun fetchModule(moduleName: String, moduleDir: Path) {
     } catch (ex: Exception) {
         logger.error(ex) { "[glslify]: There was an error getting $moduleName" }
     }
-}
+}

orx-glslify/src/test/kotlin/TestGlslify.kt

@@ -0,0 +1,85 @@
+import org.amshove.kluent.*
+import org.openrndr.extra.glslify.preprocessGlslify
+import org.openrndr.extra.glslify.preprocessGlslifyFromUrl
+import org.openrndr.resourceUrl
+import org.spekframework.spek2.Spek
+import org.spekframework.spek2.style.specification.describe
+
+const val glslifyPath = "src/test/resources/glslify"
+
+object TestGlslify : Spek({
+    describe("glslify") {
+        describe("should download shader with full path") {
+            val shader = """#version 330
+#pragma glslify: perlin = require(glsl-noise/classic/3d)
+"""
+            val processed = preprocessGlslify(shader, glslifyPath = glslifyPath)
+
+            processed `should contain` "float perlin(vec3 P)"
+        }
+
+        describe("should download shader with just the module's name") {
+            val shader = """#version 330
+#pragma glslify: luma = require(glsl-luma)
+"""
+            val processed = preprocessGlslify(shader, glslifyPath = glslifyPath)
+
+            processed `should contain` "float luma(vec4 color)"
+        }
+
+        describe("throws exception when the import doesn't lead to a shader file") {
+            invoking {
+                val shader = """#version 330
+#pragma glslify: perlin = require(glsl-noise)
+"""
+                preprocessGlslify(shader, glslifyPath = glslifyPath)
+            } `should throw` RuntimeException::class `with message`
+                    ("[glslify] glsl-noise: index doesn't lead to any shader file")
+        }
+    }
+
+    describe("preprocessGlslify") {
+        describe("should import everything in order + write naming") {
+            val url = resourceUrl("/a.glsl")
+            val processed = preprocessGlslifyFromUrl(url, glslifyPath = glslifyPath)
+
+            processed shouldContainAll listOf(
+                "float add(float a, float b)",
+                "float multiply(float a, float b)",
+                "float equation(float a, float b)",
+                "float luminance(vec3 color)",
+                "void main()"
+            )
+        }
+
+        describe("should import complex") {
+            val url = resourceUrl("/complex.glsl")
+            val processed = preprocessGlslifyFromUrl(url, glslifyPath = glslifyPath)
+
+            processed shouldContainAll listOf(
+                    "float checker(vec2 uv, float repeats)",
+                    "float noise3d(vec3 P)",
+                    "float easing(float t)",
+                    "void main()"
+            )
+        }
+
+        describe("should import only once") {
+            val shader = """#version 330
+#pragma glslify: luma = require(glsl-luma)
+#pragma glslify: luma = require(glsl-luma)
+#pragma glslify: luma = require(glsl-luma)
+"""
+            val processed = preprocessGlslify(shader, glslifyPath = glslifyPath).trimEnd()
+
+            processed shouldBeEqualTo """#version 330
+float luma(vec3 color) {
+  return dot(color, vec3(0.299, 0.587, 0.114));
+}
+
+float luma(vec4 color) {
+  return dot(color.rgb, vec3(0.299, 0.587, 0.114));
+}"""
+        }
+    }
+})

orx-glslify/src/test/resources/a-result.glsl

@@ -0,0 +1,27 @@
+#version 330
+
+float add(float a, float b) {
+    return a + b;
+}
+
+float multiply(float a, float b) {
+    return a * b;
+}
+
+
+float equation(float a, float b) {
+    return multiply(add(a, b), b);
+}
+
+float luminance(vec3 color) {
+    return dot(color, vec3(0.299, 0.587, 0.114));
+}
+
+float luminance(vec4 color) {
+    return dot(color.rgb, vec3(0.299, 0.587, 0.114));
+}
+
+
+void main() {
+
+}

orx-glslify/src/test/resources/a.glsl

@@ -0,0 +1,8 @@
+#version 330
+
+#pragma glslify: equation = require(solve)
+#pragma glslify: luminance = require(glsl-luma)
+
+void main() {
+
+}

orx-glslify/src/test/resources/complex.glsl

@@ -0,0 +1,9 @@
+#version 330
+
+#pragma glslify: checker = require(glsl-checker, foo = bar)
+#pragma glslify: noise3d = require('glsl-noise/classic/3d')
+#pragma glslify: easing = require('glsl-easings/cubic-in-out')
+
+void main() {
+
+}

orx-glslify/src/test/resources/glslify/glsl-checker/index.glsl

@@ -0,0 +1,8 @@
+float checker(vec2 uv, float repeats) {
+  float cx = floor(repeats * uv.x);
+  float cy = floor(repeats * uv.y); 
+  float result = mod(cx + cy, 2.0);
+  return sign(result);
+}
+
+#pragma glslify: export(checker)

orx-glslify/src/test/resources/glslify/glsl-easings/cubic-in-out.glsl

@@ -0,0 +1,7 @@
+float cubicInOut(float t) {
+  return t < 0.5
+    ? 4.0 * t * t * t
+    : 0.5 * pow(2.0 * t - 2.0, 3.0) + 1.0;
+}
+
+#pragma glslify: export(cubicInOut)

orx-glslify/src/test/resources/glslify/glsl-luma/index.glsl

@@ -0,0 +1,9 @@
+float luma(vec3 color) {
+  return dot(color, vec3(0.299, 0.587, 0.114));
+}
+
+float luma(vec4 color) {
+  return dot(color.rgb, vec3(0.299, 0.587, 0.114));
+}
+
+#pragma glslify: export(luma)

orx-glslify/src/test/resources/glslify/glsl-noise/classic/3d.glsl

@@ -0,0 +1,109 @@
+//
+// GLSL textureless classic 3D noise "cnoise",
+// with an RSL-style periodic variant "pnoise".
+// Author:  Stefan Gustavson (stefan.gustavson@liu.se)
+// Version: 2011-10-11
+//
+// Many thanks to Ian McEwan of Ashima Arts for the
+// ideas for permutation and gradient selection.
+//
+// Copyright (c) 2011 Stefan Gustavson. All rights reserved.
+// Distributed under the MIT license. See LICENSE file.
+// https://github.com/ashima/webgl-noise
+//
+
+vec3 mod289(vec3 x)
+{
+  return x - floor(x * (1.0 / 289.0)) * 289.0;
+}
+
+vec4 mod289(vec4 x)
+{
+  return x - floor(x * (1.0 / 289.0)) * 289.0;
+}
+
+vec4 permute(vec4 x)
+{
+  return mod289(((x*34.0)+1.0)*x);
+}
+
+vec4 taylorInvSqrt(vec4 r)
+{
+  return 1.79284291400159 - 0.85373472095314 * r;
+}
+
+vec3 fade(vec3 t) {
+  return t*t*t*(t*(t*6.0-15.0)+10.0);
+}
+
+// Classic Perlin noise
+float cnoise(vec3 P)
+{
+  vec3 Pi0 = floor(P); // Integer part for indexing
+  vec3 Pi1 = Pi0 + vec3(1.0); // Integer part + 1
+  Pi0 = mod289(Pi0);
+  Pi1 = mod289(Pi1);
+  vec3 Pf0 = fract(P); // Fractional part for interpolation
+  vec3 Pf1 = Pf0 - vec3(1.0); // Fractional part - 1.0
+  vec4 ix = vec4(Pi0.x, Pi1.x, Pi0.x, Pi1.x);
+  vec4 iy = vec4(Pi0.yy, Pi1.yy);
+  vec4 iz0 = Pi0.zzzz;
+  vec4 iz1 = Pi1.zzzz;
+
+  vec4 ixy = permute(permute(ix) + iy);
+  vec4 ixy0 = permute(ixy + iz0);
+  vec4 ixy1 = permute(ixy + iz1);
+
+  vec4 gx0 = ixy0 * (1.0 / 7.0);
+  vec4 gy0 = fract(floor(gx0) * (1.0 / 7.0)) - 0.5;
+  gx0 = fract(gx0);
+  vec4 gz0 = vec4(0.5) - abs(gx0) - abs(gy0);
+  vec4 sz0 = step(gz0, vec4(0.0));
+  gx0 -= sz0 * (step(0.0, gx0) - 0.5);
+  gy0 -= sz0 * (step(0.0, gy0) - 0.5);
+
+  vec4 gx1 = ixy1 * (1.0 / 7.0);
+  vec4 gy1 = fract(floor(gx1) * (1.0 / 7.0)) - 0.5;
+  gx1 = fract(gx1);
+  vec4 gz1 = vec4(0.5) - abs(gx1) - abs(gy1);
+  vec4 sz1 = step(gz1, vec4(0.0));
+  gx1 -= sz1 * (step(0.0, gx1) - 0.5);
+  gy1 -= sz1 * (step(0.0, gy1) - 0.5);
+
+  vec3 g000 = vec3(gx0.x,gy0.x,gz0.x);
+  vec3 g100 = vec3(gx0.y,gy0.y,gz0.y);
+  vec3 g010 = vec3(gx0.z,gy0.z,gz0.z);
+  vec3 g110 = vec3(gx0.w,gy0.w,gz0.w);
+  vec3 g001 = vec3(gx1.x,gy1.x,gz1.x);
+  vec3 g101 = vec3(gx1.y,gy1.y,gz1.y);
+  vec3 g011 = vec3(gx1.z,gy1.z,gz1.z);
+  vec3 g111 = vec3(gx1.w,gy1.w,gz1.w);
+
+  vec4 norm0 = taylorInvSqrt(vec4(dot(g000, g000), dot(g010, g010), dot(g100, g100), dot(g110, g110)));
+  g000 *= norm0.x;
+  g010 *= norm0.y;
+  g100 *= norm0.z;
+  g110 *= norm0.w;
+  vec4 norm1 = taylorInvSqrt(vec4(dot(g001, g001), dot(g011, g011), dot(g101, g101), dot(g111, g111)));
+  g001 *= norm1.x;
+  g011 *= norm1.y;
+  g101 *= norm1.z;
+  g111 *= norm1.w;
+
+  float n000 = dot(g000, Pf0);
+  float n100 = dot(g100, vec3(Pf1.x, Pf0.yz));
+  float n010 = dot(g010, vec3(Pf0.x, Pf1.y, Pf0.z));
+  float n110 = dot(g110, vec3(Pf1.xy, Pf0.z));
+  float n001 = dot(g001, vec3(Pf0.xy, Pf1.z));
+  float n101 = dot(g101, vec3(Pf1.x, Pf0.y, Pf1.z));
+  float n011 = dot(g011, vec3(Pf0.x, Pf1.yz));
+  float n111 = dot(g111, Pf1);
+
+  vec3 fade_xyz = fade(Pf0);
+  vec4 n_z = mix(vec4(n000, n100, n010, n110), vec4(n001, n101, n011, n111), fade_xyz.z);
+  vec2 n_yz = mix(n_z.xy, n_z.zw, fade_xyz.y);
+  float n_xyz = mix(n_yz.x, n_yz.y, fade_xyz.x);
+  return 2.2 * n_xyz;
+}
+
+#pragma glslify: export(cnoise)

orx-glslify/src/test/resources/glslify/multiply/index.glsl

@@ -0,0 +1,5 @@
+float multiply(float a, float b) {
+    return a * b;
+}
+
+#pragma glslify: export(multiply)

orx-glslify/src/test/resources/glslify/solve/index.glsl

@@ -0,0 +1,8 @@
+#pragma glslify: add = require(sum)
+#pragma glslify: multiply = require(multiply)
+
+float solve(float a, float b) {
+    return multiply(add(a, b), b);
+}
+
+#pragma glslify: export(solve)

orx-glslify/src/test/resources/glslify/sum/index.glsl

@@ -0,0 +1,5 @@
+float sum(float a, float b) {
+    return a + b;
+}
+
+#pragma glslify: export(sum)