Merge branch 'main' into PR/DebugUninitFalsePositives

AlexMWells · AlexMWells · commit 0eb25de3dc79 · 2025-02-24T15:18:33.000-08:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -330,7 +330,7 @@ jobs:
             fmt_ver: 10.1.1
             pybind11_ver: v2.10.0
             skip_tests: 1
-            abi_check: 42ccf223af1e929fd84a13773e8748b719b1e595
+            abi_check: e3aa2cf5a17a59e906c7125ff8d546b1561e812f
             setenvs: export OSL_CMAKE_FLAGS="-DOSL_BUILD_TESTS=0 -DBUILD_TESTING=OFF -DUSE_QT=0"
                             OPENIMAGEIO_CMAKE_FLAGS="-DUSE_PYTHON=0"
                             CMAKE_BUILD_TYPE=RelWithDebInfo
@@ -378,30 +378,30 @@ jobs:
                             USE_OPENVDB=0
                             OPENCOLORIO_CMAKE_FLAGS="-DCMAKE_CXX_COMPILER=g++"
 
-          - desc: Debug gcc7/C++17 llvm11 py3.8 oiio2.5 exr3.1 sse4
+          - desc: Debug gcc9/C++17 llvm11 py3.10 oiio2.5 exr3.1 sse4
             nametag: linux-debug-gcc7-llvm11
-            runner: ubuntu-20.04
+            runner: ubuntu-22.04
             cxx_compiler: g++-9
             cxx_std: 17
             openexr_ver: v3.1.11
             openimageio_ver: v2.5.4.0
             pybind11_ver: v2.7.0
-            python_ver: 3.8
+            python_ver: "3.10"
             simd: sse4.2
             setenvs: export CMAKE_BUILD_TYPE=Debug
                             LLVM_VERSION=11.0.0 LLVM_DISTRO_NAME=ubuntu-20.04
                             PUGIXML_VERSION=v1.9
                             CTEST_TEST_TIMEOUT=240
           - desc: gcc10/C++17 llvm11 oiio-2.5 avx2
             nametag: linux-2021ish-gcc10-llvm11
-            runner: ubuntu-20.04
+            runner: ubuntu-22.04
             cxx_compiler: g++-10
             cxx_std: 17
             fmt_ver: 7.0.1
             openexr_ver: v3.1.11
             openimageio_ver: v2.5.17.0
             pybind11_ver: v2.8.1
-            python_ver: 3.8
+            python_ver: "3.10"
             simd: avx2,f16c
             setenvs: export LLVM_VERSION=11.0.0 LLVM_DISTRO_NAME=ubuntu-20.04
                             OPENIMAGEIO_CMAKE_FLAGS="-DBUILD_FMT_VERSION=7.0.1"
@@ -426,7 +426,7 @@ jobs:
                             PTEX_VERSION=v2.4.3
                             PUGIXML_VERSION=v1.14
                             FREETYPE_VERSION=VER-2-13-3
-          - desc: bleeding edge gcc12/C++17 llvm17 oiio/ocio/exr/pybind-main py3.12 avx2 batch-b16avx512
+          - desc: bleeding edge gcc13/C++17 llvm17 oiio/ocio/exr/pybind-main py3.12 avx2 batch-b16avx512
             nametag: linux-bleeding-edge
             runner: ubuntu-24.04
             cc_compiler: gcc-13
@@ -443,9 +443,9 @@ jobs:
             setenvs: export LLVM_VERSION=17.0.6
                             LLVM_DISTRO_NAME=ubuntu-22.04
                             PUGIXML_VERSION=master
-          - desc: clang14/C++17 llvm14 py3.8 avx2 batch-b16avx512
+          - desc: clang14/C++17 llvm14 py3.10 avx2 batch-b16avx512
             nametag: linux-latest-releases-clang
-            runner: ubuntu-20.04
+            runner: ubuntu-22.04
             cxx_compiler: clang++
             cc_compiler: clang
             cxx_std: 17
@@ -454,7 +454,7 @@ jobs:
             openexr_ver: v3.1.11
             openimageio_ver: release
             pybind11_ver: v2.9.2
-            python_ver: 3.8
+            python_ver: "3.10"
             simd: avx2,f16c
             batched: b8_AVX2,b8_AVX512,b16_AVX512
             setenvs: export LLVM_VERSION=14.0.0
diff --git a/CHANGES.md b/CHANGES.md
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -4,7 +4,7 @@
 
 cmake_minimum_required (VERSION 3.19)
 
-set (OSL_VERSION "1.14.4.0")
+set (OSL_VERSION "1.15.0.0")
 set (OSL_VERSION_OVERRIDE "" CACHE STRING
      "Version override (use with caution)!")
 mark_as_advanced (OSL_VERSION_OVERRIDE)
diff --git a/INSTALL.md b/INSTALL.md
@@ -114,8 +114,8 @@ Here are the steps to check out, build, and test the OSL distribution:
 4. After compilation, you'll end up with a full OSL distribution in
    dist/
 
-5. Add the "dist/bin" to your $PATH, and "dist/lib" to your
-   $LD_LIBRAY_PATH (or $DYLD_LIBRARY_PATH on MacOS), or copy the contents
+5. Add the "dist/bin" to your `$PATH`, and "dist/lib" to your
+   `$LD_LIBRAY_PATH` (or `$DYLD_LIBRARY_PATH` on MacOS), or copy the contents
    of those files to appropriate directories.  Public include files
    (those needed when building applications that incorporate OSL)
    can be found in "dist/include", and documentation can be found
diff --git a/src/doc/datatypes.md b/src/doc/datatypes.md
@@ -558,6 +558,8 @@ are 1D and statically sized, using the usual syntax for C-like languages:
     float c[3] = { 0.1, 0.2, 3.14 };   // Initialize the array
 
     float f = c[1];                    // Access one element
+
+    float d[10][3];                    // Invalid, multi-dimensional arrays not supported
 ```
 
 The built-in function `arraylength()` returns the number of elements in an
diff --git a/src/doc/syntax.md b/src/doc/syntax.md
@@ -88,9 +88,9 @@ Some examples of variable declarations are
 
 Arrays are also supported, declared as follows:
 
-> *type variablename* `[` *arraylen* `}`
+> *type variablename* `[` *arraylen* `]`
 >
-> *type variablename* `[` *arraylen* `}` `=` `{` *init0 `,` *init1* ... `}`
+> *type variablename* `[` *arraylen* `]` `=` `{` *init0 `,` *init1* ... `}`
 
 Array variables in OSL must have a constant length (though function parameters
 and shader parameters may have undetermined length).  Some examples of array
diff --git a/src/include/OSL/rendererservices.h b/src/include/OSL/rendererservices.h
@@ -580,6 +580,19 @@ class OSLEXECPUBLIC RendererServices {
         }
     };
 
+    // Default no-op implementations of the caching api.
+    // Currently used for caching optix ptx before llvm generation.
+    virtual void cache_insert(string_view cachename, string_view key,
+                              string_view value) const
+    {
+    }
+
+    virtual bool cache_get(string_view cachename, string_view key,
+                           std::string& value) const
+    {
+        return false;
+    }
+
     /// A renderer may choose to support batched execution by providing pointers
     /// to objects satisfying the BatchedRendererServices<WidthOf<#>> interface
     /// for specific batch sizes.
diff --git a/src/liboslexec/backendllvm.h b/src/liboslexec/backendllvm.h
@@ -524,6 +524,7 @@ class BackendLLVM final : public OSOProcessorBase {
 
     /// Return whether or not we are compiling for an OptiX-based renderer.
     bool use_optix() { return m_use_optix; }
+    bool use_optix_cache() { return shadingsys().use_optix_cache(); }
 
     /// Return if we should compile against free function versions of Renderer Service.
     bool use_rs_bitcode() { return m_use_rs_bitcode; }
diff --git a/src/liboslexec/instance.cpp b/src/liboslexec/instance.cpp
@@ -848,6 +848,25 @@ ShaderGroup::setup_interactive_arena(cspan<uint8_t> paramblock)
 
 
 
+void
+ShaderGroup::generate_optix_cache_key(string_view code)
+{
+    const uint64_t ir_key = Strutil::strhash(code);
+
+    std::string safegroup;
+    safegroup = Strutil::replace(name(), "/", "_", true);
+    safegroup = Strutil::replace(safegroup, ":", "_", true);
+
+    // Cache key includes the groupname in addition to the serialized IR.
+    // This is because the groupname makes its way into the ptx's direct callable name,
+    // but isn't included in the serialization.
+    std::string cache_key = fmtformat("cache-osl-ptx-{}-{}", safegroup, ir_key);
+
+    m_optix_cache_key = cache_key;
+}
+
+
+
 std::string
 ShaderGroup::serialize() const
 {
diff --git a/src/liboslexec/llvm_gen.cpp b/src/liboslexec/llvm_gen.cpp
@@ -426,10 +426,13 @@ LLVMGEN(llvm_gen_printf_legacy)
     }
 #endif
 
-    // Some ops prepend things
-    if (op.opname() == op_error || op.opname() == op_warning) {
-        s = fmtformat("Shader {} [{}]: {}", op.opname(),
-                      rop.inst()->shadername(), s);
+    // TODO: optix cache should handle ustrings generated during llvm-gen
+    if (!rop.use_optix_cache()) {
+        // Some ops prepend things
+        if (op.opname() == op_error || op.opname() == op_warning) {
+            s = fmtformat("Shader {} [{}]: {}", op.opname(),
+                          rop.inst()->shadername(), s);
+        }
     }
 
     // Now go back and put the new format string in its place
@@ -709,10 +712,12 @@ LLVMGEN(llvm_gen_print_fmt)
             }
         }
     }
-    // Some ops prepend things
-    if (op.opname() == op_error || op.opname() == op_warning) {
-        s = fmtformat("Shader {} [{}]: {}", op.opname(),
-                      rop.inst()->shadername(), s);
+    if (!rop.use_optix_cache()) {
+        // Some ops prepend things
+        if (op.opname() == op_error || op.opname() == op_warning) {
+            s = fmtformat("Shader {} [{}]: {}", op.opname(),
+                          rop.inst()->shadername(), s);
+        }
     }
     ustring s_ustring(s.c_str());
     call_args.push_back(rop.llvm_const_hash(s_ustring));
diff --git a/src/liboslexec/llvm_instance.cpp b/src/liboslexec/llvm_instance.cpp
@@ -2500,6 +2500,14 @@ BackendLLVM::run()
                 group().llvm_compiled_layer(nlayers - 1));
     }
 
+    if (shadingsys().use_optix_cache()) {
+        std::string cache_key = group().optix_cache_key();
+        renderer()->cache_insert(
+            "optix_ptx", cache_key,
+            optix_cache_wrap(group().m_llvm_ptx_compiled_version,
+                             group().llvm_groupdata_size()));
+    }
+
     // We are destroying the entire module below,
     // no reason to bother destroying individual functions
 #if 0
diff --git a/src/liboslexec/oslexec.cpp b/src/liboslexec/oslexec.cpp
@@ -51,5 +51,32 @@ shadertype_from_name(string_view name)
 }
 
 
+
+std::string
+optix_cache_wrap(string_view ptx, size_t groupdata_size)
+{
+    // Cache string is the ptx file with groupdata size on top as a comment.
+    // This way the cache string is a valid ptx program, which can be useful
+    // for debugging.
+    return fmtformat("// {}\n{}", groupdata_size, ptx);
+}
+
+
+
+void
+optix_cache_unwrap(string_view cache_value, std::string& ptx,
+                   size_t& groupdata_size)
+{
+    size_t groupdata_end_index = cache_value.find('\n');
+    if (groupdata_end_index != std::string::npos) {
+        constexpr int offset = 3;  // Account for the "// " prefix
+        std::string groupdata_string
+            = cache_value.substr(offset, groupdata_end_index - offset);
+        groupdata_size = std::stoll(groupdata_string);
+
+        ptx = cache_value.substr(groupdata_end_index + 1);
+    }
+}
+
 };  // namespace pvt
 OSL_NAMESPACE_END
diff --git a/src/liboslexec/oslexec_pvt.h b/src/liboslexec/oslexec_pvt.h
@@ -81,6 +81,12 @@ struct PerThreadInfo {
 
 namespace pvt {
 
+void
+optix_cache_unwrap(string_view cache_value, std::string& ptx,
+                   size_t& groupdata_size);
+std::string
+optix_cache_wrap(string_view ptx, size_t groupdata_size);
+
 // forward definitions
 class ShadingSystemImpl;
 class ShaderInstance;
@@ -632,6 +638,7 @@ class ShadingSystemImpl {
     TextureSystem* texturesys() const { return m_texturesys; }
 
     bool use_optix() const { return m_use_optix; }
+    bool use_optix_cache() const { return m_use_optix_cache; }
     bool debug_nan() const { return m_debugnan; }
     bool debug_uninit() const { return m_debug_uninit; }
     bool lockgeom_default() const { return m_lockgeom_default; }
@@ -954,9 +961,10 @@ class ShadingSystemImpl {
     std::vector<ustring> m_raytypes;          ///< Names of ray types
     std::vector<ustring> m_renderer_outputs;  ///< Names of renderer outputs
     std::vector<SymLocationDesc> m_symlocs;
-    int m_max_local_mem_KB;           ///< Local storage can a shader use
-    int m_compile_report;             ///< Print compilation report?
-    bool m_use_optix;                 ///< This is an OptiX-based renderer
+    int m_max_local_mem_KB;  ///< Local storage can a shader use
+    int m_compile_report;    ///< Print compilation report?
+    bool m_use_optix;        ///< This is an OptiX-based renderer
+    bool m_use_optix_cache;  ///< Renderer-enabled caching for OptiX ptx
     int m_max_optix_groupdata_alloc;  ///< Maximum OptiX groupdata buffer allocation
     bool m_buffer_printf;             ///< Buffer/batch printf output?
     bool m_no_noise;                  ///< Substitute trivial noise calls
@@ -1843,6 +1851,10 @@ class ShaderGroup {
     void name(ustring name) { m_name = name; }
     ustring name() const { return m_name; }
 
+    // Generate and memoize the cache key so we don't calculate it twice
+    void generate_optix_cache_key(string_view code);
+    std::string optix_cache_key() const { return m_optix_cache_key; }
+
     std::string serialize() const;
 
     void lock() const { m_mutex.lock(); }
@@ -2046,6 +2058,8 @@ class ShaderGroup {
     atomic_ll m_executions { 0 };  ///< Number of times the group executed
     atomic_ll m_stat_total_shading_time_ticks { 0 };  // Shading time (ticks)
 
+    std::string m_optix_cache_key;
+
     // PTX assembly for compiled ShaderGroup
     std::string m_llvm_ptx_compiled_version;
 
diff --git a/src/liboslexec/runtimeoptimize.cpp b/src/liboslexec/runtimeoptimize.cpp
@@ -3124,6 +3124,21 @@ RuntimeOptimizer::printinst(std::ostream& out) const
 
 
 
+std::string
+RuntimeOptimizer::serialize()
+{
+    std::ostringstream ss {};
+    int nlayers = (int)group().nlayers();
+    for (int layer = 0; layer < nlayers; ++layer) {
+        set_inst(layer);
+        printinst(ss);
+    }
+
+    return ss.str();
+}
+
+
+
 void
 RuntimeOptimizer::run()
 {
diff --git a/src/liboslexec/runtimeoptimize.h b/src/liboslexec/runtimeoptimize.h
@@ -460,6 +460,8 @@ class RuntimeOptimizer final : public OSOProcessorBase {
                        fmtformat(fmt, std::forward<Args>(args)...));
     }
 
+    std::string serialize();
+
 private:
     int m_optimize;                        ///< Current optimization level
     bool m_opt_simplify_param;             ///< Turn instance params into const?
diff --git a/src/liboslexec/shadingsys.cpp b/src/liboslexec/shadingsys.cpp
diff --git a/testsuite/render-microfacet/ref/out-optix-alt2.exr b/testsuite/render-microfacet/ref/out-optix-alt2.exr
diff --git a/testsuite/render-microfacet/run.py b/testsuite/render-microfacet/run.py
diff --git a/testsuite/render-mx-generalized-schlick/run.py b/testsuite/render-mx-generalized-schlick/run.py
diff --git a/testsuite/render-raytypes/run.py b/testsuite/render-raytypes/run.py
diff --git a/testsuite/runtest.py b/testsuite/runtest.py

Original file line number	Diff line number	Diff line change
`@@ -88,9 +88,9 @@ Some examples of variable declarations are`
`88`	`88`
`89`	`89`	`Arrays are also supported, declared as follows:`
`90`	`90`
`91`		-> type variablename `[` arraylen `}`
	`91`	+> type variablename `[` arraylen `]`
`92`	`92`	`>`
`93`		-> type variablename `[` arraylen `}` `=` `{` init0 `,` init1* ... `}`
	`93`	+> type variablename `[` arraylen `]` `=` `{` init0 `,` init1* ... `}`
`94`	`94`
`95`	`95`	`Array variables in OSL must have a constant length (though function parameters`
`96`	`96`	`and shader parameters may have undetermined length). Some examples of array`
Original file line number	Diff line number	Diff line change
`@@ -426,10 +426,13 @@ LLVMGEN(llvm_gen_printf_legacy)`
`426`	`426`	`}`
`427`	`427`	`#endif`
`428`	`428`
`429`		`- // Some ops prepend things`
`430`		`- if (op.opname() == op_error \|\| op.opname() == op_warning) {`
`431`		`- s = fmtformat("Shader {} [{}]: {}", op.opname(),`
`432`		`- rop.inst()->shadername(), s);`
	`429`	`+ // TODO: optix cache should handle ustrings generated during llvm-gen`
	`430`	`+ if (!rop.use_optix_cache()) {`
	`431`	`+ // Some ops prepend things`
	`432`	`+ if (op.opname() == op_error \|\| op.opname() == op_warning) {`
	`433`	`+ s = fmtformat("Shader {} [{}]: {}", op.opname(),`
	`434`	`+ rop.inst()->shadername(), s);`
	`435`	`+ }`
`433`	`436`	`}`
`434`	`437`
`435`	`438`	`// Now go back and put the new format string in its place`
`@@ -709,10 +712,12 @@ LLVMGEN(llvm_gen_print_fmt)`
`709`	`712`	`}`
`710`	`713`	`}`
`711`	`714`	`}`
`712`		`- // Some ops prepend things`
`713`		`- if (op.opname() == op_error \|\| op.opname() == op_warning) {`
`714`		`- s = fmtformat("Shader {} [{}]: {}", op.opname(),`
`715`		`- rop.inst()->shadername(), s);`
	`715`	`+ if (!rop.use_optix_cache()) {`
	`716`	`+ // Some ops prepend things`
	`717`	`+ if (op.opname() == op_error \|\| op.opname() == op_warning) {`
	`718`	`+ s = fmtformat("Shader {} [{}]: {}", op.opname(),`
	`719`	`+ rop.inst()->shadername(), s);`
	`720`	`+ }`
`716`	`721`	`}`
`717`	`722`	`ustring s_ustring(s.c_str());`
`718`	`723`	`call_args.push_back(rop.llvm_const_hash(s_ustring));`