Plugin shader editor improvements

dfranx · Oct 1, 2021 · 729b5fa · 729b5fa
1 parent 52864c4
commit 729b5fa
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 50 deletions.
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,6 +1,9 @@
 [v1.5.5]
 + add IPlugin3
 + fix gizmo 3D model
++ fix "Save as GLSL/HLSL" when a plugin shader editor is used
++ fix a bug where cached plugin shader editor information sometimes wouldn't be cleared
++ fix a bug that would cause SHADERed to crash when pausing the preview with a plugin shader editor open
 
 [v1.5.4]
 + add "Save as HLSL" option & --convert flag can now convert GLSL -> HLSL

diff --git a/src/SHADERed/Objects/PluginAPI/Plugin.h b/src/SHADERed/Objects/PluginAPI/Plugin.h
@@ -144,7 +144,10 @@ namespace ed {
 		typedef const char* (*DebuggerImmediateFn)(void* Debugger, const char* expr);
 
 		/********** IPlugin3 **********/
-		typedef void (*RegisterPlugin)(void* pluginManager, void* plugin, const char* pname, int apiVer, int pluginVer, void* procDLL);
+		typedef void (*RegisterPluginFn)(void* pluginManager, void* plugin, const char* pname, int apiVer, int pluginVer, void* procDLL);
+		typedef void* (*GetEditorPipelineItemFn)(void* UI, void* plugin, int langID, int editorID);
+		typedef void (*SetViewportSizeFn)(float w, float h);
+		typedef int (*IsObjectBoundFn)(void* Objects, const char* name, void* pipelineItem);
 	}
 
 	// CreatePlugin(), DestroyPlugin(ptr), GetPluginAPIVersion(), GetPluginVersion(), GetPluginName()
@@ -479,7 +482,11 @@ namespace ed {
 		virtual int GetVersion() { return 3; }
 
 		virtual void PluginManager_RegisterPlugins() = 0;
-
-		pluginfn::RegisterPlugin RegisterPlugin;
+		virtual const unsigned int* CustomLanguage_CompileToSPIRV2(void* item, int langID, const char* src, size_t src_len, ed::plugin::ShaderStage stage, const char* entry, ed::plugin::ShaderMacro* macros, size_t macroCount, size_t* spv_length, bool* compiled) = 0;
+
+		pluginfn::RegisterPluginFn RegisterPlugin;
+		pluginfn::GetEditorPipelineItemFn GetEditorPipelineItem;
+		pluginfn::SetViewportSizeFn SetViewportSize;
+		pluginfn::IsObjectBoundFn IsObjectBound;
 	};
 }
diff --git a/src/SHADERed/Objects/PluginManager.cpp b/src/SHADERed/Objects/PluginManager.cpp
@@ -395,8 +395,12 @@ namespace ed {
 			return (void*)pipe->Get(name);
 		};
 		plugin->BindShaderPassVariables = [](void* shaderpass, void* item) {
-			pipe::ShaderPass* data = (pipe::ShaderPass*)shaderpass;
-			data->Variables.Bind(item);
+			PipelineItem* pitem = (PipelineItem*)shaderpass;
+
+			if (pitem != nullptr && pitem->Type == PipelineItem::ItemType::ShaderPass) {
+				pipe::ShaderPass* pass = (pipe::ShaderPass*)pitem->Data;
+				pass->Variables.Bind(item);
+			}
 		};
 		plugin->GetViewMatrix = [](float* out) {
 			glm::mat4 viewm = SystemVariableManager::Instance().GetViewMatrix();
@@ -470,7 +474,6 @@ namespace ed {
 		};
 		plugin->IsTexture = [](void* objects, const char* name) -> bool {
 			ObjectManager* obj = (ObjectManager*)objects;
-
 			return obj->Get(name)->Type == ObjectType::Texture;
 		};
 		plugin->GetTexture = [](void* objects, const char* name) -> unsigned int {
@@ -1037,6 +1040,14 @@ namespace ed {
 				IPlugin1* p = (IPlugin1*)plugin;
 				pm->RegisterPlugin(p, pname, apiVer, pluginVer, procDLL);
 			};
+			plugin3->GetEditorPipelineItem = nullptr;
+			plugin3->SetViewportSize = [](float width, float height) {
+				SystemVariableManager::Instance().SetViewportSize(width, height);
+			};
+			plugin3->IsObjectBound = [](void* Objects, const char* name, void* pipelineItem) -> int {
+				ObjectManager* obj = (ObjectManager*)Objects;
+				return obj->IsBound(obj->Get(name), (PipelineItem*)pipelineItem);
+			};
 		}
 
 #ifdef SHADERED_DESKTOP

diff --git a/src/SHADERed/Objects/RenderEngine.cpp b/src/SHADERed/Objects/RenderEngine.cpp
@@ -1147,7 +1147,7 @@ namespace ed {
 					bool psCompiled = false;
 
 					if (psLang == ShaderLanguage::Plugin)
-						psCompiled = m_pluginCompileToSpirv(shader->PSSPV, shader->PSPath, psEntry, plugin::ShaderStage::Pixel, shader->Macros.data(), shader->Macros.size());
+						psCompiled = m_pluginCompileToSpirv(item, shader->PSSPV, shader->PSPath, psEntry, plugin::ShaderStage::Pixel, shader->Macros.data(), shader->Macros.size());
 					else
 						psCompiled = ShaderCompiler::CompileToSPIRV(shader->PSSPV, psLang, shader->PSPath, ShaderStage::Pixel, psEntry, shader->Macros, m_msgs, m_project);
 
@@ -1172,7 +1172,7 @@ namespace ed {
 					bool vsCompiled = false;
 
 					if (vsLang == ShaderLanguage::Plugin)
-						vsCompiled = m_pluginCompileToSpirv(shader->VSSPV, shader->VSPath, vsEntry, plugin::ShaderStage::Vertex, shader->Macros.data(), shader->Macros.size());
+						vsCompiled = m_pluginCompileToSpirv(item, shader->VSSPV, shader->VSPath, vsEntry, plugin::ShaderStage::Vertex, shader->Macros.data(), shader->Macros.size());
 					else
 						vsCompiled = ShaderCompiler::CompileToSPIRV(shader->VSSPV, vsLang, shader->VSPath, ShaderStage::Vertex, vsEntry, shader->Macros, m_msgs, m_project);
 
@@ -1202,7 +1202,7 @@ namespace ed {
 						lineBias = 0;
 
 						if (gsLang == ShaderLanguage::Plugin)
-							gsCompiled = m_pluginCompileToSpirv(shader->GSSPV, shader->GSPath, gsEntry, plugin::ShaderStage::Geometry, shader->Macros.data(), shader->Macros.size());
+							gsCompiled = m_pluginCompileToSpirv(item, shader->GSSPV, shader->GSPath, gsEntry, plugin::ShaderStage::Geometry, shader->Macros.data(), shader->Macros.size());
 						else
 							gsCompiled = ShaderCompiler::CompileToSPIRV(shader->GSSPV, gsLang, shader->GSPath, ShaderStage::Geometry, gsEntry, shader->Macros, m_msgs, m_project);
 
@@ -1237,7 +1237,7 @@ namespace ed {
 							lineBias = 0;
 
 							if (gsLang == ShaderLanguage::Plugin)
-								tsCompiled &= m_pluginCompileToSpirv(shader->TCSSPV, shader->TCSPath, tcsEntry, plugin::ShaderStage::TessellationControl, shader->Macros.data(), shader->Macros.size());
+								tsCompiled &= m_pluginCompileToSpirv(item, shader->TCSSPV, shader->TCSPath, tcsEntry, plugin::ShaderStage::TessellationControl, shader->Macros.data(), shader->Macros.size());
 							else
 								tsCompiled &= ShaderCompiler::CompileToSPIRV(shader->TCSSPV, tcsLang, shader->TCSPath, ShaderStage::TessellationControl, tcsEntry, shader->Macros, m_msgs, m_project);
 
@@ -1268,7 +1268,7 @@ namespace ed {
 							lineBias = 0;
 
 							if (gsLang == ShaderLanguage::Plugin)
-								tsCompiled &= m_pluginCompileToSpirv(shader->TESSPV, shader->TESPath, tesEntry, plugin::ShaderStage::TessellationEvaluation, shader->Macros.data(), shader->Macros.size());
+								tsCompiled &= m_pluginCompileToSpirv(item, shader->TESSPV, shader->TESPath, tesEntry, plugin::ShaderStage::TessellationEvaluation, shader->Macros.data(), shader->Macros.size());
 							else
 								tsCompiled &= ShaderCompiler::CompileToSPIRV(shader->TESSPV, tcsLang, shader->TESPath, ShaderStage::TessellationEvaluation, tesEntry, shader->Macros, m_msgs, m_project);
 
@@ -1344,7 +1344,7 @@ namespace ed {
 					bool compiled = false;
 
 					if (lang == ShaderLanguage::Plugin)
-						compiled = m_pluginCompileToSpirv(shader->SPV, shader->Path, entry, plugin::ShaderStage::Compute, shader->Macros.data(), shader->Macros.size());
+						compiled = m_pluginCompileToSpirv(item, shader->SPV, shader->Path, entry, plugin::ShaderStage::Compute, shader->Macros.data(), shader->Macros.size());
 					else
 						compiled = ShaderCompiler::CompileToSPIRV(shader->SPV, lang, shader->Path, ShaderStage::Compute, entry, shader->Macros, m_msgs, m_project);
 
@@ -1461,7 +1461,7 @@ namespace ed {
 					if (pssrc.size() > 0) {
 						ShaderLanguage psLang = ShaderCompiler::GetShaderLanguageFromExtension(shader->PSPath);
 						if (psLang == ShaderLanguage::Plugin)
-							psCompiled = m_pluginCompileToSpirv(shader->PSSPV, shader->PSPath, shader->PSEntry, plugin::ShaderStage::Pixel, shader->Macros.data(), shader->Macros.size(), pssrc);
+							psCompiled = m_pluginCompileToSpirv(item, shader->PSSPV, shader->PSPath, shader->PSEntry, plugin::ShaderStage::Pixel, shader->Macros.data(), shader->Macros.size(), pssrc);
 						else
 							psCompiled = ShaderCompiler::CompileSourceToSPIRV(shader->PSSPV, psLang, shader->PSPath, pssrc, ShaderStage::Pixel, shader->PSEntry, shader->Macros, m_msgs, m_project);
 
@@ -1492,7 +1492,7 @@ namespace ed {
 
 						ShaderLanguage vsLang = ShaderCompiler::GetShaderLanguageFromExtension(shader->VSPath);
 						if (vsLang == ShaderLanguage::Plugin)
-							vsCompiled = m_pluginCompileToSpirv(shader->VSSPV, shader->VSPath, shader->VSEntry, plugin::ShaderStage::Vertex, shader->Macros.data(), shader->Macros.size(), vssrc);
+							vsCompiled = m_pluginCompileToSpirv(item, shader->VSSPV, shader->VSPath, shader->VSEntry, plugin::ShaderStage::Vertex, shader->Macros.data(), shader->Macros.size(), vssrc);
 						else
 							vsCompiled = ShaderCompiler::CompileSourceToSPIRV(shader->VSSPV, vsLang, shader->VSPath, vssrc, ShaderStage::Vertex, shader->VSEntry, shader->Macros, m_msgs, m_project);
 
@@ -1521,7 +1521,7 @@ namespace ed {
 
 						ShaderLanguage gsLang = ShaderCompiler::GetShaderLanguageFromExtension(shader->GSPath);
 						if (gsLang == ShaderLanguage::Plugin)
-							gsCompiled = m_pluginCompileToSpirv(shader->GSSPV, shader->GSPath, shader->GSEntry, plugin::ShaderStage::Geometry, shader->Macros.data(), shader->Macros.size(), gssrc);
+							gsCompiled = m_pluginCompileToSpirv(item, shader->GSSPV, shader->GSPath, shader->GSEntry, plugin::ShaderStage::Geometry, shader->Macros.data(), shader->Macros.size(), gssrc);
 						else
 							gsCompiled = ShaderCompiler::CompileSourceToSPIRV(shader->GSSPV, gsLang, shader->GSPath, gssrc, ShaderStage::Geometry, shader->GSEntry, shader->Macros, m_msgs, m_project);
 
@@ -1553,7 +1553,7 @@ namespace ed {
 
 						ShaderLanguage tcsLang = ShaderCompiler::GetShaderLanguageFromExtension(shader->TCSPath);
 						if (tcsLang == ShaderLanguage::Plugin)
-							tsCompiled &= m_pluginCompileToSpirv(shader->TCSSPV, shader->TCSPath, shader->TCSEntry, plugin::ShaderStage::TessellationControl, shader->Macros.data(), shader->Macros.size(), tcssrc);
+							tsCompiled &= m_pluginCompileToSpirv(item, shader->TCSSPV, shader->TCSPath, shader->TCSEntry, plugin::ShaderStage::TessellationControl, shader->Macros.data(), shader->Macros.size(), tcssrc);
 						else
 							tsCompiled &= ShaderCompiler::CompileSourceToSPIRV(shader->TCSSPV, tcsLang, shader->TCSPath, tcssrc, ShaderStage::TessellationControl, shader->TCSEntry, shader->Macros, m_msgs, m_project);
 
@@ -1584,7 +1584,7 @@ namespace ed {
 
 						ShaderLanguage tesLang = ShaderCompiler::GetShaderLanguageFromExtension(shader->TESPath);
 						if (tesLang == ShaderLanguage::Plugin)
-							tsCompiled &= m_pluginCompileToSpirv(shader->TESSPV, shader->TESPath, shader->TESEntry, plugin::ShaderStage::TessellationEvaluation, shader->Macros.data(), shader->Macros.size(), tessrc);
+							tsCompiled &= m_pluginCompileToSpirv(item, shader->TESSPV, shader->TESPath, shader->TESEntry, plugin::ShaderStage::TessellationEvaluation, shader->Macros.data(), shader->Macros.size(), tessrc);
 						else
 							tsCompiled &= ShaderCompiler::CompileSourceToSPIRV(shader->TESSPV, tesLang, shader->TESPath, tessrc, ShaderStage::TessellationEvaluation, shader->TESEntry, shader->Macros, m_msgs, m_project);
 
@@ -1649,7 +1649,7 @@ namespace ed {
 
 						ShaderLanguage lang = ShaderCompiler::GetShaderLanguageFromExtension(shader->Path);
 						if (lang == ShaderLanguage::Plugin)
-							compiled = m_pluginCompileToSpirv(shader->SPV, shader->Path, shader->Entry, plugin::ShaderStage::Compute, shader->Macros.data(), shader->Macros.size(), vssrc);
+							compiled = m_pluginCompileToSpirv(item, shader->SPV, shader->Path, shader->Entry, plugin::ShaderStage::Compute, shader->Macros.data(), shader->Macros.size(), vssrc);
 						else
 							compiled = ShaderCompiler::CompileSourceToSPIRV(shader->SPV, lang, shader->Path, vssrc, ShaderStage::Compute, shader->Entry, shader->Macros, m_msgs, m_project);
 
@@ -2049,7 +2049,7 @@ namespace ed {
 					bool vsCompiled = false;
 
 					if (vsLang == ShaderLanguage::Plugin)
-						vsCompiled = m_pluginCompileToSpirv(data->VSSPV, data->VSPath, vsEntry, plugin::ShaderStage::Vertex, data->Macros.data(), data->Macros.size());
+						vsCompiled = m_pluginCompileToSpirv(items[i], data->VSSPV, data->VSPath, vsEntry, plugin::ShaderStage::Vertex, data->Macros.data(), data->Macros.size());
 					else
 						vsCompiled = ShaderCompiler::CompileToSPIRV(data->VSSPV, vsLang, data->VSPath, ShaderStage::Vertex, vsEntry, data->Macros, m_msgs, m_project);
 
@@ -2074,7 +2074,7 @@ namespace ed {
 					bool psCompiled = false;
 
 					if (psLang == ShaderLanguage::Plugin)
-						psCompiled = m_pluginCompileToSpirv(data->PSSPV, data->PSPath, psEntry, plugin::ShaderStage::Pixel, data->Macros.data(), data->Macros.size());
+						psCompiled = m_pluginCompileToSpirv(items[i], data->PSSPV, data->PSPath, psEntry, plugin::ShaderStage::Pixel, data->Macros.data(), data->Macros.size());
 					else
 						psCompiled = ShaderCompiler::CompileToSPIRV(data->PSSPV, psLang, data->PSPath, ShaderStage::Pixel, psEntry, data->Macros, m_msgs, m_project);
 
@@ -2102,7 +2102,7 @@ namespace ed {
 						ShaderLanguage gsLang = ShaderCompiler::GetShaderLanguageFromExtension(data->GSPath);
 
 						if (gsLang == ShaderLanguage::Plugin)
-							gsCompiled = m_pluginCompileToSpirv(data->GSSPV, data->GSPath, gsEntry, plugin::ShaderStage::Geometry, data->Macros.data(), data->Macros.size());
+							gsCompiled = m_pluginCompileToSpirv(items[i], data->GSSPV, data->GSPath, gsEntry, plugin::ShaderStage::Geometry, data->Macros.data(), data->Macros.size());
 						else
 							gsCompiled = ShaderCompiler::CompileToSPIRV(data->GSSPV, gsLang, data->GSPath, ShaderStage::Geometry, gsEntry, data->Macros, m_msgs, m_project);
 
@@ -2132,7 +2132,7 @@ namespace ed {
 							ShaderLanguage tcsLang = ShaderCompiler::GetShaderLanguageFromExtension(data->TCSPath);
 
 							if (tcsLang == ShaderLanguage::Plugin)
-								tsCompiled &= m_pluginCompileToSpirv(data->TCSSPV, data->TCSPath, tcsEntry, plugin::ShaderStage::TessellationControl, data->Macros.data(), data->Macros.size());
+								tsCompiled &= m_pluginCompileToSpirv(items[i], data->TCSSPV, data->TCSPath, tcsEntry, plugin::ShaderStage::TessellationControl, data->Macros.data(), data->Macros.size());
 							else
 								tsCompiled &= ShaderCompiler::CompileToSPIRV(data->TCSSPV, tcsLang, data->TCSPath, ShaderStage::TessellationControl, tcsEntry, data->Macros, m_msgs, m_project);
 
@@ -2158,7 +2158,7 @@ namespace ed {
 							ShaderLanguage tesLang = ShaderCompiler::GetShaderLanguageFromExtension(data->TESPath);
 
 							if (tesLang == ShaderLanguage::Plugin)
-								tsCompiled &= m_pluginCompileToSpirv(data->TESSPV, data->TESPath, tesEntry, plugin::ShaderStage::TessellationEvaluation, data->Macros.data(), data->Macros.size());
+								tsCompiled &= m_pluginCompileToSpirv(items[i], data->TESSPV, data->TESPath, tesEntry, plugin::ShaderStage::TessellationEvaluation, data->Macros.data(), data->Macros.size());
 							else
 								tsCompiled &= ShaderCompiler::CompileToSPIRV(data->TESSPV, tesLang, data->TESPath, ShaderStage::TessellationEvaluation, tesEntry, data->Macros, m_msgs, m_project);
 
@@ -2258,7 +2258,7 @@ namespace ed {
 					bool compiled = false;
 
 					if (lang == ShaderLanguage::Plugin)
-						compiled = m_pluginCompileToSpirv(data->SPV, data->Path, entry, plugin::ShaderStage::Compute, data->Macros.data(), data->Macros.size());
+						compiled = m_pluginCompileToSpirv(items[i], data->SPV, data->Path, entry, plugin::ShaderStage::Compute, data->Macros.data(), data->Macros.size());
 					else
 						compiled = ShaderCompiler::CompileToSPIRV(data->SPV, lang, data->Path, ShaderStage::Compute, entry, data->Macros, m_msgs, m_project);
 
@@ -2489,7 +2489,7 @@ namespace ed {
 
 		return plugin->CustomLanguage_ProcessGeneratedGLSL(plLang, src);
 	}
-	bool RenderEngine::m_pluginCompileToSpirv(std::vector<GLuint>& spvvec, const std::string& path, const std::string& entry, plugin::ShaderStage stage, ed::ShaderMacro* macros, size_t macroCount, const std::string& actualSource)
+	bool RenderEngine::m_pluginCompileToSpirv(PipelineItem* owner, std::vector<GLuint>& spvvec, const std::string& path, const std::string& entry, plugin::ShaderStage stage, ed::ShaderMacro* macros, size_t macroCount, const std::string& actualSource)
 	{
 		Logger::Get().Log("Plugin is compiling the shader to SPIR-V");
 
@@ -2509,6 +2509,12 @@ namespace ed {
 		size_t spv_length = 0;
 		const unsigned int* spv = plugin->CustomLanguage_CompileToSPIRV(plLang, source.c_str(), source.size(), stage, entry.c_str(), (plugin::ShaderMacro*)macros, macroCount, &spv_length, &ret);
 
+		if (spv == nullptr && ret && plugin->GetVersion() >= 3) {
+			IPlugin3* plugin3 = (IPlugin3*)plugin;
+			spv_length = 0;
+			spv = plugin3->CustomLanguage_CompileToSPIRV2(owner, plLang, source.c_str(), source.size(), stage, entry.c_str(), (plugin::ShaderMacro*)macros, macroCount, &spv_length, &ret);
+		}
+
 		spvvec = std::vector<GLuint>(spv, spv + spv_length);
 
 		return ret;

diff --git a/src/SHADERed/Objects/RenderEngine.h b/src/SHADERed/Objects/RenderEngine.h
@@ -136,7 +136,7 @@ namespace ed {
 		void m_applyMacros(std::string& source, pipe::AudioPass* pass); // TODO: merge this function with the ones above
 
 		// compile to spirv - plugin edition
-		bool m_pluginCompileToSpirv(std::vector<GLuint>& spv, const std::string& path, const std::string& entry, plugin::ShaderStage stage, ed::ShaderMacro* macros, size_t macroCount, const std::string& actualSrc = "");
+		bool m_pluginCompileToSpirv(PipelineItem* owner, std::vector<GLuint>& spv, const std::string& path, const std::string& entry, plugin::ShaderStage stage, ed::ShaderMacro* macros, size_t macroCount, const std::string& actualSrc = "");
 		const char* m_pluginProcessGLSL(const char* path, const char* src);
 
 		/* picking */