Implement shader caching

* Shader compilation is now cached. Subsequent loads take less than a millisecond. * Improved game, editor and project manager startup time. * Editor uses .godot/shader_cache to store shaders. * Game uses user://shader_cache * Project manager uses $config_dir/shader_cache * Options to tweak shader caching in project settings. * Editor path configuration moved from EditorSettings to new class, EditorPaths, so it can be available early on (before shaders are compiled). * Reworked ShaderCompilerRD to ensure deterministic shader code creation (else shader may change and cache will be invalidated). * Added shader compression with SMOLV: https://github.com/aras-p/smol-v
ivalylo · May 31, 2021 · 0d2e029 · 0d2e029
1 parent 39df47b
commit 0d2e029
Show file tree

Hide file tree

Showing 39 changed files with 3,113 additions and 172 deletions.
diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt
@@ -360,6 +360,12 @@ Comment: SMAZ
 Copyright: 2006-2009, Salvatore Sanfilippo
 License: BSD-3-clause
 
+Files: ./thirdparty/misc/smolv.cpp
+ ./thirdparty/misc/smolv.h
+Comment: SMOL-V
+Copyright: 2016-2020, Aras Pranckevicius
+License: public-domain or Unlicense or Expat
+
 Files: ./thirdparty/misc/stb_rect_pack.h
  ./thirdparty/misc/stb_vorbis.c
 Comment: stb libraries

diff --git a/core/SCsub b/core/SCsub
@@ -59,6 +59,7 @@ thirdparty_misc_sources = [
     "pcg.cpp",
     "polypartition.cpp",
     "clipper.cpp",
+    "smolv.cpp",
 ]
 thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources]
 env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources)

diff --git a/core/config/engine.cpp b/core/config/engine.cpp
@@ -31,6 +31,7 @@
 #include "engine.h"
 
 #include "core/authors.gen.h"
+#include "core/config/project_settings.h"
 #include "core/donors.gen.h"
 #include "core/license.gen.h"
 #include "core/version.h"
@@ -210,6 +211,13 @@ void Engine::get_singletons(List<Singleton> *p_singletons) {
 	}
 }
 
+void Engine::set_shader_cache_path(const String &p_path) {
+	shader_cache_path = p_path;
+}
+String Engine::get_shader_cache_path() const {
+	return shader_cache_path;
+}
+
 Engine *Engine::singleton = nullptr;
 
 Engine *Engine::get_singleton() {

diff --git a/core/config/engine.h b/core/config/engine.h
@@ -72,6 +72,8 @@ class Engine {
 
 	static Engine *singleton;
 
+	String shader_cache_path;
+
 public:
 	static Engine *get_singleton();
 
@@ -121,6 +123,9 @@ class Engine {
 	Dictionary get_license_info() const;
 	String get_license_text() const;
 
+	void set_shader_cache_path(const String &p_path);
+	String get_shader_cache_path() const;
+
 	bool is_abort_on_gpu_errors_enabled() const;
 	bool is_validation_layers_enabled() const;
 

diff --git a/doc/classes/EditorInterface.xml b/doc/classes/EditorInterface.xml
@@ -57,6 +57,12 @@
 				[b]Note:[/b] This returns the main editor control containing the whole editor, not the 2D or 3D viewports specifically.
 			</description>
 		</method>
+		<method name="get_editor_paths">
+			<return type="EditorPaths">
+			</return>
+			<description>
+			</description>
+		</method>
 		<method name="get_editor_scale" qualifiers="const">
 			<return type="float">
 			</return>

diff --git a/doc/classes/EditorPaths.xml b/doc/classes/EditorPaths.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="EditorPaths" inherits="Object" version="4.0">
+	<brief_description>
+	</brief_description>
+	<description>
+	</description>
+	<tutorials>
+	</tutorials>
+	<methods>
+		<method name="get_cache_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_config_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_data_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_self_contained_file" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="get_settings_dir" qualifiers="const">
+			<return type="String">
+			</return>
+			<description>
+			</description>
+		</method>
+		<method name="is_self_contained" qualifiers="const">
+			<return type="bool">
+			</return>
+			<description>
+			</description>
+		</method>
+	</methods>
+	<constants>
+	</constants>
+</class>
diff --git a/doc/classes/EditorSettings.xml b/doc/classes/EditorSettings.xml
@@ -124,15 +124,6 @@
 				Returns the value of the setting specified by [code]name[/code]. This is equivalent to using [method Object.get] on the EditorSettings instance.
 			</description>
 		</method>
-		<method name="get_settings_dir" qualifiers="const">
-			<return type="String">
-			</return>
-			<description>
-				Gets the global settings path for the engine. Inside this path, you can find some standard paths such as:
-				[code]settings/tmp[/code] - Used for temporary storage of files
-				[code]settings/templates[/code] - Where export templates are located
-			</description>
-		</method>
 		<method name="has_setting" qualifiers="const">
 			<return type="bool">
 			</return>

diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml
@@ -1502,6 +1502,16 @@
 		<member name="rendering/reflections/sky_reflections/texture_array_reflections.mobile" type="bool" setter="" getter="" default="false">
 			Lower-end override for [member rendering/reflections/sky_reflections/texture_array_reflections] on mobile devices, due to performance concerns or driver support.
 		</member>
+		<member name="rendering/shader_compiler/shader_cache/compress" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/enabled" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/strip_debug" type="bool" setter="" getter="" default="false">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/strip_debug.release" type="bool" setter="" getter="" default="true">
+		</member>
+		<member name="rendering/shader_compiler/shader_cache/use_zstd_compression" type="bool" setter="" getter="" default="true">
+		</member>
 		<member name="rendering/shading/overrides/force_blinn_over_ggx" type="bool" setter="" getter="" default="false">
 			If [code]true[/code], uses faster but lower-quality Blinn model to generate blurred reflections instead of the GGX model.
 		</member>

diff --git a/editor/editor_export.cpp b/editor/editor_export.cpp
@@ -1065,7 +1065,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
 				}
 			} else {
 				// Use default text server data.
-				String icu_data_file = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp_icu_data");
+				String icu_data_file = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp_icu_data");
 				TS->save_support_data(icu_data_file);
 				Vector<uint8_t> array = FileAccess::get_file_as_array(icu_data_file);
 				err = p_func(p_udata, ts_data, array, idx, total, enc_in_filters, enc_ex_filters, key);
@@ -1078,7 +1078,7 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
 	}
 
 	String config_file = "project.binary";
-	String engine_cfb = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmp" + config_file);
+	String engine_cfb = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmp" + config_file);
 	ProjectSettings::get_singleton()->save_custom(engine_cfb, custom_map, custom_list);
 	Vector<uint8_t> data = FileAccess::get_file_as_array(engine_cfb);
 	DirAccess::remove_file_or_error(engine_cfb);
@@ -1100,9 +1100,9 @@ Error EditorExportPlatform::save_pack(const Ref<EditorExportPreset> &p_preset, c
 
 	// Create the temporary export directory if it doesn't exist.
 	DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
-	da->make_dir_recursive(EditorSettings::get_singleton()->get_cache_dir());
+	da->make_dir_recursive(EditorPaths::get_singleton()->get_cache_dir());
 
-	String tmppath = EditorSettings::get_singleton()->get_cache_dir().plus_file("packtmp");
+	String tmppath = EditorPaths::get_singleton()->get_cache_dir().plus_file("packtmp");
 	FileAccess *ftmp = FileAccess::open(tmppath, FileAccess::WRITE);
 	ERR_FAIL_COND_V_MSG(!ftmp, ERR_CANT_CREATE, "Cannot create file '" + tmppath + "'.");
 
@@ -1984,7 +1984,7 @@ void EditorExportTextSceneToBinaryPlugin::_export_file(const String &p_path, con
 	if (!convert) {
 		return;
 	}
-	String tmp_path = EditorSettings::get_singleton()->get_cache_dir().plus_file("tmpfile.res");
+	String tmp_path = EditorPaths::get_singleton()->get_cache_dir().plus_file("tmpfile.res");
 	Error err = ResourceFormatLoaderText::convert_file_to_binary(p_path, tmp_path);
 	if (err != OK) {
 		DirAccess::remove_file_or_error(tmp_path);

diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp
@@ -80,6 +80,7 @@
 #include "editor/editor_inspector.h"
 #include "editor/editor_layouts_dialog.h"
 #include "editor/editor_log.h"
+#include "editor/editor_paths.h"
 #include "editor/editor_plugin.h"
 #include "editor/editor_properties.h"
 #include "editor/editor_resource_picker.h"
@@ -1457,7 +1458,7 @@ void EditorNode::_save_scene_with_preview(String p_file, int p_idx) {
 			img->convert(Image::FORMAT_RGB8);
 
 			//save thumbnail directly, as thumbnailer may not update due to actual scene not changing md5
-			String temp_path = EditorSettings::get_singleton()->get_cache_dir();
+			String temp_path = EditorPaths::get_singleton()->get_cache_dir();
 			String cache_base = ProjectSettings::get_singleton()->globalize_path(p_file).md5_text();
 			cache_base = temp_path.plus_file("resthumb-" + cache_base);
 
@@ -2745,10 +2746,10 @@ void EditorNode::_menu_option_confirm(int p_option, bool p_confirmed) {
 			settings_config_dialog->popup_edit_settings();
 		} break;
 		case SETTINGS_EDITOR_DATA_FOLDER: {
-			OS::get_singleton()->shell_open(String("file://") + EditorSettings::get_singleton()->get_data_dir());
+			OS::get_singleton()->shell_open(String("file://") + EditorPaths::get_singleton()->get_data_dir());
 		} break;
 		case SETTINGS_EDITOR_CONFIG_FOLDER: {
-			OS::get_singleton()->shell_open(String("file://") + EditorSettings::get_singleton()->get_settings_dir());
+			OS::get_singleton()->shell_open(String("file://") + EditorPaths::get_singleton()->get_settings_dir());
 		} break;
 		case SETTINGS_MANAGE_EXPORT_TEMPLATES: {
 			export_template_manager->popup_manager();
@@ -3727,10 +3728,15 @@ bool EditorNode::is_scene_in_use(const String &p_path) {
 	return false;
 }
 
+void EditorNode::register_editor_paths(bool p_for_project_manager) {
+	EditorPaths::create(p_for_project_manager);
+}
+
 void EditorNode::register_editor_types() {
 	ResourceLoader::set_timestamp_on_load(true);
 	ResourceSaver::set_timestamp_on_save(true);
 
+	ClassDB::register_class<EditorPaths>();
 	ClassDB::register_class<EditorPlugin>();
 	ClassDB::register_class<EditorTranslationParserPlugin>();
 	ClassDB::register_class<EditorImportPlugin>();
@@ -3774,6 +3780,9 @@ void EditorNode::register_editor_types() {
 
 void EditorNode::unregister_editor_types() {
 	_init_callbacks.clear();
+	if (EditorPaths::get_singleton()) {
+		EditorPaths::free();
+	}
 }
 
 void EditorNode::stop_child_process(OS::ProcessID p_pid) {

diff --git a/editor/editor_node.h b/editor/editor_node.h
@@ -798,6 +798,7 @@ class EditorNode : public Node {
 
 	Error export_preset(const String &p_preset, const String &p_path, bool p_debug, bool p_pack_only);
 
+	static void register_editor_paths(bool p_for_project_manager);
 	static void register_editor_types();
 	static void unregister_editor_types();