Merge pull request #96705 from elmajime/camera_from_external_feed

Add support for external camera feed from external plugin on Android

7 files changed, 271 insertions, 1 deletions

diff --git a/drivers/gles3/effects/feed_effects.cpp b/drivers/gles3/effects/feed_effects.cpp
new file mode 100644
index 0000000000..8ca88da662
--- /dev/null
+++ b/drivers/gles3/effects/feed_effects.cpp
@@ -0,0 +1,128 @@
+/**************************************************************************/
+/*  feed_effects.cpp                                                      */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#ifdef GLES3_ENABLED
+
+#include "feed_effects.h"
+
+#ifdef ANDROID_ENABLED
+#include <GLES3/gl3ext.h>
+#endif
+
+#define GL_PROGRAM_POINT_SIZE 0x8642
+
+using namespace GLES3;
+
+FeedEffects *FeedEffects::singleton = nullptr;
+
+FeedEffects *FeedEffects::get_singleton() {
+	return singleton;
+}
+
+FeedEffects::FeedEffects() {
+	singleton = this;
+
+	feed.shader.initialize();
+	feed.shader_version = feed.shader.version_create();
+	feed.shader.version_bind_shader(feed.shader_version, FeedShaderGLES3::MODE_DEFAULT);
+
+	{ // Screen Triangle.
+		glGenBuffers(1, &screen_triangle);
+		glBindBuffer(GL_ARRAY_BUFFER, screen_triangle);
+
+		const float qv[6] = {
+			-1.0f,
+			-1.0f,
+			3.0f,
+			-1.0f,
+			-1.0f,
+			3.0f,
+		};
+
+		glBufferData(GL_ARRAY_BUFFER, sizeof(float) * 6, qv, GL_STATIC_DRAW);
+		glBindBuffer(GL_ARRAY_BUFFER, 0); //unbind
+
+		glGenVertexArrays(1, &screen_triangle_array);
+		glBindVertexArray(screen_triangle_array);
+		glBindBuffer(GL_ARRAY_BUFFER, screen_triangle);
+		glVertexAttribPointer(RS::ARRAY_VERTEX, 2, GL_FLOAT, GL_FALSE, sizeof(float) * 2, nullptr);
+		glEnableVertexAttribArray(RS::ARRAY_VERTEX);
+		glBindVertexArray(0);
+		glBindBuffer(GL_ARRAY_BUFFER, 0); //unbind
+	}
+}
+
+FeedEffects::~FeedEffects() {
+	singleton = nullptr;
+	glDeleteBuffers(1, &screen_triangle);
+	glDeleteVertexArrays(1, &screen_triangle_array);
+	feed.shader.version_free(feed.shader_version);
+}
+
+Transform3D transform3D_from_mat4(const float *p_mat4) {
+	Transform3D res;
+
+	res.basis.rows[0][0] = p_mat4[0];
+	res.basis.rows[1][0] = p_mat4[1];
+	res.basis.rows[2][0] = p_mat4[2];
+	// p_mat4[3] = 0;
+	res.basis.rows[0][1] = p_mat4[4];
+	res.basis.rows[1][1] = p_mat4[5];
+	res.basis.rows[2][1] = p_mat4[6];
+	// p_mat4[7] = 0;
+	res.basis.rows[0][2] = p_mat4[8];
+	res.basis.rows[1][2] = p_mat4[9];
+	res.basis.rows[2][2] = p_mat4[10];
+	// p_mat4[11] = 0;
+	res.origin.x = p_mat4[12];
+	res.origin.y = p_mat4[13];
+	res.origin.z = p_mat4[14];
+	// p_mat4[15] = 1;
+
+	return res;
+}
+
+void FeedEffects::draw() {
+	bool success = feed.shader.version_bind_shader(feed.shader_version, FeedShaderGLES3::MODE_DEFAULT, FeedShaderGLES3::USE_EXTERNAL_SAMPLER);
+	if (!success) {
+		OS::get_singleton()->print("Godot : FeedShaderGLES3 Could not bind version_bind_shader USE_EXTERNAL_SAMPLER");
+		return;
+	}
+
+	draw_screen_triangle();
+}
+
+void FeedEffects::draw_screen_triangle() {
+	glBindVertexArray(screen_triangle_array);
+	glDrawArrays(GL_TRIANGLES, 0, 3);
+	glBindVertexArray(0);
+}
+
+#endif // GLES3_ENABLED
diff --git a/drivers/gles3/effects/feed_effects.h b/drivers/gles3/effects/feed_effects.h
new file mode 100644
index 0000000000..5856a3e04b
--- /dev/null
+++ b/drivers/gles3/effects/feed_effects.h
@@ -0,0 +1,68 @@
+/**************************************************************************/
+/*  feed_effects.h                                                        */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#ifndef FEED_EFFECTS_GLES3_H
+#define FEED_EFFECTS_GLES3_H
+
+#ifdef GLES3_ENABLED
+
+#include "drivers/gles3/shaders/feed.glsl.gen.h"
+
+namespace GLES3 {
+
+class FeedEffects {
+private:
+	struct Feed {
+		FeedShaderGLES3 shader;
+		RID shader_version;
+	} feed;
+
+	static FeedEffects *singleton;
+
+	GLuint screen_triangle = 0;
+	GLuint screen_triangle_array = 0;
+
+public:
+	static FeedEffects *get_singleton();
+
+	FeedEffects();
+	~FeedEffects();
+
+	void draw();
+
+private:
+	void draw_screen_triangle();
+};
+
+} // namespace GLES3
+
+#endif // GLES3_ENABLED
+
+#endif // FEED_EFFECTS_GLES3_H
diff --git a/drivers/gles3/rasterizer_gles3.cpp b/drivers/gles3/rasterizer_gles3.cpp
index 843b6eac05..0fda42979f 100644
--- a/drivers/gles3/rasterizer_gles3.cpp
+++ b/drivers/gles3/rasterizer_gles3.cpp
@@ -218,6 +218,7 @@ void RasterizerGLES3::finalize() {
 	memdelete(glow);
 	memdelete(cubemap_filter);
 	memdelete(copy_effects);
+	memdelete(feed_effects);
 	memdelete(light_storage);
 	memdelete(particles_storage);
 	memdelete(mesh_storage);
@@ -366,6 +367,7 @@ RasterizerGLES3::RasterizerGLES3() {
 	cubemap_filter = memnew(GLES3::CubemapFilter);
 	glow = memnew(GLES3::Glow);
 	post_effects = memnew(GLES3::PostEffects);
+	feed_effects = memnew(GLES3::FeedEffects);
 	gi = memnew(GLES3::GI);
 	fog = memnew(GLES3::Fog);
 	canvas = memnew(RasterizerCanvasGLES3());
diff --git a/drivers/gles3/rasterizer_gles3.h b/drivers/gles3/rasterizer_gles3.h
index 6765d8b4d5..abda2a5e06 100644
--- a/drivers/gles3/rasterizer_gles3.h
+++ b/drivers/gles3/rasterizer_gles3.h
@@ -35,6 +35,7 @@
 
 #include "effects/copy_effects.h"
 #include "effects/cubemap_filter.h"
+#include "effects/feed_effects.h"
 #include "effects/glow.h"
 #include "effects/post_effects.h"
 #include "environment/fog.h"
@@ -78,6 +79,7 @@ protected:
 	GLES3::CubemapFilter *cubemap_filter = nullptr;
 	GLES3::Glow *glow = nullptr;
 	GLES3::PostEffects *post_effects = nullptr;
+	GLES3::FeedEffects *feed_effects = nullptr;
 	RasterizerCanvasGLES3 *canvas = nullptr;
 	RasterizerSceneGLES3 *scene = nullptr;
 	static RasterizerGLES3 *singleton;
diff --git a/drivers/gles3/rasterizer_scene_gles3.cpp b/drivers/gles3/rasterizer_scene_gles3.cpp
index a73f14c796..3fe5748b7a 100644
--- a/drivers/gles3/rasterizer_scene_gles3.cpp
+++ b/drivers/gles3/rasterizer_scene_gles3.cpp
@@ -31,6 +31,7 @@
 #include "rasterizer_scene_gles3.h"
 
 #include "drivers/gles3/effects/copy_effects.h"
+#include "drivers/gles3/effects/feed_effects.h"
 #include "rasterizer_gles3.h"
 #include "storage/config.h"
 #include "storage/mesh_storage.h"
@@ -39,6 +40,8 @@
 
 #include "core/config/project_settings.h"
 #include "core/templates/sort_array.h"
+#include "servers/camera/camera_feed.h"
+#include "servers/camera_server.h"
 #include "servers/rendering/rendering_server_default.h"
 #include "servers/rendering/rendering_server_globals.h"
 
@@ -2382,7 +2385,9 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 	bool draw_sky = false;
 	bool draw_sky_fog_only = false;
 	bool keep_color = false;
+	bool draw_feed = false;
 	float sky_energy_multiplier = 1.0;
+	int camera_feed_id = -1;
 
 	if (unlikely(get_debug_draw_mode() == RS::VIEWPORT_DEBUG_DRAW_OVERDRAW)) {
 		clear_color = Color(0, 0, 0, 1); //in overdraw mode, BG should always be black
@@ -2427,6 +2432,8 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 				keep_color = true;
 			} break;
 			case RS::ENV_BG_CAMERA_FEED: {
+				camera_feed_id = environment_get_camera_feed_id(render_data.environment);
+				draw_feed = true;
 			} break;
 			default: {
 			}
@@ -2538,7 +2545,7 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 		glClear(GL_DEPTH_BUFFER_BIT);
 	}
 
-	if (!keep_color) {
+	if (!keep_color && !draw_feed) {
 		clear_color.a = render_data.transparent_bg ? 0.0f : 1.0f;
 		glClearBufferfv(GL_COLOR, 0, clear_color.components);
 	} else if (fbo != rt->fbo) {
@@ -2578,6 +2585,29 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 			spec_constant_base_flags |= SceneShaderGLES3::APPLY_TONEMAPPING;
 		}
 	}
+
+	if (draw_feed && camera_feed_id > -1) {
+		RENDER_TIMESTAMP("Render Camera feed");
+
+		scene_state.enable_gl_depth_draw(false);
+		scene_state.enable_gl_depth_test(false);
+		scene_state.enable_gl_blend(false);
+		scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_BACK);
+
+		Ref<CameraFeed> feed = CameraServer::get_singleton()->get_feed_by_id(camera_feed_id);
+
+		if (feed.is_valid()) {
+			RID camera_YCBCR = feed->get_texture(CameraServer::FEED_YCBCR_IMAGE);
+			GLES3::TextureStorage::get_singleton()->texture_bind(camera_YCBCR, 0);
+
+			GLES3::FeedEffects *feed_effects = GLES3::FeedEffects::get_singleton();
+			feed_effects->draw();
+		}
+		scene_state.enable_gl_depth_draw(true);
+		scene_state.enable_gl_depth_test(true);
+		scene_state.enable_gl_blend(true);
+	}
+
 	// Render Opaque Objects.
 	RenderListParameters render_list_params(render_list[RENDER_LIST_OPAQUE].elements.ptr(), render_list[RENDER_LIST_OPAQUE].elements.size(), reverse_cull, spec_constant_base_flags, use_wireframe);
 
diff --git a/drivers/gles3/shaders/SCsub b/drivers/gles3/shaders/SCsub
index df2c515035..0207ba12b7 100644
--- a/drivers/gles3/shaders/SCsub
+++ b/drivers/gles3/shaders/SCsub
@@ -17,6 +17,7 @@ if "GLES3_GLSL" in env["BUILDERS"]:
 
     # as we have a few, not yet, converted files we name the ones we want to include:
     env.GLES3_GLSL("canvas.glsl")
+    env.GLES3_GLSL("feed.glsl")
     env.GLES3_GLSL("scene.glsl")
     env.GLES3_GLSL("sky.glsl")
     env.GLES3_GLSL("canvas_occlusion.glsl")
diff --git a/drivers/gles3/shaders/feed.glsl b/drivers/gles3/shaders/feed.glsl
new file mode 100644
index 0000000000..9d89fc699d
--- /dev/null
+++ b/drivers/gles3/shaders/feed.glsl
@@ -0,0 +1,39 @@
+/* clang-format off */
+#[modes]
+
+mode_default =
+
+#[specializations]
+
+USE_EXTERNAL_SAMPLER = false
+
+#[vertex]
+
+layout(location = 0) in vec2 vertex_attrib;
+
+out vec2 uv_interp;
+
+
+void main() {
+	uv_interp = vertex_attrib * 0.5 + 0.5;
+	gl_Position = vec4(vertex_attrib, 1.0, 1.0);
+}
+
+/* clang-format off */
+#[fragment]
+
+layout(location = 0) out vec4 frag_color;
+in vec2 uv_interp;
+
+/* clang-format on */
+#ifdef USE_EXTERNAL_SAMPLER
+uniform samplerExternalOES sourceFeed; // texunit:0
+#else
+uniform sampler2D sourceFeed; // texunit:0
+#endif
+
+void main() {
+	vec4 color = texture(sourceFeed, uv_interp);
+
+	frag_color = color;
+}