Refactor ci_testing and separate it from DevToolsPlugin (#13513)

# Objective

- We use
[`ci_testing`](https://dev-docs.bevyengine.org/bevy/dev_tools/ci_testing/index.html)
to specify per-example configuration on when to take a screenshot, when
to exit, etc.
- In the future more features may be added, such as #13512. To support
this growth, `ci_testing` should be easier to read and maintain.

## Solution

- Convert `ci_testing.rs` into the folder `ci_testing`, splitting the
configuration and systems into `ci_testing/config.rs` and
`ci_testing/systems.rs`.
- Convert `setup_app` into the plugin `CiTestingPlugin`. This new plugin
is added to both `DefaultPlugins` and `MinimalPlugins`.
- Remove `DevToolsPlugin` from `MinimalPlugins`, since it was only used
for CI testing.
- Clean up some code, add many comments, and add a few unit tests.

## Testing

The most important part is that this still passes all of the CI
validation checks (merge queue), since that is when it will be used the
most. I don't think I changed any behavior, so it should operate the
same.

You can also test it locally using:

```shell
# Run the breakout example, enabling `bevy_ci_testing` and loading the configuration used in CI.
CI_TESTING_CONFIG=".github/example-run/breakout.ron" cargo r --example breakout -F bevy_ci_testing
```

---

## Changelog

- Added `CiTestingPlugin`, which is split off from `DevToolsPlugin`.
- Removed `DevToolsPlugin` from `MinimalPlugins`.

## Migration Guide

Hi maintainers! I believe `DevToolsPlugin` was added within the same
release as this PR, so I don't think a migration guide is needed.

`DevToolsPlugin` is no longer included in `MinimalPlugins`, so you will
need to remove it manually.

```rust
// Before
App::new()
    .add_plugins(MinimalPlugins)
    .run();

// After
App::new()
    .add_plugins(MinimalPlugins)
    .add_plugins(DevToolsPlugin)
    .run();
```

---------

Co-authored-by: Alice Cecile <[email protected]>
Co-authored-by: François Mockers <[email protected]>

Author: 3 people

crates/bevy_dev_tools/src/ci_testing.rs

@@ -0,0 +1,85 @@
+use bevy_ecs::prelude::*;
+use serde::Deserialize;
+
+/// A configuration struct for automated CI testing.
+///
+/// It gets used when the `bevy_ci_testing` feature is enabled to automatically
+/// exit a Bevy app when run through the CI. This is needed because otherwise
+/// Bevy apps would be stuck in the game loop and wouldn't allow the CI to progress.
+#[derive(Deserialize, Resource, PartialEq, Debug)]
+pub struct CiTestingConfig {
+    /// The setup for this test.
+    #[serde(default)]
+    pub setup: CiTestingSetup,
+    /// Events to send, with their associated frame.
+    #[serde(default)]
+    pub events: Vec<CiTestingEventOnFrame>,
+}
+
+/// Setup for a test.
+#[derive(Deserialize, Default, PartialEq, Debug)]
+pub struct CiTestingSetup {
+    /// The amount of time in seconds between frame updates.
+    ///
+    /// This is set through the [`TimeUpdateStrategy::ManualDuration`] resource.
+    ///
+    /// [`TimeUpdateStrategy::ManualDuration`]: bevy_time::TimeUpdateStrategy::ManualDuration
+    pub fixed_frame_time: Option<f32>,
+}
+
+/// An event to send at a given frame, used for CI testing.
+#[derive(Deserialize, PartialEq, Debug)]
+pub struct CiTestingEventOnFrame(pub u32, pub CiTestingEvent);
+
+/// An event to send, used for CI testing.
+#[derive(Deserialize, PartialEq, Debug)]
+pub enum CiTestingEvent {
+    /// Takes a screenshot of the entire screen, and saves the results to
+    /// `screenshot-{current_frame}.png`.
+    Screenshot,
+    /// Stops the program by sending [`AppExit::Success`].
+    ///
+    /// [`AppExit::Success`]: bevy_app::AppExit::Success
+    AppExit,
+    /// Sends a [`CiTestingCustomEvent`] using the given [`String`].
+    Custom(String),
+}
+
+/// A custom event that can be configured from a configuration file for CI testing.
+#[derive(Event)]
+pub struct CiTestingCustomEvent(pub String);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deserialize() {
+        const INPUT: &str = r#"
+(
+    setup: (
+        fixed_frame_time: Some(0.03),
+    ),
+    events: [
+        (100, Custom("Hello, world!")),
+        (200, Screenshot),
+        (300, AppExit),
+    ],
+)"#;
+
+        let expected = CiTestingConfig {
+            setup: CiTestingSetup {
+                fixed_frame_time: Some(0.03),
+            },
+            events: vec![
+                CiTestingEventOnFrame(100, CiTestingEvent::Custom("Hello, world!".into())),
+                CiTestingEventOnFrame(200, CiTestingEvent::Screenshot),
+                CiTestingEventOnFrame(300, CiTestingEvent::AppExit),
+            ],
+        };
+
+        let config: CiTestingConfig = ron::from_str(INPUT).unwrap();
+
+        assert_eq!(config, expected);
+    }
+}

crates/bevy_dev_tools/src/ci_testing/config.rs

@@ -0,0 +1,53 @@
+//! Utilities for testing in CI environments.
+
+mod config;
+mod systems;
+
+pub use self::config::*;
+
+use bevy_app::prelude::*;
+use bevy_time::TimeUpdateStrategy;
+use std::time::Duration;
+
+/// A plugin that instruments continuous integration testing by automatically executing user-defined actions.
+///
+/// This plugin reads a [`ron`] file specified with the `CI_TESTING_CONFIG` environmental variable
+/// (`ci_testing_config.ron` by default) and executes its specified actions. For a reference of the
+/// allowed configuration, see [`CiTestingConfig`].
+///
+/// This plugin is included within `DefaultPlugins` and `MinimalPlugins` when the `bevy_ci_testing`
+/// feature is enabled. It is recommended to only used this plugin during testing (manual or
+/// automatic), and disable it during regular development and for production builds.
+pub struct CiTestingPlugin;
+
+impl Plugin for CiTestingPlugin {
+    fn build(&self, app: &mut App) {
+        #[cfg(not(target_arch = "wasm32"))]
+        let config: CiTestingConfig = {
+            let filename = std::env::var("CI_TESTING_CONFIG")
+                .unwrap_or_else(|_| "ci_testing_config.ron".to_string());
+            ron::from_str(
+                &std::fs::read_to_string(filename)
+                    .expect("error reading CI testing configuration file"),
+            )
+            .expect("error deserializing CI testing configuration file")
+        };
+
+        #[cfg(target_arch = "wasm32")]
+        let config: CiTestingConfig = {
+            let config = include_str!("../../../../ci_testing_config.ron");
+            ron::from_str(config).expect("error deserializing CI testing configuration file")
+        };
+
+        // Configure a fixed frame time if specified.
+        if let Some(fixed_frame_time) = config.setup.fixed_frame_time {
+            app.insert_resource(TimeUpdateStrategy::ManualDuration(Duration::from_secs_f32(
+                fixed_frame_time,
+            )));
+        }
+
+        app.add_event::<CiTestingCustomEvent>()
+            .insert_resource(config)
+            .add_systems(Update, systems::send_events);
+    }
+}

crates/bevy_dev_tools/src/ci_testing/mod.rs

@@ -0,0 +1,51 @@
+use super::config::*;
+use bevy_app::AppExit;
+use bevy_ecs::prelude::*;
+use bevy_render::view::screenshot::ScreenshotManager;
+use bevy_utils::tracing::{debug, info, warn};
+use bevy_window::PrimaryWindow;
+
+pub(crate) fn send_events(world: &mut World, mut current_frame: Local<u32>) {
+    let mut config = world.resource_mut::<CiTestingConfig>();
+
+    // Take all events for the current frame, leaving all the remaining alone.
+    let events = std::mem::take(&mut config.events);
+    let (to_run, remaining): (Vec<_>, _) = events
+        .into_iter()
+        .partition(|event| event.0 == *current_frame);
+    config.events = remaining;
+
+    for CiTestingEventOnFrame(_, event) in to_run {
+        debug!("Handling event: {:?}", event);
+        match event {
+            CiTestingEvent::AppExit => {
+                world.send_event(AppExit::Success);
+                info!("Exiting after {} frames. Test successful!", *current_frame);
+            }
+            CiTestingEvent::Screenshot => {
+                let mut primary_window_query =
+                    world.query_filtered::<Entity, With<PrimaryWindow>>();
+                let Ok(main_window) = primary_window_query.get_single(world) else {
+                    warn!("Requesting screenshot, but PrimaryWindow is not available");
+                    continue;
+                };
+                let Some(mut screenshot_manager) = world.get_resource_mut::<ScreenshotManager>()
+                else {
+                    warn!("Requesting screenshot, but ScreenshotManager is not available");
+                    continue;
+                };
+                let path = format!("./screenshot-{}.png", *current_frame);
+                screenshot_manager
+                    .save_screenshot_to_disk(main_window, path)
+                    .unwrap();
+                info!("Took a screenshot at frame {}.", *current_frame);
+            }
+            // Custom events are forwarded to the world.
+            CiTestingEvent::Custom(event_string) => {
+                world.send_event(CiTestingCustomEvent(event_string));
+            }
+        }
+    }
+
+    *current_frame += 1;
+}

crates/bevy_dev_tools/src/ci_testing/systems.rs

@@ -48,10 +48,5 @@ pub mod ui_debug_overlay;
 pub struct DevToolsPlugin;
 
 impl Plugin for DevToolsPlugin {
-    fn build(&self, _app: &mut App) {
-        #[cfg(feature = "bevy_ci_testing")]
-        {
-            ci_testing::setup_app(_app);
-        }
-    }
+    fn build(&self, _app: &mut App) {}
 }

crates/bevy_dev_tools/src/lib.rs

@@ -29,6 +29,7 @@ use bevy_app::{Plugin, PluginGroup, PluginGroupBuilder};
 /// * [`GilrsPlugin`](crate::gilrs::GilrsPlugin) - with feature `bevy_gilrs`
 /// * [`AnimationPlugin`](crate::animation::AnimationPlugin) - with feature `bevy_animation`
 /// * [`DevToolsPlugin`](crate::dev_tools::DevToolsPlugin) - with feature `bevy_dev_tools`
+/// * [`CiTestingPlugin`](crate::dev_tools::ci_testing::CiTestingPlugin) - with feature `bevy_ci_testing`
 ///
 /// [`DefaultPlugins`] obeys *Cargo* *feature* flags. Users may exert control over this plugin group
 /// by disabling `default-features` in their `Cargo.toml` and enabling only those features
@@ -142,6 +143,11 @@ impl PluginGroup for DefaultPlugins {
             group = group.add(bevy_dev_tools::DevToolsPlugin);
         }
 
+        #[cfg(feature = "bevy_ci_testing")]
+        {
+            group = group.add(bevy_dev_tools::ci_testing::CiTestingPlugin);
+        }
+
         group = group.add(IgnoreAmbiguitiesPlugin);
 
         group
@@ -173,7 +179,7 @@ impl Plugin for IgnoreAmbiguitiesPlugin {
 /// * [`FrameCountPlugin`](crate::core::FrameCountPlugin)
 /// * [`TimePlugin`](crate::time::TimePlugin)
 /// * [`ScheduleRunnerPlugin`](crate::app::ScheduleRunnerPlugin)
-/// * [`DevToolsPlugin`](crate::dev_tools::DevToolsPlugin) - with feature `bevy_dev_tools`
+/// * [`CiTestingPlugin`](crate::dev_tools::ci_testing::CiTestingPlugin) - with feature `bevy_ci_testing`
 ///
 /// This group of plugins is intended for use for minimal, *headless* programs –
 /// see the [*Bevy* *headless* example](https://github.com/bevyengine/bevy/blob/main/examples/app/headless.rs)
@@ -194,10 +200,12 @@ impl PluginGroup for MinimalPlugins {
             .add(bevy_core::FrameCountPlugin)
             .add(bevy_time::TimePlugin)
             .add(bevy_app::ScheduleRunnerPlugin::default());
-        #[cfg(feature = "bevy_dev_tools")]
+
+        #[cfg(feature = "bevy_ci_testing")]
         {
-            group = group.add(bevy_dev_tools::DevToolsPlugin);
+            group = group.add(bevy_dev_tools::ci_testing::CiTestingPlugin);
         }
+
         group
     }
 }