diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h new file mode 100644 index 0000000000..9946d20c12 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEventTypes.h @@ -0,0 +1,99 @@ +/* Screen Event Types + * + * From: https://github.com/PokemonAutomation/ + * + * What a frame shows, in words every game shares. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEventTypes_H + +#include +#include +#include +#include +#include "Common/Cpp/EnumStringMap.h" + +namespace PokemonAutomation{ + + +// One thing a frame shows. A game reports the ones its detectors can see. +enum class ScreenEvent : uint8_t{ + FADE_BLACK, // A uniform dark screen: a door, a cave mouth, a scene cut. + FADE_WHITE, // A uniform white screen: a cave exit. + LOCATION_BANNER, // A location banner. An observer given an expected + // map reports only that map's. + DIALOG_BOX, // A dialogue box. + CHOICE_BOX, // A choice (Yes/No, a menu of answers). + INTERACT_PROMPT, // A button prompt to interact ("A Talk"). + BATTLE_STARTED, // A battle shows (its menu, its HUD or its opening). + BATTLE_ENDED, // A battle's closing screen. + PLAYER_CAN_MOVE, // The overworld with the player in control. + SCREEN_STILL, // The screen barely changed since the frame before. + SWITCH_HOME_MENU, // The Switch's HOME menu: the game is not on the screen. +}; +// "dialog box", for the log. +const EnumStringMap& SCREEN_EVENT_NAMES(); + + +// How a screen's fade looks: the kind of FADE_BLACK / FADE_WHITE. +enum class WarpFade{ + NONE, + BLACK, // a door's black, or a cave mouth's dim fade + WHITE, // a cave exit's white, with no battle after it +}; +const EnumStringMap& WARP_FADE_NAMES(); + + +// The events of one frame, or of all the frames a watcher saw. +class ScreenEvents{ +public: + ScreenEvents() = default; + ScreenEvents(std::initializer_list events); + + bool has(ScreenEvent event) const{ + return (m_bits & bit(event)) != 0; + } + bool any_of(const ScreenEvents& events) const{ + return (m_bits & events.m_bits) != 0; + } + // Adds `event`, or removes it when `on` is false (as std::bitset::set). + void set(ScreenEvent event, bool on); + void merge(const ScreenEvents& events){ + m_bits |= events.m_bits; + } + + bool operator==(const ScreenEvents& other) const = default; + +private: + static uint32_t bit(ScreenEvent event){ + return uint32_t(1) << (uint32_t)event; + } + + uint32_t m_bits = 0; +}; + + + +} // namespace PokemonAutomation + + +// "dialog box, player can move"; "nothing" when empty. +template <> +struct std::formatter : std::formatter{ + auto format(const PokemonAutomation::ScreenEvents& events, std::format_context& context) const{ + auto out = context.out(); + bool first = true; + for (const auto& [event, name] : PokemonAutomation::SCREEN_EVENT_NAMES()){ + if (!events.has(event)){ + continue; + } + out = std::format_to(out, "{}{}", first ? "" : ", ", name); + first = false; + } + return first ? std::format_to(out, "nothing") : out; + } +}; + +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp new file mode 100644 index 0000000000..b451e6a3c8 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.cpp @@ -0,0 +1,119 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "ScreenEventTypes.h" +#include "ScreenFades.h" +#include "ScreenEvents.h" + +namespace PokemonAutomation{ + + +namespace{ + +// A fade is uniform over the middle 80% of the screen, as the shared +// Black/WhiteScreenWatcher test: a dim cave floor is textured, no fade. +constexpr double FADE_MAX_STDDEV_SUM = 10; +// Dark up to an rgb sum of 225: a cave mouth's fade is dim, not black. +constexpr double FADE_MAX_DARK_SUM = 225; +// White from an rgb sum of 500, as the shared WhiteScreenWatcher's. +constexpr double FADE_MIN_WHITE_SUM = 500; + +} // namespace + + +const EnumStringMap& SCREEN_EVENT_NAMES(){ + static const EnumStringMap names{ + {ScreenEvent::FADE_BLACK, "black fade"}, + {ScreenEvent::FADE_WHITE, "white fade"}, + {ScreenEvent::LOCATION_BANNER, "location banner"}, + {ScreenEvent::DIALOG_BOX, "dialog box"}, + {ScreenEvent::CHOICE_BOX, "choice box"}, + {ScreenEvent::INTERACT_PROMPT, "interact prompt"}, + {ScreenEvent::BATTLE_STARTED, "battle started"}, + {ScreenEvent::BATTLE_ENDED, "battle ended"}, + {ScreenEvent::PLAYER_CAN_MOVE, "player can move"}, + {ScreenEvent::SCREEN_STILL, "screen still"}, + {ScreenEvent::SWITCH_HOME_MENU, "Switch HOME menu"}, + }; + return names; +} + +const EnumStringMap& WARP_FADE_NAMES(){ + static const EnumStringMap names{ + {WarpFade::NONE, "no fade"}, + {WarpFade::BLACK, "black fade"}, + {WarpFade::WHITE, "white fade"}, + }; + return names; +} + + +ScreenEvents::ScreenEvents(std::initializer_list events){ + for (ScreenEvent event : events){ + set(event, true); + } +} +void ScreenEvents::set(ScreenEvent event, bool on){ + if (on){ + m_bits |= bit(event); + }else{ + m_bits &= ~bit(event); + } +} + +ScreenEvents fade_events(WarpFade fade){ + switch (fade){ + case WarpFade::NONE: + return {}; + case WarpFade::BLACK: + return {ScreenEvent::FADE_BLACK}; + case WarpFade::WHITE: + return {ScreenEvent::FADE_WHITE}; + } + return {}; +} +WarpFade fade_in(const ScreenEvents& events){ + if (events.has(ScreenEvent::FADE_BLACK)){ + return WarpFade::BLACK; + } + if (events.has(ScreenEvent::FADE_WHITE)){ + return WarpFade::WHITE; + } + return WarpFade::NONE; +} + + +// Black is is_black() (SolidColorTest, BlackScreenDetector's test) with +// the fade's thresholds. +// White is not is_white(): that also wants an even grey, and a fade to +// white can be tinted. +WarpFade fade_of(const ImageStats& middle){ + if (is_black(middle, FADE_MAX_DARK_SUM, FADE_MAX_STDDEV_SUM)){ + return WarpFade::BLACK; + } + if (middle.stddev.sum() > FADE_MAX_STDDEV_SUM){ + return WarpFade::NONE; + } + return middle.average.sum() >= FADE_MIN_WHITE_SUM ? WarpFade::WHITE : WarpFade::NONE; +} +WarpFade fade_of_screen(const ImageViewRGB32& screen){ + return fade_of(image_stats(extract_box_reference(screen, ImageFloatBox(0.1, 0.1, 0.8, 0.8)))); +} + + +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b){ + if (!a || !b || a->width() != b->width() || a->height() != b->height()){ + return false; + } + return ImageMatch::pixel_RMSD(a, b) < 2.0; +} + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h new file mode 100644 index 0000000000..318a062f95 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents.h @@ -0,0 +1,41 @@ +/* Screen Events + * + * From: https://github.com/PokemonAutomation/ + * + * A game's detectors, read frame by frame as shared screen events. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_H + +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// A game's detectors, as events. It may keep state between frames, so one +// observer watches one stream. +class ScreenEventObserver{ +public: + virtual ~ScreenEventObserver() = default; + + // What `frame` shows; `now` is the caller's clock. A null frame shows + // nothing. + virtual ScreenEvents observe(const VideoSnapshot& frame, WallClock now) = 0; + + // The boxes the detectors read, for the video overlay. + virtual void make_overlays(VideoOverlaySet& items) const = 0; +}; + + +// Whether the screen barely changed from `a` to `b` (SCREEN_STILL). +bool same_screen(const VideoSnapshot& a, const VideoSnapshot& b); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp new file mode 100644 index 0000000000..40787d88b9 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp @@ -0,0 +1,163 @@ +/* Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The shared event vocabulary, fades and the still-screen test. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "ScreenEventTypes.h" +#include "ScreenFades.h" +#include "ScreenEvents.h" +#include "ScreenEvents_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +UnitTestResult expect_equal(const std::string& what, const std::string& got, const std::string& expected){ + if (got == expected){ + return true; + } + return std::format("{}: expected \"{}\", got \"{}\"", what, expected, got); +} + + +UnitTestResult test_set_and_describe(){ + ScreenEvents events{ScreenEvent::DIALOG_BOX, ScreenEvent::PLAYER_CAN_MOVE}; + if (!events.has(ScreenEvent::DIALOG_BOX) || events.has(ScreenEvent::CHOICE_BOX)){ + return "has() disagrees with the constructor's events"; + } + events.set(ScreenEvent::DIALOG_BOX, false); + events.merge(fade_events(WarpFade::WHITE)); + events.merge(fade_events(WarpFade::NONE)); + if (fade_in(events) != WarpFade::WHITE){ + return "fade_events(WHITE) did not give a white fade"; + } + if (std::format("{}", ScreenEvents()) != "nothing"){ + return "an empty set does not print as nothing"; + } + return expect_equal("std::format()", std::format("{}", events), "white fade, player can move"); +} + +// A screen filled with one colour, or a checkerboard of two. +ImageRGB32 screen_of(uint32_t a, uint32_t b){ + ImageRGB32 out(64, 36); + for (size_t y = 0; y < out.height(); y++){ + for (size_t x = 0; x < out.width(); x++){ + out.pixel(x, y) = (x + y) % 2 == 0 ? a : b; + } + } + return out; +} + +UnitTestResult test_fade_of_screen(){ + struct Row{ + std::string name = {}; + uint32_t a = 0; + uint32_t b = 0; + WarpFade fade = WarpFade::NONE; + }; + const std::vector rows = { + {"black", 0xff000000, 0xff000000, WarpFade::BLACK}, + {"a cave mouth's dim fade (60 a channel)", 0xff3c3c3c, 0xff3c3c3c, WarpFade::BLACK}, + {"white", 0xffffffff, 0xffffffff, WarpFade::WHITE}, + {"grey, neither", 0xff787878, 0xff787878, WarpFade::NONE}, + {"a dark textured floor", 0xff101010, 0xff505050, WarpFade::NONE}, + }; + for (const Row& row : rows){ + const WarpFade got = fade_of_screen(screen_of(row.a, row.b)); + if (got != row.fade){ + return std::format("{}: expected {}, got {}", row.name, WARP_FADE_NAMES().get_string(row.fade), WARP_FADE_NAMES().get_string(got)); + } + } + return true; +} + +UnitTestResult test_same_screen(){ + const WallClock now = WallClock::min(); + const VideoSnapshot black(screen_of(0xff000000, 0xff000000), now); + const VideoSnapshot black_again(screen_of(0xff000000, 0xff000000), now); + const VideoSnapshot white(screen_of(0xffffffff, 0xffffffff), now); + ImageRGB32 small(32, 18); + small.fill(0xff000000); + const VideoSnapshot black_small(std::move(small), now); + if (!same_screen(black, black_again)){ + return "two black frames are not the same screen"; + } + if (same_screen(black, white)){ + return "black and white frames are the same screen"; + } + if (same_screen(black, black_small)){ + return "frames of different sizes are the same screen"; + } + if (same_screen(black, VideoSnapshot())){ + return "a missing frame is the same screen"; + } + return true; +} + + +struct ScreenEventsCase{ + struct Input{ + UnitTestResult (*body)() = nullptr; + }; + struct Expectation{ + }; + + std::string name = {}; + Input input = {}; + Expectation expectation = {}; +}; + +class Test_ScreenEvents : public UnitTest{ +public: + explicit Test_ScreenEvents(ScreenEventsCase test_case) + : UnitTest(std::format("CommonTools::ScreenEvents - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_case.input.body(); + } + +private: + ScreenEventsCase m_case; +}; + +} // namespace + + +const ScreenEventsCase SCREEN_EVENTS_CASES[] = { + { + .name = "a set of events, its fade and its log text", + .input = {.body = test_set_and_describe}, + .expectation = {}, + }, + { + .name = "a fade is uniform, dark or white; a textured floor is none", + .input = {.body = test_fade_of_screen}, + .expectation = {}, + }, + { + .name = "the same screen: equal frames of one size", + .input = {.body = test_same_screen}, + .expectation = {}, + }, +}; + +void add_tests_ScreenEvents(UnitTestDatabase& database){ + for (const auto& item : SCREEN_EVENTS_CASES){ + database.add(item); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h new file mode 100644 index 0000000000..3d1c84b44f --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h @@ -0,0 +1,21 @@ +/* Screen Events Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_Tests_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenEvents_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Registers unit tests for the shared screen events. +void add_tests_ScreenEvents(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h new file mode 100644 index 0000000000..e415e07ca9 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenEvents/ScreenFades.h @@ -0,0 +1,32 @@ +/* Screen Fades + * + * From: https://github.com/PokemonAutomation/ + * + * A fade to black or white, read from a screen. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenEvents_ScreenFades_H +#define PokemonAutomation_CommonTools_ScreenEvents_ScreenFades_H + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// The event of a fade: FADE_BLACK, FADE_WHITE, or none for WarpFade::NONE. +ScreenEvents fade_events(WarpFade fade); +// The fade `events` show: BLACK before WHITE if they have both. +WarpFade fade_in(const ScreenEvents& events); + + +// The fade `middle` (the stats of the screen's middle 80%) shows, if any. +WarpFade fade_of(const ImageStats& middle); +WarpFade fade_of_screen(const ImageViewRGB32& screen); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..66f1f11820 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,194 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/PrettyPrint.h" +#include "CommonTools/ScreenEvents/ScreenEvents.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + + +namespace{ + + +// The first of `states` that matches `observation`; null when none does. +ScreenState* match_screen_state(const ScreenObservation& observation, std::vector& states){ + for (ScreenState& state : states){ + if (state.matches && state.matches(observation)){ + return &state; + } + } + return nullptr; +} + + +// The checks between looks: no state may come up too often, and steps must +// keep changing the screen (a waiting state's only for its +// ScreenState::max_still). +class ScreenMachineChecks{ +public: + // A step that changes nothing on screen is a dead end (e.g. a menu the + // states do not know): fail after this many instead of running out the + // visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + // The problem with `state` coming up on `observation`, if any. The + // screen counts as unchanged only when it and the look before both have + // a frame and the frames match (same_screen()); how long, by the looks' + // `when`. + std::optional problem(const ScreenState& state, const ScreenObservation& observation); + // Whether `when` is more than `timeout` after the first look. + bool timed_out(WallClock when, Milliseconds timeout); + +private: + // The stall problem, if any, of `state` on a frame unchanged since + // `m_changed_at`. + std::optional unchanged_problem(const ScreenState& state, WallClock when); + + std::map> m_visits = {}; + VideoSnapshot m_last_frame = {}; + std::optional m_start = std::nullopt; + // When the screen last changed (or was first looked at). + WallClock m_changed_at = WallClock::min(); + size_t m_unchanged = 0; +}; + + +std::optional ScreenMachineChecks::unchanged_problem(const ScreenState& state, WallClock when){ + if (state.max_still > Milliseconds::zero()){ + if (when - m_changed_at > state.max_still){ + return std::format("The screen stayed still on \"{}\" for more than {}s.", state.name, state.max_still.count() / 1000); + } + return std::nullopt; + } + if (++m_unchanged >= MAX_UNCHANGED){ + return std::format("The screen stopped changing: {} steps on \"{}\" did nothing.", MAX_UNCHANGED, state.name); + } + return std::nullopt; +} + +bool ScreenMachineChecks::timed_out(WallClock when, Milliseconds timeout){ + if (!m_start){ + m_start = when; + } + return when - *m_start > timeout; +} + +std::optional ScreenMachineChecks::problem(const ScreenState& state, const ScreenObservation& observation){ + VideoSnapshot frame = observation.frame ? *observation.frame : VideoSnapshot(); + if (same_screen(m_last_frame, frame)){ + std::optional stalled = unchanged_problem(state, observation.when); + if (stalled){ + return stalled; + } + }else{ + m_unchanged = 0; + m_changed_at = observation.when; + } + m_last_frame = std::move(frame); + if (++m_visits[state.name] > state.max_visits){ + return std::format("Screen \"{}\" came up more than {} times: the step for it is not working.", state.name, state.max_visits); + } + return std::nullopt; +} + + +// Logs each change of the state a look matches, once, timed from the +// first look: "screen 2: dialog at 1.5s". +class ScreenChangeLog{ +public: + explicit ScreenChangeLog(const std::function& log) + : m_log(log) + {} + + void screen(const std::string& name, WallClock when); + +private: + const std::function& m_log; + std::optional m_start = std::nullopt; + size_t m_count = 0; + std::string m_last = {}; +}; + + +void ScreenChangeLog::screen(const std::string& name, WallClock when){ + if (!m_start){ + m_start = when; + } + if (m_count > 0 && name == m_last){ + return; + } + m_last = name; + m_count++; + if (!m_log){ + return; + } + const double seconds = std::chrono::duration(when - *m_start).count(); + m_log(std::format("screen {}: {} at {}s", m_count, name, tostr_fixed(seconds, 1))); +} + + +// How `observation` ends the run, if it does: the matching state's checks, +// then its step. +std::optional take_look( + const ScreenObservation& observation, Milliseconds timeout, + std::vector& states, ScreenMachineChecks& checks, ScreenChangeLog& changes +){ + if (checks.timed_out(observation.when, timeout)){ + return ScreenMachineFailure{.reason = std::format("Did not finish within {}s.", timeout.count() / 1000)}; + } + ScreenState* state = match_screen_state(observation, states); + if (state == nullptr){ + return ScreenMachineFailure{.reason = "The screen matches none of the known states."}; + } + changes.screen(state->name, observation.when); + std::optional found = checks.problem(*state, observation); + if (found){ + return ScreenMachineFailure{.reason = std::move(*found)}; + } + if (state->act){ + state->act(); + } + if (state->terminal){ + return ScreenMachineTerminal{.state_name = state->name}; + } + return std::nullopt; +} + + +} // namespace + + + +ScreenMachineResult run_screen_machine( + const std::function& observe, CancellableScope& scope, + std::vector states, Milliseconds timeout, + const std::function& log +){ + // Looks before the run stops even when nothing else ends it (looks with + // no time, states with no visit limit). + constexpr size_t MAX_LOOKS = 10000; + ScreenMachineChecks checks; + ScreenChangeLog changes(log); + for (size_t looks = 0; looks < MAX_LOOKS; looks++){ + scope.throw_if_cancelled(); + std::optional end = take_look(observe(), timeout, states, checks, changes); + if (end){ + return std::move(*end); + } + } + return ScreenMachineFailure{.reason = std::format("Did not finish within {} looks.", MAX_LOOKS)}; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..4bef78474d --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,68 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: look, take that screen's step, look again. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/MoveOnlyFunction.h" +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "ScreenObservation.h" + +namespace PokemonAutomation{ + + +// One recognizable screen: how to tell it, and the step to take on it. +struct ScreenState{ + std::string name = {}; + // Whether a look shows this screen; the first matching state wins. + MoveOnlyFunction matches = nullptr; + // The one step to take on this screen; none for a state that only + // waits. + MoveOnlyFunction act = nullptr; + // Fail once the screen has been seen more than this many times. + size_t max_visits = SIZE_MAX; + // End the run after this state's step. + bool terminal = false; + // How long a waiting state may leave the screen unchanged. Zero: its + // step should change the screen, and repeated steps that don't stall. + Milliseconds max_still = Milliseconds::zero(); +}; + + +// The run succeeded: this terminal state acted. +struct ScreenMachineTerminal{ + std::string state_name = {}; +}; +// The run failed before a terminal state acted. +struct ScreenMachineFailure{ + std::string reason = {}; +}; +// How a run of states ended. +using ScreenMachineResult = std::variant; + +// Runs `states` over looks from `observe` until a terminal one has acted, +// or fails after `timeout` of look time. Logs each change of state to `log` +// when set. +ScreenMachineResult run_screen_machine( + const std::function& observe, CancellableScope& scope, + std::vector states, Milliseconds timeout, + const std::function& log +); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp new file mode 100644 index 0000000000..12fd9b1386 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp @@ -0,0 +1,298 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The screen machine on scripted observations: a run that ends on its + * terminal state, the first matching state winning, the ways one fails (no state + * matches, a state comes up too often, the time or the looks run out, a + * waiting state stays still too long, steps that change nothing), and the + * log's lines. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "ScreenMachine.h" +#include "ScreenMachine_Tests.h" + +namespace PokemonAutomation{ + + +namespace{ + +using namespace std::chrono_literals; + + +std::string joined(const std::vector& items, std::string_view separator){ + std::string text; + for (const std::string& item : items){ + text += text.empty() ? item : std::format("{}{}", separator, item); + } + return text; +} + + +// One scripted look: its time from the first, what it shows, and its frame +// as one uniform shade (none: no frame, as in a simulation). +struct Look{ + Milliseconds at = 0ms; + ScreenEvents events = {}; + std::optional shade = std::nullopt; +}; + +ScreenObservation observation_of(const Look& look, WallClock start){ + ScreenObservation observation; + observation.when = start + look.at; + observation.events = look.events; + if (look.shade){ + ImageRGB32 image(16, 16); + image.fill(*look.shade); + observation.frame = VideoSnapshot(std::move(image), observation.when); + } + return observation; +} + +// `count` looks a second apart from `from` s, each showing `events` on +// a frame of `shade`, if any. +std::vector looks_every_second(int from, int count, ScreenEvents events, std::optional shade){ + std::vector looks; + for (int second = from; second < from + count; second++){ + looks.push_back({.at = std::chrono::seconds(second), .events = events, .shade = shade}); + } + return looks; +} + +// `first`, then `then`. +std::vector concat(std::vector first, const std::vector& then){ + first.insert(first.end(), then.begin(), then.end()); + return first; +} + +constexpr uint32_t BLACK = 0xff000000; +constexpr uint32_t WHITE = 0xffffffff; + + +// The states every case runs, in this order: +// - "fade" on FADE_BLACK: waits, the screen may stay still up to 30 s; +// - "choice" on CHOICE_BOX: terminal; +// - "prompt" on INTERACT_PROMPT: a step, no visit limit; +// - "text" on DIALOG_BOX: a step, at most 10 visits. +// Each step's name goes to `acted`. +std::vector test_states(std::vector& acted){ + const auto state = [&acted](std::string name, ScreenEvent event){ + ScreenState result; + result.name = name; + result.matches = [event](const ScreenObservation& observation){ + return observation.events.has(event); + }; + result.act = [&acted, name]{ + acted.push_back(name); + }; + return result; + }; + std::vector states; + states.push_back(state("fade", ScreenEvent::FADE_BLACK)); + states.back().act = nullptr; + states.back().max_still = 30s; + states.push_back(state("choice", ScreenEvent::CHOICE_BOX)); + states.back().terminal = true; + states.push_back(state("prompt", ScreenEvent::INTERACT_PROMPT)); + states.push_back(state("text", ScreenEvent::DIALOG_BOX)); + states.back().max_visits = 10; + return states; +} + + +struct RunCase{ + std::string name = {}; + struct Input{ + // The looks, the last repeating. + std::vector looks = {}; + Milliseconds timeout = 600s; + }; + Input input = {}; + struct Expectation{ + // The terminal state, or "failed: ". + std::string end = {}; + // The steps taken, joined by ", "; none: not checked. + std::optional acted = std::nullopt; + // The log's lines, joined by "; "; none: not checked. + std::optional log = std::nullopt; + }; + Expectation expectation = {}; +}; + +const RunCase RUN_CASES[] = { + { + .name = "a run ends on its terminal state, after each step", + .input = {.looks = { + {.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 1s, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 2s, .events = {ScreenEvent::CHOICE_BOX}}, + }}, + .expectation = {.end = "choice", .acted = "prompt, text, choice"}, + }, + { + .name = "the first state that matches a look wins", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::DIALOG_BOX, ScreenEvent::CHOICE_BOX}}}}, + .expectation = {.end = "choice", .acted = "choice"}, + }, + { + .name = "the log has each change of screen once, timed from the first look", + .input = {.looks = { + {.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 1s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 2500ms, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 3s, .events = {ScreenEvent::CHOICE_BOX}}, + }}, + .expectation = { + .end = "choice", + .log = "screen 1: prompt at 0.0s; screen 2: text at 2.5s; screen 3: choice at 3.0s", + }, + }, + { + .name = "no state matches", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::BATTLE_STARTED}}}}, + .expectation = {.end = "failed: The screen matches none of the known states.", .acted = ""}, + }, + { + .name = "a look with nothing on it matches no state", + .input = {.looks = {{.at = 0s, .events = {}}}}, + .expectation = {.end = "failed: The screen matches none of the known states.", .acted = ""}, + }, + { + .name = "a state comes up more often than its limit", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/1, {ScreenEvent::DIALOG_BOX}, std::nullopt)}, + .expectation = {.end = "failed: Screen \"text\" came up more than 10 times: the step for it is not working."}, + }, + { + .name = "the looks' time runs out", + .input = { + .looks = { + {.at = 0s, .events = {ScreenEvent::DIALOG_BOX}}, + {.at = 1s, .events = {ScreenEvent::INTERACT_PROMPT}}, + {.at = 2s, .events = {ScreenEvent::DIALOG_BOX}}, + }, + .timeout = 1s, + }, + .expectation = {.end = "failed: Did not finish within 1s.", .acted = "text, prompt"}, + }, + { + .name = "looks with no time and a state with no limit run out of looks", + .input = {.looks = {{.at = 0s, .events = {ScreenEvent::INTERACT_PROMPT}}}}, + .expectation = {.end = "failed: Did not finish within 10000 looks."}, + }, + { + .name = "a waiting state may stay still up to its limit", + .input = {.looks = concat( + looks_every_second(/*from=*/0, /*count=*/12, {ScreenEvent::FADE_BLACK}, BLACK), + {{.at = 12s, .events = {ScreenEvent::CHOICE_BOX}, .shade = WHITE}} + )}, + .expectation = {.end = "choice", .acted = "choice"}, + }, + { + .name = "a waiting state still past its limit fails", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/40, {ScreenEvent::FADE_BLACK}, BLACK)}, + .expectation = {.end = "failed: The screen stayed still on \"fade\" for more than 30s."}, + }, + { + .name = "steps that change nothing on screen stall", + .input = {.looks = looks_every_second(/*from=*/0, /*count=*/12, {ScreenEvent::DIALOG_BOX}, WHITE)}, + .expectation = { + .end = "failed: The screen stopped changing: 8 steps on \"text\" did nothing.", + .acted = "text, text, text, text, text, text, text, text", + }, + }, + { + .name = "a changed frame starts the stall count again", + .input = {.looks = concat( + concat( + looks_every_second(/*from=*/0, /*count=*/5, {ScreenEvent::DIALOG_BOX}, WHITE), + looks_every_second(/*from=*/5, /*count=*/5, {ScreenEvent::DIALOG_BOX}, BLACK) + ), + {{.at = 10s, .events = {ScreenEvent::CHOICE_BOX}, .shade = WHITE}} + )}, + .expectation = {.end = "choice"}, + }, + { + .name = "looks with no frame never stall", + .input = {.looks = concat( + looks_every_second(/*from=*/0, /*count=*/9, {ScreenEvent::DIALOG_BOX}, std::nullopt), + {{.at = 9s, .events = {ScreenEvent::CHOICE_BOX}}} + )}, + .expectation = {.end = "choice"}, + }, +}; + +// The terminal state's name, or "failed: ". +std::string end_of(const ScreenMachineResult& result){ + const ScreenMachineFailure* failure = std::get_if(&result); + return failure != nullptr + ? std::format("failed: {}", failure->reason) + : std::get(result).state_name; +} + +UnitTestResult check_run(const RunCase& test, CancellableScope& scope){ + std::vector acted; + std::vector lines; + const std::function log = [&lines](std::string_view line){ + lines.emplace_back(line); + }; + const WallClock start = current_time(); + size_t next = 0; + const std::function observe = [&]{ + const Look& look = test.input.looks[std::min(next, test.input.looks.size() - 1)]; + next++; + return observation_of(look, start); + }; + const std::string end = end_of(run_screen_machine(observe, scope, test_states(acted), test.input.timeout, log)); + if (end != test.expectation.end){ + return std::format("expected \"{}\", got \"{}\"", test.expectation.end, end); + } + if (test.expectation.acted && joined(acted, ", ") != *test.expectation.acted){ + return std::format("expected the steps \"{}\", got \"{}\"", *test.expectation.acted, joined(acted, ", ")); + } + if (test.expectation.log && joined(lines, "; ") != *test.expectation.log){ + return std::format("expected the log \"{}\", got \"{}\"", *test.expectation.log, joined(lines, "; ")); + } + return true; +} + + +class Test_RunScreenMachine : public UnitTest{ +public: + explicit Test_RunScreenMachine(const RunCase& test) + : UnitTest(std::format("CommonTools::ScreenMachine - run: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope& scope) const override{ + return check_run(m_test, scope); + } + +private: + const RunCase& m_test; +}; + + +} // namespace + + +void add_tests_ScreenMachine(UnitTestDatabase& database){ + for (const RunCase& test : RUN_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h new file mode 100644 index 0000000000..b0eac935f7 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h @@ -0,0 +1,22 @@ +/* Screen Machine Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_ScreenMachine_Tests_H +#define PokemonAutomation_CommonTools_ScreenMachine_ScreenMachine_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Registers the screen machine's tests: scripted observations, no frames +// from disk. +void add_tests_ScreenMachine(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h new file mode 100644 index 0000000000..2ee4c6c64d --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenObservation.h @@ -0,0 +1,37 @@ +/* Screen Observation + * + * From: https://github.com/PokemonAutomation/ + * + * What one look at the game showed, as facts rather than pixels. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenObservation_H +#define PokemonAutomation_CommonTools_ScreenObservation_H + +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/ScreenEvents/ScreenEventTypes.h" + +namespace PokemonAutomation{ + + +// One look at the game. +struct ScreenObservation{ + WallClock when = WallClock::min(); + ScreenEvents events = {}; + // Named numbers the game's readers found, by names the game chooses; + // "heading" is degrees clockwise from north, [0, 360). + std::map> readings = {}; + // The frame looked at; none in a simulation. + std::optional frame = std::nullopt; +}; + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 45745e0d81..904f6e3954 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -29,6 +29,8 @@ #include "CommonFramework/Windows/LabControl.h" #include "CommonTools/VisualDetectors/BlackBorderDetector.h" #include "CommonTools/OCR/OCR_Tests.h" +#include "CommonTools/ScreenEvents/ScreenEvents_Tests.h" +#include "CommonTools/ScreenMachine/ScreenMachine_Tests.h" #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" @@ -80,6 +82,8 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ OCR::add_tests(ret); Kernels::add_tests(ret); Pokemon::add_tests_AutoStory(ret); + add_tests_ScreenEvents(ret); + add_tests_ScreenMachine(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 6e9971e0ab..5eefe64cc4 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -925,6 +925,17 @@ file(GLOB LIBRARY_SOURCES Source/CommonTools/Options/TrainOCRModeOption.h Source/CommonTools/Resources/SpriteDatabase.cpp Source/CommonTools/Resources/SpriteDatabase.h + Source/CommonTools/ScreenEvents/ScreenEventTypes.h + Source/CommonTools/ScreenEvents/ScreenEvents.cpp + Source/CommonTools/ScreenEvents/ScreenEvents.h + Source/CommonTools/ScreenEvents/ScreenEvents_Tests.cpp + Source/CommonTools/ScreenEvents/ScreenEvents_Tests.h + Source/CommonTools/ScreenEvents/ScreenFades.h + Source/CommonTools/ScreenMachine/ScreenMachine.cpp + Source/CommonTools/ScreenMachine/ScreenMachine.h + Source/CommonTools/ScreenMachine/ScreenMachine_Tests.cpp + Source/CommonTools/ScreenMachine/ScreenMachine_Tests.h + Source/CommonTools/ScreenMachine/ScreenObservation.h Source/CommonTools/StartupChecks/StartProgramChecks.cpp Source/CommonTools/StartupChecks/StartProgramChecks.h Source/CommonTools/StartupChecks/VideoResolutionCheck.cpp