diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index efb3c80a04..87877cbfb6 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -245,7 +245,11 @@ jobs: uses: actions/checkout@v7 if: inputs.run-tests with: - repository: 'PokemonAutomation/CommandLineTests' + # The fork's fixtures: upstream's plus the PLZA AutoStory frames (Confirm Choices, opening and arrival folders), dayfine/CommandLineTests PR #18, + # and the 01.01 banner and controls hint in English and Italian (branch plza/hint-banner-any-language), + # and the 01.00 new game frames, HOME tiles (FireRed first, Z-A second too) and the 01.02 Italian choice menu and talk prompt (branch plza/new-game-intro), and the 03.02b battle frame with no lock-on and Map & Missions card (branch plza/battle-panel), and Wild Zone 1's full map frames (branch plza/zone1-map-pin), and 03.09's lab gate dialog and 03.08b's wild battle at the zone pillar (branch plza/0309-gate-frames), and Wild Zone 1's frames with the lead's panel up and no battle (branch plza/0308b-pillar-frame), and the first X's Save card over the menu and the menu after it (branch plza/menu-save-frames), and the 01.02 French choice menu (branch plza/choice-fr), and 02.05's German lobby with a party of one and the elevator's A prompt (branch plza/party-slots). Merged with upstream main b79ca7f (branch plza/upstream-sync-b79ca7f) for the PokemonHome BoxView fixtures and 7-value FRLG StatsReader goldens, plus the console's dark-theme name keyboard (empty and with text) and HOME frames (branch plza/dark-theme-keyboard). Fork-only pin: dropped at upstream handoff. + repository: 'dayfine/CommandLineTests' + ref: '43b922a5ba061e99ac059fd91c49d72a76317ac7' path: 'CommandLineTests' # The unit tests assume the downloadable resources they need are already downloaded diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 66d4f0cc4b..4987adf588 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -117,6 +117,10 @@ if(PA_CORE_ONLY) include(cmake/CoreLib.cmake) if(PA_LOGIC_TESTS) include(cmake/LogicTests.cmake) + # PLZA AutoStory's checkpoint files, which its logic tests read. + include(cmake/EmbedTextFiles.cmake) + pa_embed_text_files(LogicLib Source/VideoGames/PokemonLZA/Programs/AutoStory/StoryData + VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryStoryFiles.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_checkpoint_json) endif() if(COMMAND_LINE_EXE) include(Source/CommandLine/CommandLineExecutable.cmake) @@ -233,6 +237,13 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) +# Data files built into the program (see cmake/EmbedTextFiles.cmake). +include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/StoryData + VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryStoryFiles.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_checkpoint_json) +pa_embed_text_files(SerialProgramsLib Source/VideoGames/PokemonLZA/Programs/AutoStory/ReferenceData + VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.h PokemonAutomation::NintendoSwitch::PokemonLZA autostory_reference_json) + # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 904f6e3954..6fdd761201 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -34,6 +34,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_CheckOnlineDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_FailedToConnectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" #include "NintendoSwitch/Controllers/SysbotBase/SysbotBase_Connection.h" @@ -86,6 +87,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_ScreenMachine(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); + NintendoSwitch::add_tests_SystemKeyboardDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); NintendoSwitch::PokemonBDSP::add_tests_DialogDetector(ret); diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp new file mode 100644 index 0000000000..f8bc2b5445 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -0,0 +1,385 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "Tests/ImageUnitTest.h" +#include "NintendoSwitch_SystemKeyboardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +namespace{ + +// Centres of the five rows of keys, and of the first column and the spacing +// between columns, as fractions of the screen. +constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770, 0.858}; +constexpr double KEY_COL_X0 = 0.077; +constexpr double KEY_COL_STEP = 0.075; + +// In the right column, Return spans rows 1-2 and OK rows 3-4. These are the +// gaps between those keys. +constexpr double RIGHT_COLUMN_RETURN_TOP = 0.548; +constexpr double RIGHT_COLUMN_OK_TOP = 0.725; + +// The cursor border and everything else the reader looks at, from the top of the +// digit row to the bottom of the bottom row. +const ImageFloatBox KEYS_AREA(0.0, 0.43, 1.0, 0.49); + +// The labels of the bottom row's page keys (globe to the accent pages), inside +// the border a cursor on one of them draws (y 875-979 at 1080). In the dark +// theme the current page's label and underline are cyan (about 0, 230, 210, +// y 909-955 for ABC), like the cursor, so the cursor read skips this band. +const ImageFloatBox PAGE_LABELS(0.0, 0.829, 0.49, 0.065); + +// The bar below the keys, which tells the console's theme. +const ImageFloatBox BOTTOM_BAR(0.10, 0.935, 0.35, 0.030); + +enum class KeyboardTheme{ + LIGHT, + DARK, // The console's "Basic Black" theme. +}; + +// The bar is white (245) in the light theme and flat grey (76) in the dark +// one; nothing if it is neither. Measured on Legends: Z-A. +std::optional keyboard_theme(const ImageViewRGB32& screen){ + const ImageStats bar = image_stats(extract_box_reference(screen, BOTTOM_BAR)); + if (is_white(bar)){ + return KeyboardTheme::LIGHT; + } + if (is_grey(bar, /*min_rgb_sum=*/180, /*max_rgb_sum=*/270)){ + return KeyboardTheme::DARK; + } + return std::nullopt; +} + +bool in_box(double x, double y, const ImageFloatBox& box){ + return x >= box.x && x < box.x + box.width && y >= box.y && y < box.y + box.height; +} + +// The cursor border pulses between bright cyan (about 50, 240, 220) and a darker +// teal (about 0, 185, 210), measured on Legends: Z-A; in the dark theme it is +// about (115, 245, 250). The blue of the OK button and of the light theme's ABC +// underline (40, 90, 250) is not green enough to match. +bool is_cursor_pixel(uint32_t pixel){ + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + return red < 130 && green > 160 && blue > 170 && green - red > 60; +} + +uint8_t nearest_row(double y){ + uint8_t row = 0; + for (uint8_t r = 1; r < 5; r++){ + if (std::abs(KEY_ROW_Y[r] - y) < std::abs(KEY_ROW_Y[row] - y)){ + row = r; + } + } + return row; +} + +// The key whose cursor border has its centre at (x, y), or nothing if that is +// off the keyboard. +std::optional key_at(double x, double y){ + const long col = std::lround((x - KEY_COL_X0) / KEY_COL_STEP); + if (col < 0 || col > 11){ + return std::nullopt; + } + if (col < 11){ + return SystemKeyboardKey{nearest_row(y), (uint8_t)col}; + } + if (y < RIGHT_COLUMN_RETURN_TOP){ + return SystemKeyboardKey{0, 11}; + } + if (y < RIGHT_COLUMN_OK_TOP){ + return SystemKeyboardKey{1, 11}; + } + return SystemKeyboardKey{3, 11}; +} + +// The key whose cell holds (x, y), for a point anywhere in KEYS_AREA: the +// columns left of the first and right of the right column belong to them. +SystemKeyboardKey key_cell_at(double x, double y){ + const double col = std::clamp((x - KEY_COL_X0) / KEY_COL_STEP, 0.0, 11.0); + return *key_at(KEY_COL_X0 + col * KEY_COL_STEP, y); +} + +// The cyan pixels in one key's cell, against all pixels of the cell. +struct KeyCellPixels{ + double area = 0; + double count = 0; + double sum_x = 0; + double sum_y = 0; +}; + +// The cursor's border covers about a fifth of its key's cell. A key whose cell +// is more cyan than this is solid cyan, not the cursor: the dark theme's OK +// button once the field has text (0, 233, 211; 60% of its cell). +constexpr double MAX_BORDER_SHARE = 0.35; + +// The cyan pixels of each key's cell in `keys` (KEYS_AREA), by row and column. +using KeyCells = std::array, 5>; +KeyCells cyan_pixels_by_key(const ImageViewRGB32& keys){ + KeyCells cells{}; + for (size_t y = 0; y < keys.height(); y++){ + const double screen_y = KEYS_AREA.y + ((double)y + 0.5) / keys.height() * KEYS_AREA.height; + for (size_t x = 0; x < keys.width(); x++){ + const double screen_x = KEYS_AREA.x + ((double)x + 0.5) / keys.width() * KEYS_AREA.width; + if (in_box(screen_x, screen_y, PAGE_LABELS)){ + continue; + } + const SystemKeyboardKey key = key_cell_at(screen_x, screen_y); + KeyCellPixels& cell = cells[key.row][key.col]; + cell.area++; + if (is_cursor_pixel(keys.pixel(x, y))){ + cell.count++; + cell.sum_x += (double)x; + cell.sum_y += (double)y; + } + } + } + return cells; +} + +} // namespace + + + +SystemKeyboardDetector::SystemKeyboardDetector(Color color) + : m_color(color) + , m_key_panel_upper(0.15, 0.532, 0.65, 0.036) + , m_key_panel_lower(0.15, 0.715, 0.65, 0.032) + , m_bottom_bar(BOTTOM_BAR) + , m_ok_button(0.875, 0.760, 0.045, 0.020) +{} +void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_key_panel_upper); + items.add(m_color, m_key_panel_lower); + items.add(m_color, m_bottom_bar); + items.add(m_color, m_ok_button); +} +bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ + const std::optional theme = keyboard_theme(screen); + if (!theme){ + return false; + } + // Each strip crosses the keys and the gaps between them, which gives a + // stddev sum of about 11 (white keys, measured on BDSP and Legends: Z-A) + // or 12 (dark grey keys, about 84, on Legends: Z-A). + const double max_panel_stddev = 20; + const ImageStats upper = image_stats(extract_box_reference(screen, m_key_panel_upper)); + const ImageStats lower = image_stats(extract_box_reference(screen, m_key_panel_lower)); + const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); + if (*theme == KeyboardTheme::DARK){ + const bool panel = + is_grey(upper, /*min_rgb_sum=*/210, /*max_rgb_sum=*/300, max_panel_stddev) || + is_grey(lower, /*min_rgb_sum=*/210, /*max_rgb_sum=*/300, max_panel_stddev); + // OK is flat grey (100, 100, 100) while the field is empty and solid + // cyan (0, 233, 211) with text. + return panel && ok.stddev.sum() < 10; + } + const bool panel = + is_white(upper, /*min_rgb_sum=*/500, max_panel_stddev) || + is_white(lower, /*min_rgb_sum=*/500, max_panel_stddev); + if (!panel){ + return false; + } + // OK is blue (40, 93, 255), or flat grey (221, 221, 221) while the field is + // empty (disabled). Measured on BDSP and Legends: Z-A. + return is_solid(ok, {0.103, 0.240, 0.657}) || is_grey(ok, 600, 690); +} + + +std::optional read_system_keyboard_cursor(const ImageViewRGB32& screen){ + // The cursor is a cyan border around one key. Neighbouring keys overlap that border, + // so take the centroid of all cyan pixels outside solid cyan keys and map it to + // the nearest key. + const ImageViewRGB32 keys = extract_box_reference(screen, KEYS_AREA); + const KeyCells cells = cyan_pixels_by_key(keys); + double count = 0, sum_x = 0, sum_y = 0; + for (const auto& row : cells){ + for (const KeyCellPixels& cell : row){ + if (cell.count > MAX_BORDER_SHARE * cell.area){ + continue; + } + count += cell.count; + sum_x += cell.sum_x; + sum_y += cell.sum_y; + } + } + // The border is about 0.18% of the screen; a few stray pixels are not a cursor. + if (count < 0.0005 * screen.width() * screen.height()){ + return std::nullopt; + } + return key_at( + KEYS_AREA.x + sum_x / count / keys.width() * KEYS_AREA.width, + KEYS_AREA.y + sum_y / count / keys.height() * KEYS_AREA.height + ); +} + +bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ + // Left part of the backspace key, clear of its icon and of the cursor border. + // Light theme: light grey (221) with an empty field, dark grey (52) with + // text. Dark theme: grey (100) with an empty field, white (254) with text. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.878, 0.470, 0.008, 0.060))); + if (keyboard_theme(screen) == KeyboardTheme::DARK){ + return !is_grey(stats, /*min_rgb_sum=*/255, /*max_rgb_sum=*/345); + } + return stats.average.sum() < 400; +} + +size_t system_keyboard_text_pixels(const ImageViewRGB32& screen){ + // The field, above its underline, which is white in some games. + const ImageViewRGB32 field = extract_box_reference(screen, ImageFloatBox(0.03, 0.17, 0.87, 0.105)); + size_t count = 0; + for (size_t y = 0; y < field.height(); y++){ + for (size_t x = 0; x < field.width(); x++){ + const uint32_t pixel = field.pixel(x, y); + if (((pixel >> 16) & 0xff) > 190 && ((pixel >> 8) & 0xff) > 190 && (pixel & 0xff) > 190){ + count++; + } + } + } + return count; +} + + + +namespace{ + +// One screenshot and what each function should read from it. +class Test_SystemKeyboardDetector : public ImageUnitTest{ +public: + // `cursor`: the key under the cursor, or nothing for a screen without the keyboard. + Test_SystemKeyboardDetector(const std::string& image, std::optional cursor, bool has_text) + : ImageUnitTest(std::format("NintendoSwitch::SystemKeyboardDetector - {}", image), {image}) + , m_cursor(cursor) + , m_has_text(has_text) + {} + + virtual UnitTestResult run_on_images( + Logger& logger, CancellableScope& scope, + const std::vector& images + ) const override{ + const ImageRGB32& image = images[0]; + const bool keyboard = SystemKeyboardDetector().detect(image); + if (keyboard != m_cursor.has_value()){ + return std::format("detect() returned {}.", keyboard); + } + if (!keyboard){ + return true; + } + const std::optional cursor = read_system_keyboard_cursor(image); + if (cursor != m_cursor){ + return cursor + ? std::format("Read the cursor at ({}, {}).", cursor->row, cursor->col) + : std::format("Read no cursor."); + } + const bool has_text = system_keyboard_field_has_text(image); + if (has_text != m_has_text){ + return std::format("system_keyboard_field_has_text() returned {}.", has_text); + } + // An empty field has no white text pixels; one typed character has many. + const size_t text_pixels = system_keyboard_text_pixels(image); + if ((text_pixels >= 20) != m_has_text){ + return std::format("system_keyboard_text_pixels() returned {}.", text_pixels); + } + return true; + } + +private: + std::optional m_cursor; + bool m_has_text; +}; + +std::string test_image_path(std::string_view image){ + return std::format("NintendoSwitch/SystemKeyboardDetector/{}", image); +} + +// The cursor on keys other than "1". +void add_cursor_tests(UnitTestDatabase& database){ + // Legends: Z-A, English and Spanish prompts. The + // border pulses between bright cyan and a darker teal; T, / and ! are dark. + const std::pair cursors[] = { + {"PLZA-Cursor-T.png", {1, 4}}, + {"PLZA-Cursor-Slash.png", {1, 10}}, + {"PLZA-Cursor-Exclamation.png", {3, 10}}, + {"PLZA-Cursor-0.png", {0, 9}}, + {"PLZA-Cursor-Minus.png", {0, 10}}, + {"PLZA-Cursor-Backspace.png", {0, 11}}, + {"PLZA-Cursor-Return.png", {1, 11}}, + {"PLZA-Cursor-OK.png", {3, 11}}, + {"PLZA-Cursor-Globe.png", {4, 0}}, + {"PLZA-Cursor-NTilde.png", {4, 5}}, + {"PLZA-Cursor-Space.png", {4, 8}}, + }; + for (const auto& [image, cursor] : cursors){ + database.add(test_image_path(image), cursor, false); + } +} + +// Tests whose screen is not captured yet. +void add_planned_tests(UnitTestDatabase& database){ + const SystemKeyboardKey key_1{0, 0}; + // Screens still to be captured: AZERTY needs a console whose system language + // uses it (the layout follows the system, not the game). Each test is added + // once its image is in the test repo. + struct Planned{ + std::string_view image; + std::optional cursor; + bool has_text; + }; + const Planned planned[] = { + {"AZERTY-Empty.png", key_1, false}, + {"PLZA-Korean.png", std::nullopt, false}, + }; + for (const Planned& test : planned){ + const std::string image = test_image_path(test.image); + if (Filesystem::exists(std::format("{}{}", UNIT_TEST_RESOURCE_PATH(), image))){ + database.add(image, test.cursor, test.has_text); + } + } +} + +} // namespace + + +void add_tests_SystemKeyboardDetector(UnitTestDatabase& database){ + const SystemKeyboardKey key_1{0, 0}; + // The OK button is blue with text in the field and grey without. + database.add(test_image_path("BDSP-PlayerName-Empty.png"), key_1, false); + database.add(test_image_path("BDSP-FriendName-Typed.png"), key_1, true); + for (std::string_view language : {"English", "French", "German", "Italian", "Japanese", "Spanish"}){ + database.add(test_image_path(std::format("PLZA-{}-Empty.png", language)), key_1, false); + } + // The console's dark theme (Basic Black), console B 2026-11-18: the cyan + // ABC label must not pull the cursor read off "1". + database.add(test_image_path("PLZA-English-Empty-Dark.png"), key_1, false); + // "K" typed: OK turns solid cyan, which is not the cursor (on "8"). + database.add(test_image_path("PLZA-English-Typed-Dark.png"), SystemKeyboardKey{0, 7}, true); + for (std::string_view image : {"BDSP-Dialog.png", "HomeMenu.png", "HomeMenu-OnlineLicensePopup.png", "UserSelect.png"}){ + database.add(test_image_path(image), std::nullopt, false); + } + add_cursor_tests(database); + add_planned_tests(database); +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h new file mode 100644 index 0000000000..f5224a5c8b --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -0,0 +1,101 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own on-screen keyboard, which games open for names (e.g. the + * player's name in BDSP and in Legends: Z-A): detect it, read the key under its + * cursor, and tell whether its text field is empty. Typing with it is in + * Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h. + * + * Supported: the Latin keyboard (QWERTY; AZERTY shares the key grid but has no + * test screenshot yet). Its layout follows the console's system language, not + * the game's. Measured on 1920x1080 Switch 1 captures of BDSP's and Legends: + * Z-A's name entry, the latter with the game in English, Spanish, German, + * Italian, French and Japanese (all QWERTY on an English console). Not + * supported: the Korean keyboard, whose candidate strip + * moves the keys down (the detector returns false on it), and probably the + * Chinese ones (not tested). Switch 2 is not tested. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H + +#include +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// The key panel, the bar below it, and the OK button: white panel and bar and a +// blue OK (grey while the field is empty) in the light theme, dark grey panel +// and bar and a flat OK in the dark one (Basic Black). Works wherever the +// cursor is. +class SystemKeyboardDetector : public StaticScreenDetector{ +public: + SystemKeyboardDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + // Two strips across the key panel, one over the gap between rows 0 and 1 and + // one over the gap between rows 2 and 3. The cursor's border reaches into at + // most one of them, so the panel is white if either is. + ImageFloatBox m_key_panel_upper; + ImageFloatBox m_key_panel_lower; + ImageFloatBox m_bottom_bar; + ImageFloatBox m_ok_button; +}; + +// Finds SystemKeyboardDetector on the video, checking every 250 ms. +class SystemKeyboardWatcher : public DetectorToFinder{ +public: + SystemKeyboardWatcher(Color color = COLOR_CYAN) + : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) + {} +}; + + +// A key of the keyboard. +// Rows 0-3 are the character rows (0: digits .. 3: "zxcv"), row 4 is the bottom +// row (globe, Shift, ABC, #+=, the accent pages and Space). +// Columns 0-10 are the character columns. Column 11 is the right column, whose +// keys are taller: backspace is (0, 11), Return (1, 11) and OK (3, 11). +// A row-4 key reports the character column nearest its centre (Space: 8). +struct SystemKeyboardKey{ + uint8_t row; + uint8_t col; + + bool operator==(const SystemKeyboardKey&) const = default; +}; + +// The key under the cursor, or nothing if no cursor border is visible. +// The keyboard remembers the cursor between uses, so read it, don't assume it. +std::optional read_system_keyboard_cursor(const ImageViewRGB32& screen); + +// Whether the text field holds any text: the backspace key is dark then and light +// grey while the field is empty. Pressing B on an empty field closes the +// keyboard, so delete only while this is true. +bool system_keyboard_field_has_text(const ImageViewRGB32& screen); + +// The number of white pixels in the text field. Each typed character adds some +// (a "1" adds about 110 at 1920x1080), so comparing two counts tells whether a +// key press reached the field. +size_t system_keyboard_text_pixels(const ImageViewRGB32& screen); + + + +// Register the tests of this file. +void add_tests_SystemKeyboardDetector(UnitTestDatabase& database); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h index 7048582ea0..1aa290cdc1 100644 --- a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h @@ -8,10 +8,12 @@ #define PokemonAutomation_NintendoSwitch_KeyboardCodeEntry_H #include +#include #include "Controllers/StandardHid/StandardHid_Keyboard.h" #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" #include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" #include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_CodeEntryTools.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -19,6 +21,13 @@ namespace FastCodeEntry{ +// The D-pad moves from "source" to "destination" on the on-screen keyboard, +// wrapping around the sides when that is shorter, then ENTER_CHAR. +std::vector keyboard_get_path( + KeyboardEntryPosition source, + KeyboardEntryPosition destination +); + // Automatically select between keyboard or pro controller depending on the // controller type. void keyboard_enter_code( diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp new file mode 100644 index 0000000000..511e36ecad --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.cpp @@ -0,0 +1,249 @@ +/* System Keyboard Typing + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/NintendoSwitch_Settings.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_CodeEntryTools.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardCodeEntry.h" +#include "NintendoSwitch_SystemKeyboardTyping.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace FastCodeEntry{ + +using namespace std::chrono_literals; + + +namespace{ + +// Waits after each press. They are longer than the code entry delays because +// the next step reads the screen: the cursor animation and the capture latency +// must be over first. +constexpr Milliseconds MOVE_RELEASE = 200ms; +constexpr Milliseconds WRAP_RELEASE = 400ms; // Wrapping around a side animates for longer. +constexpr Milliseconds SETTLE = 300ms; // Before reading the cursor after moving it. +constexpr Milliseconds SHIFT_RELEASE = 250ms; +constexpr Milliseconds KEY_RELEASE = 300ms; + +// A press made while the keyboard is still sliding in is ignored, so the +// cursor is read again after each try until it is on the target. +constexpr size_t MAX_MOVE_TRIES = 10; + +// Reads of the text field after a key press, SETTLE apart, before giving up. +constexpr size_t MAX_FIELD_READS = 3; + +// The fewest white pixels a typed character adds to the field ("." or "'"). +constexpr size_t MIN_CHARACTER_PIXELS = 10; + + +// The key of each character of `text`. Throws UserSetupError, listing every +// character that has no key, before anything is typed. +std::vector text_keys(Logger& logger, const std::string& text, KeyboardLayout layout){ + const std::map& positions = KEYBOARD_POSITIONS(layout); + std::vector keys; + std::string missing; + for (char ch : text){ + // Letters are listed under their uppercase key. + auto iter = positions.find((char)std::toupper((unsigned char)ch)); + if (iter == positions.end()){ + missing += ch; + }else{ + keys.emplace_back(iter->second); + } + } + if (!missing.empty()){ + throw UserSetupError( + logger, + std::format("System keyboard: \"{}\" has characters with no key on the keyboard: \"{}\".", text, missing) + ); + } + return keys; +} + +// How long to hold each button: the hold of the code entry timings for Switch 1. +Milliseconds button_hold(ProControllerContext& context){ + const ConsoleSettings& settings = ConsoleSettings::instance(); + const CodeEntryDelays delays = context->performance_class() == ControllerPerformanceClass::SerialPABotBase_Wired + ? CodeEntryDelays(settings.CODEBOARD_ENTRY_SWITCH1_WIRED) + : CodeEntryDelays(settings.CODEBOARD_ENTRY_SWITCH1_WIRELESS); + return delays.hold; +} + +// The D-pad direction of a move. +DpadPosition move_direction(CodeEntryAction action){ + switch (action){ + case CodeEntryAction::NORM_MOVE_UP: + case CodeEntryAction::WRAP_MOVE_UP: + return DpadPosition::DPAD_UP; + case CodeEntryAction::NORM_MOVE_RIGHT: + case CodeEntryAction::WRAP_MOVE_RIGHT: + return DpadPosition::DPAD_RIGHT; + case CodeEntryAction::NORM_MOVE_DOWN: + case CodeEntryAction::WRAP_MOVE_DOWN: + return DpadPosition::DPAD_DOWN; + case CodeEntryAction::NORM_MOVE_LEFT: + case CodeEntryAction::WRAP_MOVE_LEFT: + return DpadPosition::DPAD_LEFT; + default: + return DpadPosition::DPAD_NONE; + } +} + +// The moves toward `target`. Off the character grid (the right column or the +// bottom row), one step back onto it, after which the cursor is read again. +std::vector path_toward(SystemKeyboardKey cursor, KeyboardEntryPosition target){ + if (cursor.col == 11){ + return {CodeEntryAction::NORM_MOVE_LEFT}; + } + if (cursor.row == 4){ + return {CodeEntryAction::NORM_MOVE_UP}; + } + std::vector path = keyboard_get_path({cursor.row, cursor.col}, target); + std::erase_if(path, [](CodeEntryAction action){ return !is_move(action); }); + return path; +} + +// The moves as letters for the log, e.g. "DLL" or "R*" for a wrap to the right. +std::string path_to_string(const std::vector& path){ + std::string ret; + for (CodeEntryAction action : path){ + switch (move_direction(action)){ + case DpadPosition::DPAD_UP: + ret += 'U'; + break; + case DpadPosition::DPAD_RIGHT: + ret += 'R'; + break; + case DpadPosition::DPAD_DOWN: + ret += 'D'; + break; + case DpadPosition::DPAD_LEFT: + ret += 'L'; + break; + default: + ret += '?'; + break; + } + if (is_wrap(action)){ + ret += '*'; + } + } + return ret; +} + +std::optional read_cursor(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return std::nullopt; + } + return read_system_keyboard_cursor(screen); +} + +[[noreturn]] void throw_cursor_failure( + VideoStream& stream, + std::optional last, KeyboardEntryPosition target, char ch +){ + std::string message = last + ? std::format("the cursor is at ({}, {}), not on '{}' at ({}, {})", last->row, last->col, ch, target.row, target.col) + : std::format("the cursor could not be read while moving to '{}' at ({}, {})", ch, target.row, target.col); + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("System keyboard: {}, after {} tries.", message, MAX_MOVE_TRIES), + stream + ); +} + +void move_cursor( + VideoStream& stream, ProControllerContext& context, Milliseconds hold, + KeyboardEntryPosition target, char ch +){ + std::optional last; + for (size_t attempt = 0; attempt < MAX_MOVE_TRIES; attempt++){ + const std::optional cursor = read_cursor(stream, context); + if (!cursor){ + stream.logger().log(std::format("System keyboard: no cursor seen, try {}.", attempt + 1)); + pbf_wait(context, SETTLE); + continue; + } + last = cursor; + if (cursor->row == target.row && cursor->col == target.col){ + return; + } + const std::vector path = path_toward(*cursor, target); + stream.logger().log(std::format( + "System keyboard: cursor ({}, {}) -> '{}' ({}, {}): {}, try {}.", + cursor->row, cursor->col, ch, target.row, target.col, path_to_string(path), attempt + 1 + )); + for (CodeEntryAction action : path){ + pbf_press_dpad(context, move_direction(action), hold, is_wrap(action) ? WRAP_RELEASE : MOVE_RELEASE); + } + pbf_wait(context, SETTLE); + } + throw_cursor_failure(stream, last, target, ch); +} + +size_t read_text_pixels(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + return screen ? system_keyboard_text_pixels(screen) : 0; +} + +// Types `ch` with the cursor on its key, and checks that the text field grew. +void press_key(VideoStream& stream, ProControllerContext& context, Milliseconds hold, char ch){ + const size_t before = read_text_pixels(stream, context); + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, hold, SHIFT_RELEASE); + } + pbf_press_button(context, BUTTON_A, hold, KEY_RELEASE); + size_t after = 0; + for (size_t read = 0; read < MAX_FIELD_READS; read++){ + after = read_text_pixels(stream, context); + if (after >= before + MIN_CHARACTER_PIXELS){ + return; + } + pbf_wait(context, SETTLE); + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("System keyboard: pressed '{}', but the text field did not change ({} -> {} white pixels).", ch, before, after), + stream + ); +} + +} // namespace + + + +void type_on_system_keyboard( + VideoStream& stream, ProControllerContext& context, + const std::string& text, + KeyboardLayout layout +){ + const std::vector keys = text_keys(stream.logger(), text, layout); + const Milliseconds hold = button_hold(context); + for (size_t c = 0; c < text.size(); c++){ + move_cursor(stream, context, hold, keys[c], text[c]); + press_key(stream, context, hold, text[c]); + } + context.wait_for_all_requests(); +} + + + +} // namespace FastCodeEntry +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h new file mode 100644 index 0000000000..3d2e2dcc2f --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h @@ -0,0 +1,56 @@ +/* System Keyboard Typing + * + * From: https://github.com/PokemonAutomation/ + * + * Types text on the Switch's on-screen keyboard with a Pro Controller, reading + * the cursor from the video before each key. Unlike keyboard_enter_code(), + * which moves blind from an assumed starting key, it starts wherever the + * keyboard left its cursor and corrects moves the keyboard ignored. + * + * Supported keyboards are those of NintendoSwitch_SystemKeyboardDetector.h: the + * Latin keyboard, QWERTY or AZERTY. Console-tested on Switch 1 (2026-11-13): + * the player's name in Legends: Z-A with the game in English, Spanish, German, + * Italian, French and Japanese (QWERTY, English system language), and earlier + * the player's and rival's names in BDSP. AZERTY is not tested. Not + * supported: the Korean and Chinese keyboards. Switch 2 is not tested. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardTyping_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardTyping_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace FastCodeEntry{ + + +// Types `text` on the open keyboard: letters, digits and the symbols on the +// keyboard's first page. An uppercase letter is typed with Shift (left stick +// click), which assumes the keyboard opened with Shift off, as BDSP's and +// Legends: Z-A's do. +// +// Before the first press, throws UserSetupError if `text` has a character with +// no key. While typing, throws OperationFailedException, with a screenshot, if +// the cursor can't be read or doesn't reach a key, or if a key press doesn't +// add to the text field. The check counts the field's white pixels, so it +// catches a dropped press but not a wrong case. +// +// Presses are held for the "Fast Code Entry Timings" hold of Switch 1 with the +// controller's connection type, as keyboard_enter_code() does. The waits after +// them are fixed, since each move is read back from the video. +void type_on_system_keyboard( + VideoStream& stream, ProControllerContext& context, + const std::string& text, + KeyboardLayout layout = KeyboardLayout::QWERTY +); + + +} // namespace FastCodeEntry +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/Tests/LogicTests/LogicTests_Main.cpp b/SerialPrograms/Source/Tests/LogicTests/LogicTests_Main.cpp index cb6f7b0636..3016848a45 100644 --- a/SerialPrograms/Source/Tests/LogicTests/LogicTests_Main.cpp +++ b/SerialPrograms/Source/Tests/LogicTests/LogicTests_Main.cpp @@ -34,6 +34,11 @@ #include "CommonTools/ScreenEvents/ScreenEvents_Tests.h" #include "CommonTools/ScreenMachine/ScreenMachine_Tests.h" #include "Pokemon/AutoStory/Pokemon_AutoStory_Tests.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend_Tests.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryChainStart_Tests.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryGame_Tests.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStorySceneMachine_Tests.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStorySceneReplay_Tests.h" namespace PokemonAutomation{ namespace{ @@ -44,6 +49,11 @@ UnitTestDatabase make_logic_tests(){ Pokemon::add_tests_AutoStory(database); add_tests_ScreenEvents(database); add_tests_ScreenMachine(database); + NintendoSwitch::PokemonLZA::add_tests_AutoStoryBackend(database); + NintendoSwitch::PokemonLZA::add_tests_AutoStoryChainStart(database); + NintendoSwitch::PokemonLZA::add_tests_AutoStoryGame(database); + NintendoSwitch::PokemonLZA::add_tests_AutoStorySceneMachine(database); + NintendoSwitch::PokemonLZA::add_tests_AutoStorySceneReplay(database); return database; } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp index 3f350a27a0..c866d2419f 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.cpp @@ -205,8 +205,11 @@ void FlatWhiteDialogDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_bottom); items.add(m_color, DIALOG_ARROW_BOX); } +bool FlatWhiteDialogDetector::detect_box(const ImageViewRGB32& screen) const{ + return is_white(extract_box_reference(screen, m_bottom), /*min_rgb_sum=*/500.0, /*max_stddev_sum=*/20.0); +} bool FlatWhiteDialogDetector::detect(const ImageViewRGB32& screen){ - if (!is_white(extract_box_reference(screen, m_bottom), 500.0, 20.0)){ + if (!detect_box(screen)){ m_last_detected_box_scope.reset(); return false; } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h index b459a15596..56a37ef40f 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h @@ -31,6 +31,10 @@ class FlatWhiteDialogDetector : public StaticScreenDetector{ // This is not const so that detectors can save/cache state. virtual bool detect(const ImageViewRGB32& screen) override; + // Only the box's white bottom edge, without the advance arrow: the box + // shows while its text is still being typed. + bool detect_box(const ImageViewRGB32& screen) const; + private: friend class FlatWhiteDialogWatcher; diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.cpp new file mode 100644 index 0000000000..6fd41e8d87 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.cpp @@ -0,0 +1,286 @@ +/* Party Bar Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "Tests/ImageUnitTest.h" +#include "PokemonLZA_PartyBarDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +namespace{ + +// The slot's HP bar: 52 of 1920 wide from x = 86 + 76.4 * i, rows +// 1028-1031 of 1080; the box stays inside the bar. +constexpr double SLOT_X = 0.045; +constexpr double SLOT_PITCH = 0.0398; +constexpr double BAR_WIDTH = 0.026; +constexpr double BAR_Y = 0.9519; +constexpr double BAR_HEIGHT = 0.0037; +// The card between the icon and the bar: rows 1021-1025. +constexpr double CARD_Y = 0.9454; +constexpr double CARD_HEIGHT = 0.0046; + +// The trough is the card darkened: a black overlay at about half. +constexpr double TROUGH_BRIGHTNESS_RATIO = 0.65; +// A bar's box reads 0.79 and up (a nearly empty red bar over a bright +// teal scene); a slot with none 0.16 at most (the Poke Ball of the slot +// sent out). +constexpr double BAR_SHARE_MIN = 0.5; + +// An HP bar's green, yellow or red. +bool is_hp_color(Color color){ + const uint32_t r = color.red(); + const uint32_t g = color.green(); + const uint32_t b = color.blue(); + const bool green = g > 140 && g > r + 60 && g > b + 60; + const bool yellow = r > 170 && g > 150 && b < 100; + const bool red = r > 170 && g < 110 && b < 110; + return green || yellow || red; +} + +double brightness(Color color){ + return (color.red() + color.green() + color.blue()) / 3.0; +} + +// On 1920x1080 (console B, 2026-11-16, German 02.05 with a party of one): +// an empty slot's frame at x 78 + 76 * i for slot i (from 0), y 954, 66 px +// square, its edges 2 px wide. The last slot's at x 458-523, y 954-1019. +constexpr double LAST_SLOT_X = 458; +constexpr double SLOT_FRAME_Y = 954; +constexpr double SLOT_FRAME_SIZE = 66; +constexpr double FRAME_LINE = 2; +// The strips each edge is held against: 3 px wide, 2 px off the line on +// either side. Each edge is read along its middle, 8 px in from the +// corners, which are rounded. +constexpr double FRAME_STRIP = 3; +constexpr double FRAME_GAP = 2; +constexpr double FRAME_CORNER = 8; + +// A frame's line is the scene under a light veil, its inside the scene +// darkened. On the six frames with the last slot empty (a dark lobby to +// bright ground) every edge's line is lighter than inside by 21-60 and +// than outside by 10-61; on 686 cutscene frames of 01.00-01.03 (no party +// bar) by 2 at most inside. +constexpr double FRAME_INSIDE_MARGIN = 10; +constexpr double FRAME_OUTSIDE_MARGIN = 0; + +ImageFloatBox pixel_box(double x, double y, double width, double height){ + return {x / 1920, y / 1080, width / 1920, height / 1080}; +} + +double box_brightness(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return image_average(extract_box_reference(screen, box)).sum() / 3; +} + +// One edge of a slot's frame: the 2 px line, the strip inside it and the +// strip outside it. +struct FrameEdge{ + ImageFloatBox line; + ImageFloatBox inside; + ImageFloatBox outside; +}; + +// The last slot's four edges. +std::array last_slot_edges(){ + const double left = LAST_SLOT_X; + const double right = LAST_SLOT_X + SLOT_FRAME_SIZE - FRAME_LINE; + const double top = SLOT_FRAME_Y; + const double bottom = SLOT_FRAME_Y + SLOT_FRAME_SIZE - FRAME_LINE; + const double span = SLOT_FRAME_SIZE - 2 * FRAME_CORNER; + const double step = FRAME_LINE + FRAME_GAP; + return { + FrameEdge{ + .line = pixel_box(left + FRAME_CORNER, top, span, FRAME_LINE), + .inside = pixel_box(left + FRAME_CORNER, top + step, span, FRAME_STRIP), + .outside = pixel_box(left + FRAME_CORNER, top - FRAME_GAP - FRAME_STRIP, span, FRAME_STRIP), + }, + FrameEdge{ + .line = pixel_box(left + FRAME_CORNER, bottom, span, FRAME_LINE), + .inside = pixel_box(left + FRAME_CORNER, bottom - FRAME_GAP - FRAME_STRIP, span, FRAME_STRIP), + .outside = pixel_box(left + FRAME_CORNER, bottom + step, span, FRAME_STRIP), + }, + FrameEdge{ + .line = pixel_box(left, top + FRAME_CORNER, FRAME_LINE, span), + .inside = pixel_box(left + step, top + FRAME_CORNER, FRAME_STRIP, span), + .outside = pixel_box(left - FRAME_GAP - FRAME_STRIP, top + FRAME_CORNER, FRAME_STRIP, span), + }, + FrameEdge{ + .line = pixel_box(right, top + FRAME_CORNER, FRAME_LINE, span), + .inside = pixel_box(right - FRAME_GAP - FRAME_STRIP, top + FRAME_CORNER, FRAME_STRIP, span), + .outside = pixel_box(right + step, top + FRAME_CORNER, FRAME_STRIP, span), + }, + }; +} + +// Whether the last slot shows empty: every edge of its frame lighter than +// the strip inside it and no darker than the strip outside it. +bool last_slot_empty(const ImageViewRGB32& screen){ + static const std::array edges = last_slot_edges(); + for (const FrameEdge& edge : edges){ + const double line = box_brightness(screen, edge.line); + if (line - box_brightness(screen, edge.inside) < FRAME_INSIDE_MARGIN || line - box_brightness(screen, edge.outside) < FRAME_OUTSIDE_MARGIN){ + return false; + } + } + return true; +} + +} // namespace + + +PartyBarDetector::PartyBarDetector(Color color, VideoOverlay* overlay) + : m_selection(color, overlay) + , m_last_slot(pixel_box(LAST_SLOT_X, SLOT_FRAME_Y, SLOT_FRAME_SIZE, SLOT_FRAME_SIZE)) +{ + for (size_t i = 0; i < 6; i++){ + const double x = SLOT_X + SLOT_PITCH * i; + m_bar_boxes[i] = {x, BAR_Y, BAR_WIDTH, BAR_HEIGHT}; + m_card_boxes[i] = {x, CARD_Y, BAR_WIDTH, CARD_HEIGHT}; + } +} + +void PartyBarDetector::make_overlays(VideoOverlaySet& items) const{ + m_selection.make_overlays(items); + items.add(COLOR_GREEN, m_last_slot); + for (size_t i = 0; i < 6; i++){ + items.add(COLOR_GREEN, m_bar_boxes[i]); + } +} + +bool PartyBarDetector::slot_has_hp_bar(const ImageViewRGB32& screen, size_t index) const{ + const ImageViewRGB32 card = extract_box_reference(screen, m_card_boxes[index]); + const ImageViewRGB32 bar = extract_box_reference(screen, m_bar_boxes[index]); + const size_t total = bar.width() * bar.height(); + if (total == 0 || card.width() * card.height() == 0){ + return false; + } + const double trough_max = image_average(card).sum() / 3.0 * TROUGH_BRIGHTNESS_RATIO; + size_t count = 0; + for (size_t y = 0; y < bar.height(); y++){ + for (size_t x = 0; x < bar.width(); x++){ + const Color color(bar.pixel(x, y)); + if (is_hp_color(color) || brightness(color) < trough_max){ + count++; + } + } + } + return (double)count / (double)total >= BAR_SHARE_MIN; +} + +bool PartyBarDetector::detect(const ImageViewRGB32& screen){ + m_filled_slots = 0; + if (screen.width() == 0 || screen.height() == 0){ + return false; + } + // The marker is read first: it also names the slot sent out. + if (!m_selection.detect(screen) && !last_slot_empty(screen)){ + return false; + } + const uint8_t sent_out = m_selection.dpad_down_idx(); + for (size_t i = 0; i < 6; i++){ + if (i == sent_out || slot_has_hp_bar(screen, i)){ + m_filled_slots++; + } + } + return true; +} + +namespace{ + +// The filled slots of the party bar on `screen`: 0 when no bar is on it. +int party_bar_filled_slots(const ImageViewRGB32& screen){ + PartyBarDetector detector; + detector.detect(screen); + return detector.filled_slots(); +} + +struct PartyBarCase{ + std::string name; + struct Input{ + std::string image; // In CommandLineTests/PokemonLZA/. + }; + Input input; + struct Expectation{ + int filled_slots; + }; + Expectation expectation; +}; + +const PartyBarCase PARTY_BAR_CASES[] = { + { + .name = "five filled, slot 0 selected (D-pad up), the sixth empty", + .input = {.image = "PartyBarDetector/party5_slot0_selected.png"}, + .expectation = {.filled_slots = 5}, + }, + { + .name = "five filled, slot 0's partner sent out (Poke Ball, no bar), Mareep's bar red", + .input = {.image = "PartyBarDetector/party5_slot0_sent_out.png"}, + .expectation = {.filled_slots = 5}, + }, + { + .name = "five filled, slot 0's partner sent out, over bright ground", + .input = {.image = "PartyBarDetector/party5_b.png"}, + .expectation = {.filled_slots = 5}, + }, + { + // No D-pad marker with a party of one: the empty last slot shows the bar. + .name = "one filled in Hotel Z's lobby, five empty", + .input = {.image = "AutoStoryScreenEvents/de_0205_lobby_start_party1.png"}, + .expectation = {.filled_slots = 1}, + }, + { + .name = "one filled at the lobby elevator's A prompt", + .input = {.image = "AutoStoryScreenEvents/de_0205_elevator_prompt_steady.png"}, + .expectation = {.filled_slots = 1}, + }, +}; + +class Test_PartyBarDetector : public ImageUnitTest{ +public: + explicit Test_PartyBarDetector(const PartyBarCase& test_case) + : ImageUnitTest( + std::format("PokemonLZA::PartyBarDetector - {}", test_case.name), + {std::format("PokemonLZA/{}", test_case.input.image)} + ) + , m_case(test_case) + {} + + virtual UnitTestResult run_on_images( + Logger& logger, CancellableScope& scope, + const std::vector& images + ) const override{ + const int filled = party_bar_filled_slots(images[0]); + if (filled != m_case.expectation.filled_slots){ + return std::format("Expected {} filled slots, got {}.", m_case.expectation.filled_slots, filled); + } + return true; + } + +private: + const PartyBarCase& m_case; +}; + +} // namespace + + +void add_tests_PartyBarDetector(UnitTestDatabase& database){ + for (const PartyBarCase& item : PARTY_BAR_CASES){ + database.add(item); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.h new file mode 100644 index 0000000000..bb6410d664 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.h @@ -0,0 +1,75 @@ +/* Party Bar Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_PartyBarDetector_H +#define PokemonAutomation_PokemonLZA_PartyBarDetector_H + +#include +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" +#include "PokemonLZA_OverworldPartySelectionDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Counts the filled slots of the overworld party bar (bottom left, 6 +// slots). The bar is on screen when its D-pad marker shows +// (OverworldPartySelectionDetector, a party of 2 or more) or when its last +// slot (slot 6) shows empty (a party of 1 to 5, which the marker misses +// with a party of 1: Hotel Z's lobby on 02.05, console B 2026-11-16). +// Cutscenes and dialogs hide the bar. A filled slot has an HP bar under +// its icon: green, yellow or red by its HP, over a dark trough. The slot +// whose partner is sent out (the D-pad down marker's) shows a Poke Ball +// and no HP bar: it counts too. An empty slot shows only its frame: a +// thin light line over the scene darkened inside it. +// Boxes measured on 1920x1080 frames (console B, 2026-11-14): the bars +// on rows 1028-1031, slot i from x = 86 + 76.4 * i, 52 wide. +class PartyBarDetector : public StaticScreenDetector{ +public: + PartyBarDetector(Color color = COLOR_RED, VideoOverlay* overlay = nullptr); + + virtual void make_overlays(VideoOverlaySet& items) const override; + + // Whether the party bar is on screen; then filled_slots() is its count. + virtual bool detect(const ImageViewRGB32& screen) override; + + virtual void reset_state() override{ + m_selection.reset_state(); + m_filled_slots = 0; + } + + // The filled slots of the bar on the last detect()'s screen: 0 when + // no bar was on it. + int filled_slots() const{ return m_filled_slots; } + +private: + // Whether slot `index`'s HP bar shows: the bar's box is mostly HP + // color or darker than the card above it (the trough). + bool slot_has_hp_bar(const ImageViewRGB32& screen, size_t index) const; + + OverworldPartySelectionDetector m_selection; + // The last slot's frame, read when it is empty. + ImageFloatBox m_last_slot; + // Each slot's HP bar and, above it, the card between icon and bar. + std::array m_bar_boxes; + std::array m_card_boxes; + int m_filled_slots = 0; +}; + + +// Registers the party bar detector's tests on the PartyBarDetector frames. +void add_tests_PartyBarDetector(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Panels.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Panels.cpp index bd5e98adae..cfc1cd9bd3 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Panels.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Panels.cpp @@ -54,6 +54,7 @@ #include "Programs/TestPrograms/PokemonLZA_MoveBoxArrow.h" #include "Programs/TestPrograms/PokemonLZA_TestBoxCellInfo.h" #include "InferenceTraining/PokemonLZA_GenerateLocationNameOCR.h" +#include "Programs/AutoStory/PokemonLZA_AutoStory.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -68,6 +69,7 @@ PanelListFactory::PanelListFactory() ) {} +// code-health: allow function_lines -- the list reads top to bottom as the program menu does; splitting it by section hides that order std::vector PanelListFactory::make_panels() const{ std::vector ret; @@ -121,6 +123,7 @@ std::vector PanelListFactory::make_panels() const{ ret.emplace_back("---- Public Betas ----"); ret.emplace_back(make_SingleSwitchProgram()); + ret.emplace_back(make_SingleSwitchProgram()); if (STATIC_GLOBALS.DEVELOPER_MODE){ ret.emplace_back("---- Developer Tools ----"); diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp index ead2ea2d67..97991dd6bb 100644 --- a/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/PokemonLZA_Tests.cpp @@ -18,7 +18,22 @@ #include "Inference/Map/PokemonLZA_MapDetector.h" #include "Inference/Map/PokemonLZA_MapIconDetector.h" #include "Inference/PokemonLZA_OverworldPartySelectionDetector.h" +#include "Inference/PokemonLZA_PartyBarDetector.h" #include "Inference/PokemonLZA_SelectionArrowDetector.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.h" +#include "Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.h" +#include "Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.h" +#include "Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStoryBackend_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStoryChainStart_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStoryGame_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStorySceneMachine_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStorySceneReplay_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStoryScreenEvents_Tests.h" +#include "Programs/AutoStory/PokemonLZA_AutoStoryStarts_Tests.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -40,7 +55,22 @@ void add_tests(UnitTestDatabase& database){ add_tests_MapDetector(database); add_tests_MapIconDetector(database); add_tests_OverworldPartySelectionDetector(database); + add_tests_PartyBarDetector(database); add_tests_SelectionArrowDetector(database); + add_tests_AutoStoryDetectors(database); + add_tests_AutoStoryDialogChoiceBox(database); + add_tests_AutoStoryConfirmChoicesReader(database); + add_tests_AutoStoryDialogDetector(database); + add_tests_AutoStoryIntroScreens(database); + add_tests_AutoStoryLaunch(database); + add_tests_AutoStoryNewGame(database); + add_tests_AutoStoryBackend(database); + add_tests_AutoStoryGame(database); + add_tests_AutoStoryChainStart(database); + add_tests_AutoStorySceneMachine(database); + add_tests_AutoStorySceneReplay(database); + add_tests_AutoStoryScreenEvents(database); + add_tests_AutoStoryStarts(database); } diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.cpp new file mode 100644 index 0000000000..c5424a99c1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.cpp @@ -0,0 +1,57 @@ +/* PokemonLZA AutoStory - Battles + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "VideoGames/PokemonLZA/Programs/PokemonLZA_TrainerBattle.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h" +#include "PokemonLZA_AutoStoryBattles.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +void fight_story_battle( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai, std::chrono::seconds timeout +){ + TrainerBattleState battle_state(battle_ai); + StoryDialogDetector any_dialog; + const WallClock deadline = current_time() + timeout; + bool attacked = false; + while (true){ + context.wait_for_all_requests(); + const VideoSnapshot screen = console.video().snapshot(); + const bool dialog = any_dialog.detect(screen); + if (dialog && attacked){ + return; + } + if (current_time() > deadline){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "fight_story_battle(): no dialogue after the battle within the timeout.", + console + ); + } + if (dialog || !battle_state.attempt_one_attack(env, console, context)){ + if (!attacked){ + pbf_press_button(context, BUTTON_A, /*hold=*/160ms, /*release=*/400ms); + } + continue; + } + attacked = true; + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.h new file mode 100644 index 0000000000..dec7a7a101 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryBattles.h @@ -0,0 +1,40 @@ +/* PokemonLZA AutoStory - Battles + * + * From: https://github.com/PokemonAutomation/ + * + * The story's battles on the console, over the LZA battle code + * (TrainerBattleState): what PlzaConsoleBackend::battle() runs. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryBattles_H +#define PokemonAutomation_PokemonLZA_AutoStoryBattles_H + +#include +#include "CommonFramework/Tools/ProgramEnvironment.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "VideoGames/PokemonLZA/Options/PokemonLZA_BattleAIOption.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A story battle, from its HUD or the talk before it: attacks with +// TrainerBattleState (`battle_ai`'s moves) until a dialogue box shows (the +// speech after a win or a loss). Until the first attack lands, a dialogue +// box or a failed lock-on gets A instead: the trainer's lines and the +// tutorial cards. Story battles show no Run prompt, which upstream's +// battle loops start on. Throws OperationFailedException after `timeout`. +void fight_story_battle( + ProgramEnvironment& env, ConsoleHandle& console, ProControllerContext& context, + BattleAIOption& battle_ai, std::chrono::seconds timeout +); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.cpp new file mode 100644 index 0000000000..23ede89408 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.cpp @@ -0,0 +1,124 @@ +/* PokemonLZA AutoStory - Confirm Choices + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Tools/DebugDumper.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h" +#include "PokemonLZA_AutoStoryConfirmChoices.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The "Confirm Choices" highlight on the screen now, reading a few frames +// over `window` with no press. Nullopt if none of them is the screen. +// `dump`: save the frame read to DebugDumps under that name. +std::optional read_confirm_choices_now( + ConsoleHandle& console, ProControllerContext& context, Milliseconds window, const std::string& dump +){ + const WallClock deadline = current_time() + window; + while (true){ + context.wait_for_all_requests(); + const VideoSnapshot frame = console.video().snapshot(); + const std::optional cursor = frame ? read_confirm_choices_cursor(frame) : std::nullopt; + if (cursor || current_time() >= deadline){ + if (frame){ + dump_debug_image(console, "PokemonLZA/AutoStory/Intro", dump, frame); + } + return cursor; + } + context.wait_for(250ms); + } +} + +// Fails, having pressed nothing, unless "Confirm Choices" shows the name +// and the language of `choices`. +void check_profile(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices){ + context.wait_for_all_requests(); + const VideoSnapshot frame = console.video().snapshot(); + if (!frame){ + return; + } + const std::string name = read_confirm_choices_name(frame); + if (name != choices.name){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("AutoStory intro: Confirm Choices shows the name \"{}\", not \"{}\". Nothing was confirmed.", name, choices.name), + console + ); + } + const NewGameLanguageInfo& info = new_game_language_info(choices.language); + if (!OCR::ocr_language_available(info.ocr_language)){ + console.log(std::format("AutoStory intro: Confirm Choices: no OCR for {}; its language is not checked.", info.tag)); + return; + } + const std::string language = read_confirm_choices_language(frame, info.ocr_language); + if (language.find(info.confirm_text) == std::string::npos){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("AutoStory intro: Confirm Choices shows the language \"{}\", not {}. Nothing was confirmed.", language, info.tag), + console + ); + } + console.log(std::format("AutoStory intro: Confirm Choices shows \"{}\" in {}.", name, info.tag)); +} + +} // namespace + + +size_t require_confirm_choices(ConsoleHandle& console, ProControllerContext& context){ + const std::optional cursor = read_confirm_choices_now(console, context, /*window=*/2000ms, "confirm-choices"); + if (!cursor){ + throw UserSetupError( + console, + "AutoStory: 01.00b starts on the new game's \"Confirm Choices\" screen, after the language, look " + "and name (01.00a, or made by hand). It is not on screen; nothing was pressed." + ); + } + console.log(std::format("AutoStory intro: Confirm Choices on screen, row {} highlighted.", *cursor)); + return *cursor; +} + +void confirm_new_game_choices(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices){ + std::optional cursor = require_confirm_choices(console, context); + check_profile(console, context, choices); + for (size_t press = 0; press < 6 && cursor; press++){ + if (*cursor == CONFIRM_CHOICES_READY_ROW){ + console.log("AutoStory intro: Confirm Choices: A on \"Ready!\"."); + pbf_press_button(context, BUTTON_A, /*hold=*/160ms, /*release=*/2000ms); + context.wait_for_all_requests(); + return; + } + console.log("AutoStory intro: Confirm Choices: DOWN."); + pbf_press_dpad(context, DPAD_DOWN, /*hold=*/160ms, /*release=*/500ms); + cursor = read_confirm_choices_now(console, context, /*window=*/1000ms, "confirm-choices-moved"); + if (cursor){ + console.log(std::format("AutoStory intro: Confirm Choices: the highlight is on row {}.", *cursor)); + } + } + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "AutoStory intro: the Confirm Choices highlight was not read on \"Ready!\".", + console + ); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.h new file mode 100644 index 0000000000..534cabfb0f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryConfirmChoices.h @@ -0,0 +1,44 @@ +/* PokemonLZA AutoStory - Confirm Choices + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's "Confirm Choices" screen, after the language, look and + * name (PokemonLZA_AutoStoryNewGame.h). Its reader is in + * Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryConfirmChoices_H +#define PokemonAutomation_PokemonLZA_AutoStoryConfirmChoices_H + +#include +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "PokemonLZA_AutoStoryNewGameChoices.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The highlighted row of "Confirm Choices", read from frames over up to +// 2 s with no press. Throws UserSetupError if the screen does not show. +size_t require_confirm_choices(ConsoleHandle& console, ProControllerContext& context); + +// 01.00b's first step: "Confirm Choices" is on screen. Checks +// that screen first (require_confirm_choices()): on any other screen +// throws UserSetupError and presses nothing. Then reads the name and the +// language it shows (OCR) and, if either is not `choices`', throws +// OperationFailedException having pressed nothing (the language is not +// checked when its OCR is not installed). Then moves the highlight to +// "Ready!" one DOWN at a time, reading it after each, and presses A there: +// the save's only change is this press, on a screen it has read. Throws +// OperationFailedException if the highlight is lost on the way. +void confirm_new_game_choices(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.cpp new file mode 100644 index 0000000000..d29703a693 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.cpp @@ -0,0 +1,84 @@ +/* PokemonLZA AutoStory - Launch + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/NintendoSwitch_Settings.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h" +#include "PokemonLZA_AutoStoryLaunch.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr Milliseconds HOME_TILE_RELEASE = 500ms; + +// The highlighted HOME tile is PLZA's; false without a frame. +bool highlighted_tile_is_plza(ConsoleHandle& console, ProControllerContext& context){ + context.wait_for_all_requests(); + const VideoSnapshot frame = console.video().snapshot(); + return frame && is_home_tile_plza(frame); +} + +} // namespace + + +std::optional find_plza_tile( + const std::function& highlighted_is_plza, + const std::function& move_right +){ + for (size_t tile = 0; tile < HOME_TILES_SEARCHED; tile++){ + if (tile > 0){ + move_right(); + } + if (highlighted_is_plza()){ + return tile; + } + } + return std::nullopt; +} + +void launch_plza_from_home(ConsoleHandle& console, ProControllerContext& context){ + // HOME from the game or from HOME: the cursor on the first tile, the + // running game's when one runs. + go_home(console, context); + const bool first_is_plza = highlighted_tile_is_plza(console, context); + console.log(std::format( + "AutoStory launch: HOME's first tile (the running game, if any) is {}; closing it.", + first_is_plza ? "PLZA" : "another game" + )); + close_game_from_home(console, context); + + pbf_press_button(context, BUTTON_HOME, /*hold=*/160ms, ConsoleSettings::instance().SETTINGS_TO_HOME_DELAY0); + const std::optional tile = find_plza_tile( + [&]{ return highlighted_tile_is_plza(console, context); }, + [&]{ pbf_press_dpad(context, DPAD_RIGHT, /*hold=*/80ms, HOME_TILE_RELEASE); } + ); + if (!tile){ + throw UserSetupError( + console, + std::format( + "AutoStory: Pokemon Legends: Z-A is not on HOME's first {} tiles. Play it once so its tile moves to the front.", + HOME_TILES_SEARCHED + ) + ); + } + console.log(std::format("AutoStory launch: launching PLZA, HOME's tile {}.", *tile)); + // Slot 0: the cursor stays on PLZA's tile. + start_game_from_home(console, context, /*tolerate_update_menu=*/true, /*game_slot=*/0, /*user_slot=*/0); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.h new file mode 100644 index 0000000000..473130b180 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch.h @@ -0,0 +1,50 @@ +/* PokemonLZA AutoStory - Launch + * + * From: https://github.com/PokemonAutomation/ + * + * The one way AutoStory starts or restarts PLZA from the Switch HOME: for + * a new game and for the latest save. HOME's tiles are + * in last-played order, so the first tile is not always PLZA's. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryLaunch_H +#define PokemonAutomation_PokemonLZA_AutoStoryLaunch_H + +#include +#include +#include +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// How many of HOME's tiles, from the first, are searched for PLZA's. +constexpr size_t HOME_TILES_SEARCHED = 8; + +// HOME's tile search from the highlighted tile rightward: reads each tile +// with `highlighted_is_plza`, moving one tile right with `move_right` +// between reads, and returns how many moves put PLZA's tile under the +// cursor; nullopt if none of HOME_TILES_SEARCHED tiles is PLZA's. +std::optional find_plza_tile( + const std::function& highlighted_is_plza, + const std::function& move_right +); + +// From the game or HOME: goes HOME, closes the running game whichever it +// is (the first tile, the last played), selects PLZA's tile by its title +// and launches it through the user select, until the game's black screen. +// Logs whether the closed game was PLZA. Throws UserSetupError, having +// launched nothing, if PLZA's tile is not among HOME's first +// HOME_TILES_SEARCHED. +void launch_plza_from_home(ConsoleHandle& console, ProControllerContext& context); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.cpp new file mode 100644 index 0000000000..b66bd42c23 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.cpp @@ -0,0 +1,132 @@ +/* PokemonLZA AutoStory - Launch Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Tests/ImageUnitTest.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h" +#include "PokemonLZA_AutoStoryLaunch.h" +#include "PokemonLZA_AutoStoryLaunch_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +std::string describe(std::optional value){ + return value ? std::format("{}", *value) : std::string("none"); +} + +// HOME as the console showed it, one frame per tile from the first: the +// search reads each frame in turn, a right move showing the next one. +struct TileSearchCase{ + std::string name; + struct Input{ + std::vector files; // Under the test images' PokemonLZA/. + // The last frame repeats past the end (HOME's row ends there). + }; + Input input; + struct Expectation{ + std::optional tile = std::nullopt; + }; + Expectation expectation; +}; + +const TileSearchCase TILE_SEARCH_CASES[] = { + { + // The 01.01 load that launched FireRed (console B, 2026-11-15): + // FireRed played last, Z-A second. + .name = "FireRed first, Z-A second", + .input = {.files = { + "AutoStoryNewGame/2026-11-15/home_firered_first.png", + "AutoStoryNewGame/2026-11-15/home_plza.png", + }}, + .expectation = {.tile = 1}, + }, + { + .name = "Z-A first", + .input = {.files = {"AutoStoryNewGame/2026-11-15/home_plza.png"}}, + .expectation = {.tile = 0}, + }, + { + .name = "no Z-A tile", + .input = {.files = {"AutoStoryNewGame/2026-11-15/home_firered.png"}}, + .expectation = {.tile = std::nullopt}, + }, + { + // The console in its dark theme (Basic Black), 2026-11-18: Pokemon + // HOME played last, Z-A second. + .name = "dark theme, Pokemon HOME first, Z-A second", + .input = {.files = { + "AutoStoryNewGame/2026-11-18/home_dark_tile0.png", + "AutoStoryNewGame/2026-11-18/home_dark_tile1.png", + }}, + .expectation = {.tile = 1}, + }, +}; + +std::vector home_tile_images(const TileSearchCase& test){ + std::vector paths; + for (const std::string& file : test.input.files){ + paths.push_back(std::format("PokemonLZA/{}", file)); + } + return paths; +} + +class Test_HomeTileSearch : public ImageUnitTest{ +public: + explicit Test_HomeTileSearch(const TileSearchCase& test) + : ImageUnitTest(std::format("PokemonLZA::AutoStory - launch: HOME tile search, {}", test.name), home_tile_images(test)) + , m_test(test) + {} + virtual UnitTestResult run_on_images(Logger&, CancellableScope&, const std::vector& images) const override{ + size_t shown = 0; + size_t moves = 0; + const std::optional tile = find_plza_tile( + [&]{ + return is_home_tile_plza(images[shown]); + }, + [&]{ + moves++; + if (shown + 1 < images.size()){ + shown++; + } + } + ); + if (tile != m_test.expectation.tile){ + return std::format("Expected PLZA on tile {}, found {}.", describe(m_test.expectation.tile), describe(tile)); + } + const size_t expected_moves = tile ? *tile : HOME_TILES_SEARCHED - 1; + if (moves != expected_moves){ + return std::format("Expected {} right moves, made {}.", expected_moves, moves); + } + return true; + } + +private: + const TileSearchCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryLaunch(UnitTestDatabase& database){ + for (const TileSearchCase& test : TILE_SEARCH_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.h new file mode 100644 index 0000000000..717b123506 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryLaunch_Tests.h @@ -0,0 +1,25 @@ +/* PokemonLZA AutoStory - Launch Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryLaunch_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryLaunch_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the launch's tests: HOME's tile search over console frames. +void add_tests_AutoStoryLaunch(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.cpp new file mode 100644 index 0000000000..7f9d018187 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.cpp @@ -0,0 +1,136 @@ +/* PokemonLZA AutoStory - Moves + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "PokemonLZA_AutoStoryMoves.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +namespace{ + +// Whether the screen on a fresh look has not changed since the look +// before (SCREEN_STILL): the input in between did nothing, a dropped turn +// or a dropped hold. A walk into a wall that already stops the player +// reads still too, and plays its leg twice. +bool screen_still(PlzaBackend& backend){ + return backend.observe().events.has(ScreenEvent::SCREEN_STILL); +} + +// A recorded turn: story_turn(), then L to recenter the camera, since the +// walk goes where it looks. +void turn_and_recenter(PlzaBackend& backend, double degrees){ + story_turn(backend, degrees); + backend.press(BUTTON_L, /*hold=*/100ms, /*release=*/150ms); +} + +// `input`, played again once when the screen stays still after it. +template +void play_checked(PlzaBackend& backend, std::string_view what, Input&& input){ + input(); + if (screen_still(backend)){ + backend.log(std::format("[PLZA] walk: the screen stayed still after {}; once more", what)); + input(); + } +} + +// StoryWalkStyle::HELD: L before leg `index` when it turns after an +// earlier leg. +void recenter_before_held_leg(PlzaBackend& backend, size_t index, const StoryWalkLeg& leg){ + if (index > 0 && leg.turn != 0){ + backend.press(BUTTON_L, /*hold=*/100ms, /*release=*/150ms); + } +} + +// StoryWalkStyle::HELD: each leg's stick held at its angle for its time +// (a turn-only leg a 150 ms tap); L before each later leg that turns. +void run_held_legs(PlzaBackend& backend, const std::vector& legs){ + for (size_t index = 0; index < legs.size(); index++){ + const StoryWalkLeg& leg = legs[index]; + recenter_before_held_leg(backend, index, leg); + if (leg.walk > 0ms){ + backend.stick(leg.turn, leg.walk, /*release=*/0ms, Pokemon::StoryPace::WALK); + }else if (leg.turn != 0){ + backend.stick(leg.turn, /*duration=*/150ms, /*release=*/100ms, Pokemon::StoryPace::WALK); + } + } +} + +// "leg 2/5 turn 90 walk 1200 ms": leg `index` (from 0) of `count`, for the +// log. A leg with no walk leaves it out. +std::string story_walk_leg_label(size_t index, size_t count, const StoryWalkLeg& leg){ + std::string label = std::format("leg {}/{} turn {:g}", index + 1, count, leg.turn); + if (leg.walk > Milliseconds(0)){ + label += std::format(" walk {} ms", leg.walk.count()); + } + return label; +} + +} // namespace + + +Pokemon::Outcome run_story_move(PlzaBackend& backend, const StoryMove& move){ + if (move.style == StoryWalkStyle::HELD){ + run_held_legs(backend, move.legs); + return Pokemon::outcome_ok(); + } + // The look the first input is checked against. + backend.observe(); + for (size_t index = 0; index < move.legs.size(); index++){ + const StoryWalkLeg& leg = move.legs[index]; + const std::string label = story_walk_leg_label(index, move.legs.size(), leg); + if (leg.turn != 0){ + play_checked(backend, std::format("the turn of {}", label), [&]{ + turn_and_recenter(backend, leg.turn); + }); + } + if (leg.walk > Milliseconds(0)){ + play_checked(backend, std::format("the walk of {}", label), [&]{ + backend.stick(/*degrees=*/0, leg.walk, /*release=*/0ms, Pokemon::StoryPace::WALK); + }); + } + } + return Pokemon::outcome_ok(); +} + +Pokemon::Outcome run_story_move_but_last_walk(PlzaBackend& backend, const StoryMove& move){ + if (move.legs.empty()){ + return Pokemon::outcome_ok(); + } + StoryMove head = move; + if (move.style == StoryWalkStyle::HELD){ + if (move.legs.back().walk == 0ms){ + run_held_legs(backend, move.legs); + return Pokemon::outcome_ok(); + } + head.legs.pop_back(); + run_held_legs(backend, head.legs); + recenter_before_held_leg(backend, head.legs.size(), move.legs.back()); + return Pokemon::outcome_ok(); + } + head.legs.back().walk = 0ms; + return run_story_move(backend, head); +} + +void hold_last_leg(PlzaBackend& backend, const StoryMove& move, Milliseconds slice){ + if (move.style == StoryWalkStyle::HELD && !move.legs.empty()){ + backend.stick(move.legs.back().turn, slice, /*release=*/0ms, Pokemon::StoryPace::WALK); + }else{ + backend.stick(/*degrees=*/0, slice, /*release=*/0ms, Pokemon::StoryPace::WALK); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.h new file mode 100644 index 0000000000..ab7f79abcf --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryMoves.h @@ -0,0 +1,82 @@ +/* PokemonLZA AutoStory - Moves + * + * From: https://github.com/PokemonAutomation/ + * + * The story's walks in the overworld: recorded turns and timed walks. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMoves_H +#define PokemonAutomation_PokemonLZA_AutoStoryMoves_H + +#include +#include +#include "Common/Cpp/Time.h" +#include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// One leg of a walk: a turn, then a walk straight on. +struct StoryWalkLeg{ + // Degrees clockwise from where the player faces (negative: left), as + // the story was recorded: L recenters the camera behind the player, + // the stick is tapped that way for 150 ms, and L again. The tap also + // takes a step. 0: no turn. + double turn = 0; + + // How long to hold the stick forward after the turn, walking (not + // dashing). 0: the turn only. + Milliseconds walk = Milliseconds(0); +}; + +// How a walk's legs are played. +enum class StoryWalkStyle{ + // Each turn is L, a 150 ms tap that way, L, then the stick held + // forward (the 2026-11-08 recording). + TURN_THEN_WALK, + // The stick is held at the leg's angle for its whole time, from the + // camera as it is; L recenters the camera only before a later leg that + // turns (02.01, console B 2026-11-10). L near an NPC turns to it, so a + // held walk starts without one. + HELD, +}; + +// A walk in the overworld. Lumiose has no tiles: a move is legs of turns +// and timed walks, as recorded on the console. +struct StoryMove{ + std::vector legs = {}; + StoryWalkStyle style = StoryWalkStyle::TURN_THEN_WALK; + // A wait before the walk looks for free control, for the NPCs to move + // off (02.01). The walk's caller waits it; run_story_move() does not. + Milliseconds wait_first = Milliseconds(0); +}; + +// From the overworld, walks as `move` says. TURN_THEN_WALK: each leg's +// turn (and L to recenter the camera, since the walk goes where it looks) +// and walk, with a look after each turn and each walk; one that shows the +// screen still (the game dropped the input) plays that turn or walk once +// more, then goes on. HELD: each leg's stick held, unchecked. A leg cut short or blocked part way goes unnoticed, and a step +// that needs a prompt after the walk fails there instead. +Pokemon::Outcome run_story_move(PlzaBackend& backend, const StoryMove& move); + +// `move` as run_story_move() plays it, but for its last leg's walk, which +// the caller holds a slice at a time (hold_last_leg()) until what it walks +// to shows: TURN_THEN_WALK plays the last leg's turn; HELD presses L +// before the last leg when it turns after an earlier one. A HELD last leg +// with no walk is played whole, its turn a tap. +Pokemon::Outcome run_story_move_but_last_walk(PlzaBackend& backend, const StoryMove& move); + +// `slice` of `move`'s last leg's walk: the stick forward after the leg's +// turn (TURN_THEN_WALK), or at the leg's angle (HELD). +void hold_last_leg(PlzaBackend& backend, const StoryMove& move, Milliseconds slice); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.cpp new file mode 100644 index 0000000000..78426afd97 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.cpp @@ -0,0 +1,404 @@ +/* PokemonLZA AutoStory - New game + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Tools/DebugDumper.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_Superscalar.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_KeyboardEntryMappings.h" +#include "NintendoSwitch/Programs/FastCodeEntry/NintendoSwitch_SystemKeyboardTyping.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryTable.h" +#include "PokemonLZA_AutoStoryLaunch.h" +#include "PokemonLZA_AutoStoryNewGame.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// Whether every character of `name` has a key on the name keyboard's +// first page, which the Switch keyboard typer types (letters by their +// capital). +bool on_keyboard(std::string_view name){ + const std::map& keys = KEYBOARD_POSITIONS(KeyboardLayout::QWERTY); + for (const char ch : name){ + if (!keys.contains((char)std::toupper((unsigned char)ch))){ + return false; + } + } + return true; +} + +// Why the program cannot type `choices`' name on its language's keyboard; +// nullopt when it can. +std::optional name_typing_problem(const NewGameChoices& choices){ + const NewGameLanguageInfo& info = new_game_language_info(choices.language); + if (!info.keyboard_supported){ + return std::format( + "AutoStory: the program cannot type a name on the {} keyboard (Hangul).", + info.tag + ); + } + if (!on_keyboard(choices.name)){ + return std::format( + "AutoStory: the player name \"{}\" has a character that is not on the name keyboard's first page " + "(letters, digits and - / : ' , . ? !).", + choices.name + ); + } + return std::nullopt; +} + + +using FrameCheck = std::function; + +constexpr std::string_view DUMP_FOLDER = "PokemonLZA/AutoStory/Intro"; + +// From the user select to the language list: about 15 s of logos, with no +// press needed (the console, 2026-11-15, all ten languages). +constexpr Milliseconds LAUNCH_LIMIT = 60000ms; + +// The language list's D-pad taps: each registered at this pace (the +// console, 2026-11-15). +constexpr Milliseconds LANGUAGE_TAP_RELEASE = 600ms; + +// The look picker's stick tilts: one portrait each. +constexpr Milliseconds LOOK_TILT_HOLD = 150ms; +constexpr Milliseconds LOOK_TILT_RELEASE = 500ms; + +// The + presses after the name. One submits the European languages' and +// Japanese's keyboard; the Chinese ones first take one to accept the typed +// text (the button reads "Accept"), then one to submit. Each is checked. +constexpr size_t SUBMIT_PRESSES = 3; +constexpr Milliseconds SUBMIT_WAIT = 5000ms; + +VideoSnapshot snapshot(ConsoleHandle& console, ProControllerContext& context){ + context.wait_for_all_requests(); + return console.video().snapshot(); +} + +// Until `check` holds on a frame, up to `timeout`, reading every 250 ms. +bool wait_for_frame(ConsoleHandle& console, ProControllerContext& context, const FrameCheck& check, Milliseconds timeout){ + const WallClock deadline = current_time() + timeout; + while (true){ + const VideoSnapshot frame = snapshot(console, context); + if (frame && check(frame)){ + return true; + } + if (current_time() >= deadline){ + return false; + } + context.wait_for(250ms); + } +} + +[[noreturn]] void fail(ConsoleHandle& console, const std::string& message){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, std::format("AutoStory intro: {}", message), console); +} + +void press(ProControllerContext& context, IntroKey key, Milliseconds release){ + switch (key){ + case IntroKey::UP: + pbf_press_dpad(context, DPAD_UP, /*hold=*/80ms, release); + break; + case IntroKey::DOWN: + pbf_press_dpad(context, DPAD_DOWN, /*hold=*/80ms, release); + break; + case IntroKey::LEFT: + pbf_press_dpad(context, DPAD_LEFT, /*hold=*/80ms, release); + break; + case IntroKey::RIGHT: + pbf_press_dpad(context, DPAD_RIGHT, /*hold=*/80ms, release); + break; + case IntroKey::A: + pbf_press_button(context, BUTTON_A, /*hold=*/80ms, release); + break; + } +} + +// The look picker's presses are left stick tilts (it says "Use the Left +// Stick"; the D-pad does not move it). +void tilt(ProControllerContext& context, IntroKey key){ + switch (key){ + case IntroKey::UP: + pbf_move_left_joystick(context, /*position=*/{0, +1}, LOOK_TILT_HOLD, LOOK_TILT_RELEASE); + break; + case IntroKey::DOWN: + pbf_move_left_joystick(context, /*position=*/{0, -1}, LOOK_TILT_HOLD, LOOK_TILT_RELEASE); + break; + case IntroKey::LEFT: + pbf_move_left_joystick(context, /*position=*/{-1, 0}, LOOK_TILT_HOLD, LOOK_TILT_RELEASE); + break; + case IntroKey::RIGHT: + pbf_move_left_joystick(context, /*position=*/{+1, 0}, LOOK_TILT_HOLD, LOOK_TILT_RELEASE); + break; + default: + break; + } +} + +// After the launch, until the language list, pressing nothing (see +// LAUNCH_LIMIT). +void wait_for_language_list(ConsoleHandle& console, ProControllerContext& context){ + const WallClock start = current_time(); + while (current_time() - start < LAUNCH_LIMIT){ + const VideoSnapshot frame = snapshot(console, context); + if (frame && read_language_list_cursor(frame)){ + console.log("AutoStory intro: the language list shows."); + dump_debug_image(console, std::string(DUMP_FOLDER), "language-list", frame); + return; + } + context.wait_for(250ms); + } + fail(console, std::format("the language list did not show within {} s of the launch.", LAUNCH_LIMIT.count() / 1000)); +} + +// On the language list (it opens on English): one tap at a time to +// `language`, reading the highlight after each, then A, which opens the +// look picker with no confirmation. +void pick_language(ConsoleHandle& console, ProControllerContext& context, NewGameLanguage language){ + const NewGameLanguageInfo& info = new_game_language_info(language); + for (size_t attempt = 0; attempt < 20; attempt++){ + const VideoSnapshot frame = snapshot(console, context); + const std::optional cursor = frame ? read_language_list_cursor(frame) : std::nullopt; + if (!cursor){ + context.wait_for(250ms); + continue; + } + if (*cursor == info.list_entry){ + console.log(std::format("AutoStory intro: language {} (entry {}): A.", info.tag, info.list_entry)); + press(context, IntroKey::A, 500ms); + return; + } + const std::optional> moves = language_list_moves(*cursor, info.list_entry); + if (!moves || moves->empty()){ + break; + } + press(context, moves->front(), LANGUAGE_TAP_RELEASE); + } + fail(console, std::format("the language list's highlight did not reach {}.", info.tag)); +} + +// The look picker's highlight once settled: two reads in a row agree, +// 150 ms apart, within 2 s. +std::optional read_settled_look(ConsoleHandle& console, ProControllerContext& context){ + std::optional previous; + for (size_t read = 0; read < 14; read++){ + const VideoSnapshot frame = snapshot(console, context); + const std::optional cursor = frame ? read_look_picker_cursor(frame) : std::nullopt; + if (cursor && cursor == previous){ + return cursor; + } + previous = cursor; + context.wait_for(150ms); + } + return std::nullopt; +} + +// On the look picker: it opens with no portrait highlighted, and the first +// tilt highlights one that differs from launch to launch (the console, +// 2026-11-15: looks 1, 2, 5, 6, 7 and 8 after a tilt left). So it tilts +// left until a highlight reads, then toward `look`, reading the highlight +// after each tilt, and presses A only once it reads `look` (A opens the +// name keyboard). +void pick_look(ConsoleHandle& console, ProControllerContext& context, size_t look){ + if (!wait_for_frame(console, context, is_look_picker, 20000ms)){ + fail(console, "the look picker did not show within 20 s of the language's A."); + } + const VideoSnapshot picker = snapshot(console, context); + if (picker){ + dump_debug_image(console, std::string(DUMP_FOLDER), "look-picker", picker); + } + size_t unread = 0; + for (size_t attempt = 0; attempt < 24 && unread <= 6; attempt++){ + const std::optional cursor = read_settled_look(console, context); + if (!cursor){ + unread++; + tilt(context, IntroKey::LEFT); + continue; + } + if (*cursor == look){ + console.log(std::format("AutoStory intro: look {}: A.", look)); + press(context, IntroKey::A, 500ms); + return; + } + const std::optional> moves = look_picker_moves(*cursor, look); + if (!moves || moves->empty()){ + break; + } + tilt(context, moves->front()); + } + fail(console, std::format("the look picker's highlight was not read on look {}.", look)); +} + +// Until the name keyboard has shown and finished sliding in (presses +// during that are dropped). +void wait_for_keyboard(ConsoleHandle& console, ProControllerContext& context){ + if (!wait_for_frame(console, context, is_name_keyboard, 20000ms)){ + fail(console, "the name keyboard did not show within 20 s of the look's A."); + } + context.wait_for(1000ms); +} + +// The keyboard's "0/N" counter (12 in the European languages, 6 in +// Japanese, Korean and Chinese): throws UserSetupError, nothing typed, if +// the name is longer. +void check_name_limit(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices){ + const VideoSnapshot frame = snapshot(console, context); + if (frame){ + dump_debug_image(console, std::string(DUMP_FOLDER), "keyboard", frame); + } + const std::optional limit = frame ? read_name_keyboard_limit(frame) : std::nullopt; + if (!limit){ + console.log("AutoStory intro: keyboard: the 0/N counter did not read; the language's limit holds."); + }else if (*limit < choices.name.size()){ + throw UserSetupError(console, std::format( + "AutoStory: the name keyboard takes {} characters; the player name \"{}\" has {}. Nothing was typed.", + *limit, choices.name, choices.name.size() + )); + }else{ + console.log(std::format("AutoStory intro: keyboard: takes {} characters.", *limit)); + } +} + +// Simplified Chinese's keyboard opens in Pinyin, where letters compose +// Chinese; the bottom row's English key, under column 3, switches it to +// letters (Traditional Chinese's opens on letters). Moves the cursor +// there one D-pad tap at a time, reading it after each, then A. +void switch_to_letters(ConsoleHandle& console, ProControllerContext& context){ + constexpr SystemKeyboardKey LETTERS_KEY{.row = 4, .col = 3}; + for (size_t attempt = 0; attempt < 16; attempt++){ + const VideoSnapshot frame = snapshot(console, context); + const std::optional cursor = frame ? read_system_keyboard_cursor(frame) : std::nullopt; + if (!cursor){ + context.wait_for(300ms); + continue; + } + if (*cursor == LETTERS_KEY){ + console.log("AutoStory intro: keyboard: English (letters) key: A."); + press(context, IntroKey::A, 400ms); + return; + } + if (cursor->col == 11 || cursor->col > LETTERS_KEY.col){ + press(context, IntroKey::LEFT, 300ms); + }else if (cursor->row < LETTERS_KEY.row){ + press(context, IntroKey::DOWN, 300ms); + }else{ + press(context, IntroKey::RIGHT, 300ms); + } + } + fail(console, "the keyboard's cursor did not reach the English (letters) key."); +} + +// After the name: + until Confirm Choices shows, checking the screen after +// each press (see SUBMIT_PRESSES). +void submit_name(ConsoleHandle& console, ProControllerContext& context){ + const FrameCheck confirm_choices = [](const ImageViewRGB32& frame){ + return read_confirm_choices_cursor(frame).has_value(); + }; + for (size_t pressed = 1; pressed <= SUBMIT_PRESSES; pressed++){ + console.log(std::format("AutoStory intro: keyboard: + ({} of at most {}).", pressed, SUBMIT_PRESSES)); + pbf_press_button(context, BUTTON_PLUS, /*hold=*/80ms, /*release=*/500ms); + if (wait_for_frame(console, context, confirm_choices, SUBMIT_WAIT)){ + console.log("AutoStory intro: Confirm Choices shows."); + return; + } + const VideoSnapshot frame = snapshot(console, context); + if (!frame || !is_name_keyboard(frame)){ + break; + } + console.log("AutoStory intro: keyboard: still open after +."); + } + if (!wait_for_frame(console, context, confirm_choices, 15000ms)){ + fail(console, "Confirm Choices did not show after the name's +."); + } +} + +// On the keyboard: checks its counter, types the name with the Switch +// keyboard typer (which reads the cursor before each key and checks each +// press reached the field), then submits it. +void type_name(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices){ + wait_for_keyboard(console, context); + check_name_limit(console, context, choices); + if (choices.language == NewGameLanguage::ChineseSimplified){ + switch_to_letters(console, context); + } + console.log(std::format("AutoStory intro: typing \"{}\".", choices.name)); + FastCodeEntry::type_on_system_keyboard(console, context, choices.name); + submit_name(console, context); +} + +} // namespace + + +void require_new_game_home(ConsoleHandle& console, ProControllerContext& context){ + HomeMenuDetector home(console, COLOR_RED); + const FrameCheck on_home = [&home](const ImageViewRGB32& frame){ + return home.detect_only(frame); + }; + if (!wait_for_frame(console, context, on_home, 2000ms)){ + // An idle Switch dims HOME (screen saver), and HomeMenuDetector does + // not match the dimmed screen. B wakes it and acts on nothing on HOME. + console.log("AutoStory intro: HOME not detected; pressing B once to wake a dimmed screen."); + pbf_press_button(context, BUTTON_B, /*hold=*/80ms, /*release=*/1000ms); + } + if (!wait_for_frame(console, context, on_home, 2000ms)){ + throw UserSetupError( + console, + std::format( + "AutoStory: 01.00a starts a new game (no save) from the Switch HOME (PLZA's tile among the first {}). " + "It is not on screen; pressed B once to wake the screen, nothing else was pressed. {}", + HOME_TILES_SEARCHED, NEW_GAME_BY_HAND_HINT + ) + ); + } + console.log("AutoStory intro: starting on the Switch HOME."); +} + +std::optional new_game_run_problem(std::span rows, const NewGameChoices& choices){ + for (const Pokemon::StoryCheckpoint& row : rows){ + if (!story_row_makes_new_game_choices(row)){ + continue; + } + if (const std::optional problem = name_typing_problem(choices)){ + return std::format("{} {} types the name. {}", *problem, row.id, NEW_GAME_BY_HAND_HINT); + } + } + return std::nullopt; +} + +void play_new_game_intro(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices){ + if (const std::optional problem = new_game_problem(choices)){ + throw UserSetupError(console, *problem); + } + if (const std::optional problem = name_typing_problem(choices)){ + throw UserSetupError(console, *problem); + } + require_new_game_home(console, context); + launch_plza_from_home(console, context); + wait_for_language_list(console, context); + pick_language(console, context, choices.language); + pick_look(console, context, choices.look); + type_name(console, context, choices); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.h new file mode 100644 index 0000000000..b8053f0625 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame.h @@ -0,0 +1,61 @@ +/* PokemonLZA AutoStory - New game + * + * From: https://github.com/PokemonAutomation/ + * + * 01.00a's step, up to "Confirm Choices": launch the game from HOME, pick + * the language, the look and type the name (PokemonLZA_AutoStoryNewGameChoices.h). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryNewGame_H +#define PokemonAutomation_PokemonLZA_AutoStoryNewGame_H + +#include +#include +#include +#include +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h" +#include "Pokemon/AutoStory/Types/Pokemon_StoryCheckpoint.h" +#include "PokemonLZA_AutoStoryNewGameChoices.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The end of every refusal of 01.00a and of its failures: a user whose +// choices the program cannot make makes them by hand and starts at 01.00b. +constexpr std::string_view NEW_GAME_BY_HAND_HINT = + "Make the new game's choices by hand up to \"Confirm Choices\", then start the run at 01.00b."; + +// Checks that the Switch HOME is on screen, read over up to 2 s with no +// press. Throws UserSetupError, having pressed nothing, if it is not. +void require_new_game_home(ConsoleHandle& console, ProControllerContext& context); + +// Why a run of `rows` cannot play `choices`; nullopt when it can. A run +// that plays the choices (story_row_makes_new_game_choices(): 01.00a) +// types the name, so the language's keyboard and the name's characters +// must be ones the program types; a run from 01.00b finds the name typed. +// A refusal ends with NEW_GAME_BY_HAND_HINT. +std::optional new_game_run_problem(std::span rows, const NewGameChoices& choices); + +// 01.00a's step: from HOME (require_new_game_home(), then +// launch_plza_from_home()), plays the language, the look and the name of +// `choices` and ends on "Confirm Choices", confirming nothing. Each +// screen's highlight is read before and after each press; the name is +// typed with the Switch keyboard typer. Throws UserSetupError before any +// press if `choices` cannot be played (new_game_problem(), or the name +// cannot be typed) or HOME is not on screen, or when the keyboard's +// counter takes fewer characters than the name; +// OperationFailedException (with a screenshot) if a screen does not show, +// its highlight does not reach the choice, or a key press does not reach +// the name's field. +void play_new_game_intro(ConsoleHandle& console, ProControllerContext& context, const NewGameChoices& choices); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.cpp new file mode 100644 index 0000000000..d8753123b7 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.cpp @@ -0,0 +1,108 @@ +/* PokemonLZA AutoStory - New game choices + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Strings/Unicode.h" +#include "PokemonLZA_AutoStoryNewGameChoices.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The language list's rows per column. +constexpr size_t LANGUAGE_LIST_ROWS = 5; +constexpr size_t LANGUAGE_LIST_SIZE = 2 * LANGUAGE_LIST_ROWS; + +// The look picker: 2 rows of LOOK_COLUMNS portraits. +constexpr size_t LOOK_COLUMNS = 4; +constexpr size_t LOOK_COUNT = 2 * LOOK_COLUMNS; + +// The presses from (column, row) `from` to `to` on a grid: across first, +// then up or down. +std::vector grid_moves(size_t from_column, size_t from_row, size_t to_column, size_t to_row){ + std::vector moves; + for (size_t c = from_column; c < to_column; c++){ + moves.push_back(IntroKey::RIGHT); + } + for (size_t c = to_column; c < from_column; c++){ + moves.push_back(IntroKey::LEFT); + } + for (size_t r = from_row; r < to_row; r++){ + moves.push_back(IntroKey::DOWN); + } + for (size_t r = to_row; r < from_row; r++){ + moves.push_back(IntroKey::UP); + } + return moves; +} + +} // namespace + + +const NewGameLanguageInfo& new_game_language_info(NewGameLanguage language){ + static const std::array table{{ + {.tag = "en", .list_entry = 0, .name_limit = 12, .ocr_language = Language::English, .confirm_text = "English"}, + {.tag = "es-ES", .list_entry = 1, .name_limit = 12, .ocr_language = Language::Spanish, .confirm_text = "Espa\xc3\xb1" "a)"}, + {.tag = "es-419", .list_entry = 2, .name_limit = 12, .ocr_language = Language::Spanish, .confirm_text = "Latino"}, + {.tag = "fr", .list_entry = 3, .name_limit = 12, .ocr_language = Language::French, .confirm_text = "Fran"}, + {.tag = "de", .list_entry = 4, .name_limit = 12, .ocr_language = Language::German, .confirm_text = "Deutsch"}, + {.tag = "it", .list_entry = 5, .name_limit = 12, .ocr_language = Language::Italian, .confirm_text = "Italiano"}, + {.tag = "ja", .list_entry = 6, .name_limit = 6, .ocr_language = Language::Japanese, .confirm_text = "\xe6\x97\xa5\xe6\x9c\xac"}, + {.tag = "ko", .list_entry = 7, .name_limit = 6, .ocr_language = Language::Korean, .confirm_text = "\xed\x95\x9c\xea\xb5\xad", .keyboard_supported = false}, + {.tag = "zh-Hans", .list_entry = 8, .name_limit = 6, .ocr_language = Language::ChineseSimplified, .confirm_text = "\xe7\xae\x80\xe4\xbd\x93"}, + {.tag = "zh-Hant", .list_entry = 9, .name_limit = 6, .ocr_language = Language::ChineseTraditional, .confirm_text = "\xe7\xb9\x81\xe9\xab\x94"}, + }}; + const size_t index = (size_t)language; + return table[index < table.size() ? index : 0]; +} + +std::optional new_game_problem(const NewGameChoices& choices){ + const NewGameLanguageInfo& info = new_game_language_info(choices.language); + if (choices.look < 1 || choices.look > LOOK_COUNT){ + return std::format("AutoStory: the player look must be 1 to {}, not {}.", LOOK_COUNT, choices.look); + } + if (choices.name.empty()){ + return std::string("AutoStory: the player name is empty."); + } + const size_t characters = utf8_to_utf32(choices.name).size(); + if (characters > info.name_limit){ + return std::format( + "AutoStory: the player name \"{}\" has {} characters; the game takes at most {} in this language ({}).", + choices.name, characters, info.name_limit, info.tag + ); + } + return std::nullopt; +} + +std::optional> language_list_moves(size_t from, size_t to){ + if (from >= LANGUAGE_LIST_SIZE || to >= LANGUAGE_LIST_SIZE){ + return std::nullopt; + } + return grid_moves( + from / LANGUAGE_LIST_ROWS, from % LANGUAGE_LIST_ROWS, + to / LANGUAGE_LIST_ROWS, to % LANGUAGE_LIST_ROWS + ); +} + +std::optional> look_picker_moves(size_t from, size_t to){ + if (from < 1 || from > LOOK_COUNT || to < 1 || to > LOOK_COUNT){ + return std::nullopt; + } + return grid_moves( + (from - 1) % LOOK_COLUMNS, (from - 1) / LOOK_COLUMNS, + (to - 1) % LOOK_COLUMNS, (to - 1) / LOOK_COLUMNS + ); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.h new file mode 100644 index 0000000000..b9296f511f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGameChoices.h @@ -0,0 +1,107 @@ +/* PokemonLZA AutoStory - New game choices + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's choices (language, look, name) and the presses each takes + * on its screen, as pure functions. The screens' readers are in + * Detectors/PokemonLZA_AutoStoryIntroScreens.h; the name is typed with the + * Switch keyboard typer (NintendoSwitch_SystemKeyboardTyping.h). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryNewGameChoices_H +#define PokemonAutomation_PokemonLZA_AutoStoryNewGameChoices_H + +#include +#include +#include +#include +#include +#include "CommonFramework/Language.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The game's 10 languages, in the language list's order: down the left +// column (English, Espanol (Espana), Espanol (Latinoamerica), Francais, +// Deutsch), then down the right one (Italiano, Japanese, Korean, +// Simplified and Traditional Chinese). The cursor starts on English. +enum class NewGameLanguage{ + English, + SpanishSpain, + SpanishLatinAmerica, + French, + German, + Italian, + Japanese, + Korean, + ChineseSimplified, + ChineseTraditional, +}; + +// What the intro needs to know of one language. +struct NewGameLanguageInfo{ + // Short name for logs and capture files: "en", "es-ES", "ja", ... + std::string_view tag; + // Its entry on the language list (the order of NewGameLanguage). + size_t list_entry = 0; + // The name keyboard's character limit: its "0/N" counter (the + // console's frames of 2026-11-15: 12 for English and the other + // European languages, 6 for Japanese, Korean and Chinese). + size_t name_limit = 0; + // The OCR language that reads its name on Confirm Choices. + Language ocr_language = Language::None; + // A part of its name as Confirm Choices shows it ("Deutsch", "Latino"). + std::string_view confirm_text; + // The program can type a name on its keyboard (Korean: no; its + // keyboard is Hangul, which the Switch keyboard typer does not read). + // A run from 01.00b types nothing. + bool keyboard_supported = true; +}; + +// `language`'s row of the table above. +const NewGameLanguageInfo& new_game_language_info(NewGameLanguage language); + +// One press on an intro screen. +enum class IntroKey{ + UP, + DOWN, + LEFT, + RIGHT, + A, +}; + +// The new game's choices, from the program's options. +struct NewGameChoices{ + NewGameLanguage language = NewGameLanguage::English; + // 1 to 8: the look picker's portraits, left to right along the top + // row, then the bottom one. + size_t look = 1; + // At most the language's name_limit characters. A run that types it + // (one that plays 01.00a) takes only letters, digits + // and the symbols on the name keyboard's first page (- / : ' , . ? !). + std::string name = "Kosmos"; +}; + +// Why `choices` cannot be played from any start screen, before any press; +// nullopt when they can. +std::optional new_game_problem(const NewGameChoices& choices); + +// The presses that move the language list's cursor from entry `from` to +// entry `to`: LEFT or RIGHT to the column, then UP or DOWN to the row. +// Nullopt for an entry outside the list. +std::optional> language_list_moves(size_t from, size_t to); + +// The presses (stick tilts) that move the look picker's highlight from +// look `from` to look `to` (1 to 8): LEFT or RIGHT, then UP or DOWN. +// Nullopt for a look outside 1 to 8. +std::optional> look_picker_moves(size_t from, size_t to); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.cpp new file mode 100644 index 0000000000..071f294f3b --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.cpp @@ -0,0 +1,304 @@ +/* PokemonLZA AutoStory - New game Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryStoryFiles.h" +#include "PokemonLZA_AutoStoryNewGame.h" +#include "PokemonLZA_AutoStoryNewGameChoices.h" +#include "PokemonLZA_AutoStoryNewGame_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// Presses as letters: U D L R, A. +std::string press_text(const std::optional>& presses){ + if (!presses){ + return "none"; + } + std::string text; + for (const IntroKey key : *presses){ + switch (key){ + case IntroKey::UP: + text += 'U'; + break; + case IntroKey::DOWN: + text += 'D'; + break; + case IntroKey::LEFT: + text += 'L'; + break; + case IntroKey::RIGHT: + text += 'R'; + break; + case IntroKey::A: + text += 'A'; + break; + } + } + return text; +} + + +// Hangul names: 6 and 7 characters of 3 UTF-8 bytes each. +const std::string HANGUL_6 = "\xec\xbd\x94\xec\x8a\xa4\xeb\xaa\xa8\xec\x8a\xa4\xec\xbd\x94\xec\x8a\xa4"; +const std::string HANGUL_7 = "\xec\xbd\x94\xec\x8a\xa4\xeb\xaa\xa8\xec\x8a\xa4\xec\xbd\x94\xec\x8a\xa4\xeb\xaa\xa8"; + + +// new_game_problem(): whether the program refuses the choices before any +// press, whatever screen the run starts on. +struct ProblemCase{ + std::string name; + struct Input{ + NewGameChoices choices; + }; + Input input; + struct Expectation{ + bool refused = false; + }; + Expectation expectation; +}; + +const ProblemCase PROBLEM_CASES[] = { + {.name = "the defaults", .input = {.choices = {}}, .expectation = {.refused = false}}, + { + .name = "12 characters in English", + .input = {.choices = {.name = "Abcdefghijkl"}}, + .expectation = {.refused = false}, + }, + { + .name = "13 characters in English", + .input = {.choices = {.name = "Abcdefghijklm"}}, + .expectation = {.refused = true}, + }, + { + .name = "12 characters in German", + .input = {.choices = {.language = NewGameLanguage::German, .name = "Abcdefghijkl"}}, + .expectation = {.refused = false}, + }, + { + .name = "6 characters in Japanese", + .input = {.choices = {.language = NewGameLanguage::Japanese, .name = "Kosmos"}}, + .expectation = {.refused = false}, + }, + { + .name = "7 characters in Japanese", + .input = {.choices = {.language = NewGameLanguage::Japanese, .name = "Kosmoss"}}, + .expectation = {.refused = true}, + }, + { + .name = "7 characters in Traditional Chinese", + .input = {.choices = {.language = NewGameLanguage::ChineseTraditional, .name = "Kosmoss"}}, + .expectation = {.refused = true}, + }, + { + .name = "6 Hangul characters in Korean", + .input = {.choices = {.language = NewGameLanguage::Korean, .name = HANGUL_6}}, + .expectation = {.refused = false}, + }, + { + .name = "7 Hangul characters in Korean", + .input = {.choices = {.language = NewGameLanguage::Korean, .name = HANGUL_7}}, + .expectation = {.refused = true}, + }, + {.name = "an empty name", .input = {.choices = {.name = ""}}, .expectation = {.refused = true}}, + {.name = "look 8", .input = {.choices = {.look = 8}}, .expectation = {.refused = false}}, + {.name = "look 0", .input = {.choices = {.look = 0}}, .expectation = {.refused = true}}, + {.name = "look 9", .input = {.choices = {.look = 9}}, .expectation = {.refused = true}}, +}; + +class Test_NewGameProblem : public UnitTest{ +public: + explicit Test_NewGameProblem(const ProblemCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - new game choices: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional problem = new_game_problem(m_test.input.choices); + if (problem.has_value() != m_test.expectation.refused){ + return std::format( + "Expected {}, got {}.", + m_test.expectation.refused ? "a refusal" : "no problem", + problem ? *problem : std::string("no problem") + ); + } + return true; + } + +private: + const ProblemCase& m_test; +}; + + +// new_game_run_problem(): whether the program refuses the choices for a +// run of the built-in rows `rows`: a run that plays 01.00a types the name, +// one from 01.00b finds it typed. Every refusal ends with +// NEW_GAME_BY_HAND_HINT. +struct RunProblemCase{ + std::string name; + struct Input{ + std::vector rows; + NewGameChoices choices; + }; + Input input; + struct Expectation{ + bool refused = false; + }; + Expectation expectation; +}; + +const RunProblemCase RUN_PROBLEM_CASES[] = { + {.name = "the defaults from 01.00a", .input = {.rows = {"01.00a", "01.00b"}, .choices = {}}, .expectation = {.refused = false}}, + { + .name = "Korean from 01.00a", + .input = {.rows = {"01.00a", "01.00b"}, .choices = {.language = NewGameLanguage::Korean, .name = HANGUL_6}}, + .expectation = {.refused = true}, + }, + { + .name = "Korean through 01.00a only", + .input = {.rows = {"01.00a"}, .choices = {.language = NewGameLanguage::Korean, .name = HANGUL_6}}, + .expectation = {.refused = true}, + }, + { + .name = "Korean from 01.00b", + .input = {.rows = {"01.00b", "01.01"}, .choices = {.language = NewGameLanguage::Korean, .name = HANGUL_6}}, + .expectation = {.refused = false}, + }, + {.name = "Japanese from 01.00a", .input = {.rows = {"01.00a"}, .choices = {.language = NewGameLanguage::Japanese}}, .expectation = {.refused = false}}, + {.name = "a space in the name from 01.00a", .input = {.rows = {"01.00a"}, .choices = {.name = "Kos mos"}}, .expectation = {.refused = true}}, + { + .name = "digits and the first page's symbols from 01.00a", + .input = {.rows = {"01.00a"}, .choices = {.name = "a1-/:',.?!"}}, + .expectation = {.refused = false}, + }, + {.name = "an accented letter from 01.00a", .input = {.rows = {"01.00a"}, .choices = {.name = "Jos\xc3\xa9"}}, .expectation = {.refused = true}}, + {.name = "an accented letter from 01.00b", .input = {.rows = {"01.00b"}, .choices = {.name = "Jos\xc3\xa9"}}, .expectation = {.refused = false}}, + {.name = "a symbol of the second page from 01.00a", .input = {.rows = {"01.00a"}, .choices = {.name = "a#"}}, .expectation = {.refused = true}}, +}; + +// The built-in rows `ids`, in that order. +std::vector builtin_rows(const std::vector& ids){ + std::vector rows; + for (const std::string& id : ids){ + for (const Pokemon::StoryCheckpoint& row : builtin_story_data().checkpoints){ + if (row.id == id){ + rows.push_back(row); + } + } + } + return rows; +} + +class Test_NewGameRunProblem : public UnitTest{ +public: + explicit Test_NewGameRunProblem(const RunProblemCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - new game choices by run: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::vector rows = builtin_rows(m_test.input.rows); + if (rows.size() != m_test.input.rows.size()){ + return std::string("A row of the case is not in the built-in story."); + } + const std::optional problem = new_game_run_problem(rows, m_test.input.choices); + if (problem.has_value() != m_test.expectation.refused){ + return std::format( + "Expected {}, got {}.", + m_test.expectation.refused ? "a refusal" : "no problem", + problem ? *problem : std::string("no problem") + ); + } + if (problem && !problem->ends_with(NEW_GAME_BY_HAND_HINT)){ + return std::format("Expected the refusal to end with the hint, got \"{}\".", *problem); + } + return true; + } + +private: + const RunProblemCase& m_test; +}; + + +// The language list's moves (entries down the left column, then the right +// one) and the look picker's (looks along the top row, then the bottom). +enum class Grid{ + LANGUAGE_LIST, + LOOK_PICKER, +}; + +struct GridCase{ + std::string name; + struct Input{ + Grid grid = Grid::LANGUAGE_LIST; + size_t from = 0; + size_t to = 0; + }; + Input input; + struct Expectation{ + std::string presses; + }; + Expectation expectation; +}; + +const GridCase GRID_CASES[] = { + {.name = "English to Deutsch", .input = {.from = 0, .to = 4}, .expectation = {.presses = "DDDD"}}, + {.name = "English to Italiano", .input = {.from = 0, .to = 5}, .expectation = {.presses = "R"}}, + {.name = "English to Traditional Chinese", .input = {.from = 0, .to = 9}, .expectation = {.presses = "RDDDD"}}, + {.name = "Simplified Chinese to Francais", .input = {.from = 8, .to = 3}, .expectation = {.presses = "L"}}, + {.name = "an entry past the list", .input = {.from = 0, .to = 10}, .expectation = {.presses = "none"}}, + {.name = "look 1 to look 8", .input = {.grid = Grid::LOOK_PICKER, .from = 1, .to = 8}, .expectation = {.presses = "RRRD"}}, + {.name = "look 7 to look 2", .input = {.grid = Grid::LOOK_PICKER, .from = 7, .to = 2}, .expectation = {.presses = "LU"}}, + {.name = "look 0", .input = {.grid = Grid::LOOK_PICKER, .from = 0, .to = 2}, .expectation = {.presses = "none"}}, +}; + +class Test_GridMoves : public UnitTest{ +public: + explicit Test_GridMoves(const GridCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - intro moves: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string presses = press_text(m_test.input.grid == Grid::LANGUAGE_LIST + ? language_list_moves(m_test.input.from, m_test.input.to) + : look_picker_moves(m_test.input.from, m_test.input.to) + ); + if (presses != m_test.expectation.presses){ + return std::format("Expected {}, got {}.", m_test.expectation.presses, presses); + } + return true; + } + +private: + const GridCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryNewGame(UnitTestDatabase& database){ + for (const ProblemCase& test : PROBLEM_CASES){ + database.add(test); + } + for (const RunProblemCase& test : RUN_PROBLEM_CASES){ + database.add(test); + } + for (const GridCase& test : GRID_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.h new file mode 100644 index 0000000000..cc79bcfee1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryNewGame_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - New game Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryNewGame_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryNewGame_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the new game's tests: the choices' checks, the language list's +// and look picker's moves and the name's key presses. +void add_tests_AutoStoryNewGame(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.cpp new file mode 100644 index 0000000000..76b2256562 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.cpp @@ -0,0 +1,409 @@ +/* PokemonLZA AutoStory - Walk screen checks + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryReadings.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStorySceneMachine.h" +#include "PokemonLZA_AutoStoryWalkScreenChecks.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace std::chrono_literals; + + +namespace{ + +// How long each slice of the held last leg is, between looks for its end. +constexpr Milliseconds HOLD_SLICE = 250ms; + +// How long a walk waits for free control, and how often it looks. +constexpr Milliseconds FREE_CONTROL_WAIT = 30000ms; +constexpr Milliseconds FREE_CONTROL_LOOK = 250ms; +// Nothing busy for this long, with no sign of control, is taken as +// control, for places that show no sign (Hotel Z's halls show no party +// bar). Longer than the ~7 s from the end of 01.01's scenes to control +// (console B, 2026-11-15). +constexpr Milliseconds QUIET_IS_CONTROL = 10000ms; + +// An A prompt held this long with nothing busy is control: it shows only +// in the overworld, and where no party bar shows (01.03 before the +// starter has no party) it is the one sign there is. A second, as the scene machine holds it, so a +// prompt left up under a late fade after A at a door is not taken. +constexpr Milliseconds PROMPT_IS_CONTROL = 1000ms; + +// A walk to an A prompt ends on it only when it shows again this long +// after a look that saw it, the stick let go: a first glimpse can be the +// prompt fading in or the player still moving, and the talk after it +// then finds none (02.05's elevator in German, console B 2026-11-16). +constexpr Milliseconds PROMPT_STEADY_GAP = 300ms; + +// Whether `look` shows that A at a door's prompt was taken: a fade, or +// text. +bool prompt_taken(const ScreenObservation& look){ + return fade_shows(look) || look.events.has(ScreenEvent::DIALOG_BOX) || look.events.has(ScreenEvent::CHOICE_BOX); +} + +// What no walk may start on, by name; empty when free. +std::string_view not_free_control(const ScreenObservation& look){ + if (look.readings.contains(ADVANCE_ARROW_READING)){ + return "text waiting for A"; + } + if (look.events.has(ScreenEvent::DIALOG_BOX)){ + return "a text box"; + } + if (fade_shows(look)){ + return "a fade"; + } + return ""; +} + +// A sign that the player has control: the party bar, the station +// platform after the opening (PlatformArrivalDetector), or the +// tutorial's controls hint. +bool control_shows(const ScreenObservation& look){ + return look.events.has(ScreenEvent::PLAYER_CAN_MOVE) || look.readings.contains(PLATFORM_READING) || + look.readings.contains(CONTROLS_HINT_READING); +} + +// Whether the A prompt a look just saw shows again PROMPT_STEADY_GAP +// later, the player standing. +bool prompt_steady(PlzaBackend& backend){ + backend.wait(PROMPT_STEADY_GAP); + return backend.observe().events.has(ScreenEvent::INTERACT_PROMPT); +} + +// How long "talk" looks for its A prompt before A, and how often it looks. +// The prompt can take seconds to come up after the walk to it (02.05's +// elevator in German, console B 2026-11-16: none within 1 s, steady a +// few seconds later). +constexpr Milliseconds TALK_PROMPT_WAIT = 3000ms; +constexpr Milliseconds TALK_POLL = 250ms; + +// Whether the A prompt shows on two looks in a row within +// TALK_PROMPT_WAIT: a walk that ends as it comes into reach can drift out +// of it again, and an A on that one look is lost (02.05's elevator, +// console B 2026-11-16). +bool talk_prompt_settled(PlzaBackend& backend){ + size_t prompt_looks = 0; + for (Milliseconds waited(0); ; waited += TALK_POLL){ + backend.scope().throw_if_cancelled(); + prompt_looks = backend.observe().events.has(ScreenEvent::INTERACT_PROMPT) ? prompt_looks + 1 : 0; + if (prompt_looks >= 2){ + return true; + } + if (waited >= TALK_PROMPT_WAIT){ + return false; + } + backend.wait(TALK_POLL); + } +} + +// How a_at_prompt() goes on after A at the A prompt the player stands at. +struct AtPrompt{ + // What shows the game took A, and on how many looks in a row. + bool (*until)(const ScreenObservation& look); + size_t until_looks; + // As in all, and how long after an A the prompt must still be up + // with `until` not shown for the next. + size_t retries; + Milliseconds retry_gap; + // How long it looks after the first A, at most. None: each A gets + // `retry_gap`, decided on its last look, and ends there with the + // prompt gone. + std::optional wait; + // The release after each A. + Milliseconds release; + // The name it logs an A again under. + std::string_view name; +}; + +// How a_at_prompt() ended, after how many As, and its last look. +enum class AtPromptEnd{ + TAKEN, + PROMPT_STAYS, + NOTHING, +}; +struct AtPromptResult{ + AtPromptEnd end; + size_t presses; + Milliseconds since_press; + Pokemon::Observation look; +}; + +// A at a prompt, then a look every TALK_POLL until `at.until` shows on +// `at.until_looks` looks in a row (TAKEN); while the prompt stays up +// with nothing open `at.retry_gap` after A, A again, `at.retries` As at +// most (then PROMPT_STAYS). NOTHING: `at.wait` passed, or with no +// `at.wait`, the prompt gone at the end of an A's `at.retry_gap`. +AtPromptResult a_at_prompt(PlzaBackend& backend, const AtPrompt& at){ + backend.press(BUTTON_A, /*hold=*/160ms, at.release); + size_t presses = 1; + Milliseconds since_press(0); + size_t until_looks = 0; + for (Milliseconds waited(0); ; ){ + backend.scope().throw_if_cancelled(); + Pokemon::Observation look = backend.observe(); + until_looks = at.until(look) ? until_looks + 1 : 0; + if (until_looks >= at.until_looks){ + return {AtPromptEnd::TAKEN, presses, since_press, std::move(look)}; + } + const bool prompt_stays = until_looks == 0 && look.events.has(ScreenEvent::INTERACT_PROMPT); + bool again = false; + if (!at.wait && since_press + TALK_POLL >= at.retry_gap){ + backend.wait(TALK_POLL); + since_press += TALK_POLL; + if (!prompt_stays){ + return {AtPromptEnd::NOTHING, presses, since_press, std::move(look)}; + } + again = true; + } + if (at.wait && prompt_stays && since_press >= at.retry_gap){ + again = true; + } + if (again){ + if (presses >= at.retries){ + return {AtPromptEnd::PROMPT_STAYS, presses, since_press, std::move(look)}; + } + backend.log(std::format( + "[PLZA] {}: the A prompt still up {} ms after A {} with nothing open; A again", at.name, since_press.count(), presses + )); + backend.press(BUTTON_A, /*hold=*/160ms, at.release); + presses++; + since_press = Milliseconds(0); + continue; + } + if (at.wait && waited >= *at.wait){ + return {AtPromptEnd::NOTHING, presses, since_press, std::move(look)}; + } + backend.wait(TALK_POLL); + waited += TALK_POLL; + since_press += TALK_POLL; + } +} + +// "talk"'s A: its scene under way on two looks in a row within 5 s, A +// again every second the prompt stays up with nothing open, 3 As at most. +const AtPrompt TALK{ + .until = scene_under_way, + .until_looks = 2, + .retries = 3, + .retry_gap = 1000ms, + .wait = 5000ms, + .release = 240ms, + .name = "talk", +}; + +// A door's A: a fade or text within 3 s (a late fade starts 1.8 s after +// the A), A again while the prompt stays with neither (a dropped A), 3 +// As at most. +const AtPrompt THROUGH_PROMPT{ + .until = prompt_taken, + .until_looks = 1, + .retries = 3, + .retry_gap = 3000ms, + .wait = std::nullopt, + .release = 400ms, + .name = "press_through_prompt", +}; + +// Whether walk_until() of `move` to `end` from `end_after` ends before +// its first leg on the A prompt it starts at: a walk of one straight leg +// to the first A prompt ahead. A walk that turns or has legs before its +// last goes to a prompt elsewhere, and one with an `end_after` passes +// prompts on its way. +bool ends_before_walking( + PlzaBackend& backend, std::string_view walk, const StoryMove& move, ScreenEvent end, Milliseconds end_after +){ + const bool straight_to_prompt = end == ScreenEvent::INTERACT_PROMPT && end_after == 0ms && + move.legs.size() == 1 && move.legs[0].turn == 0; + if (!straight_to_prompt || !backend.observe().events.has(end) || !prompt_steady(backend)){ + return false; + } + backend.log(std::format("[PLZA] {}: the A prompt shows before the walk: no walk.", walk)); + return true; +} + +// What a look `held` into a walk's last leg says for walk_until() with +// the event `end`: how the walk ends, or none to walk on. +std::optional end_event_look( + PlzaBackend& backend, std::string_view walk, const ScreenObservation& look, + ScreenEvent end, Milliseconds held, Milliseconds end_after +){ + if (look.events.has(end) && held < end_after){ + backend.log(std::format( + "[PLZA] {}: the end event after {} ms of the last leg, before its {} ms: passed on the way, on.", + walk, held.count(), end_after.count() + )); + return std::nullopt; + } + if (look.events.has(end) && end == ScreenEvent::INTERACT_PROMPT && !prompt_steady(backend)){ + backend.log(std::format("[PLZA] {}: the A prompt after {} ms of the last leg went again: on.", walk, held.count())); + return std::nullopt; + } + if (look.events.has(end)){ + backend.log(std::format("[PLZA] {}: the end event after {} ms of the last leg.", walk, held.count())); + backend.log_detail(std::format("[PLZA] {}: ended on: {}", walk, describe_scene_look(look))); + return Pokemon::outcome_ok(); + } + if (look.events.has(ScreenEvent::FADE_BLACK)){ + backend.log(std::format("[PLZA] {}: a black fade after {} ms of the last leg: the scene takes over.", walk, held.count())); + return Pokemon::outcome_ok(); + } + return std::nullopt; +} + +// A last leg that only turns (to face the clerk): what the walk goes to +// shows once the turn is done, or the walk missed it. +Pokemon::Outcome end_after_turn(PlzaBackend& backend, std::string_view walk, ScreenEvent end){ + if (backend.observe().events.has(end)){ + backend.log(std::format("[PLZA] {}: the end event after the last turn.", walk)); + return Pokemon::outcome_ok(); + } + return Pokemon::outcome_failed("walk_no_end", std::format("{}: no end event after the last turn", walk)); +} + +} // namespace + + +Pokemon::Outcome check_can_walk(PlzaBackend& backend, std::string_view walk){ + std::string_view last_busy = "nothing"; + int control_looks = 0; + std::optional quiet_since; + std::optional prompt_since; + for (Milliseconds waited = 0ms; waited <= FREE_CONTROL_WAIT; waited += FREE_CONTROL_LOOK){ + const Pokemon::Observation look = backend.observe(); + const std::string_view busy = not_free_control(look); + if (!busy.empty()){ + last_busy = busy; + quiet_since.reset(); + }else if (!quiet_since){ + quiet_since = waited; + } + control_looks = busy.empty() && control_shows(look) ? control_looks + 1 : 0; + if (control_looks >= 2){ + return Pokemon::outcome_ok(); + } + if (busy.empty() && look.events.has(ScreenEvent::INTERACT_PROMPT)){ + if (!prompt_since){ + prompt_since = waited; + } + }else{ + prompt_since.reset(); + } + if (prompt_since && waited - *prompt_since >= PROMPT_IS_CONTROL){ + backend.log(std::format("[PLZA] {}: the A prompt held {} ms with nothing busy: control.", walk, PROMPT_IS_CONTROL.count())); + return Pokemon::outcome_ok(); + } + if (quiet_since && waited - *quiet_since >= QUIET_IS_CONTROL){ + backend.log(std::format( + "[PLZA] {}: nothing busy for {} ms and no sign of control; taken as control.", walk, QUIET_IS_CONTROL.count() + )); + return Pokemon::outcome_ok(); + } + backend.wait(FREE_CONTROL_LOOK); + } + return Pokemon::outcome_failed( + "walk_not_free", + std::format("{}: no free control within {} ms, last on {}", walk, FREE_CONTROL_WAIT.count(), last_busy) + ); +} + +void finish_walk(PlzaBackend& backend, std::string_view walk){ + const Pokemon::Observation look = backend.observe(); + backend.log(std::format("[PLZA] {}: legs done, screen now: {}", walk, describe(look.events))); +} + +Pokemon::Outcome walk_until( + PlzaBackend& backend, std::string_view walk, const StoryMove& move, std::optional end, Milliseconds end_after +){ + if (end && ends_before_walking(backend, walk, move, *end, end_after)){ + return Pokemon::outcome_ok(); + } + const Milliseconds hold = move.legs.empty() ? 0ms : move.legs.back().walk; + const Pokemon::Outcome walked = run_story_move_but_last_walk(backend, move); + if (!walked.succeeded()){ + return walked; + } + if (hold == 0ms && end){ + return end_after_turn(backend, walk, *end); + } + size_t scene_looks = 0; + // The slices stop the stick between them for as long as a look takes; + // the player barely slows. + for (Milliseconds held = 0ms; held < hold;){ + backend.scope().throw_if_cancelled(); + // The last slice holds only what is left, so the leg walks its time. + const Milliseconds slice = std::min(HOLD_SLICE, hold - held); + hold_last_leg(backend, move, slice); + held += slice; + const Pokemon::Observation look = backend.observe(); + if (!end){ + const std::string_view busy = not_free_control(look); + scene_looks = busy.empty() ? 0 : scene_looks + 1; + if (scene_looks >= 2){ + backend.log(std::format( + "[PLZA] {}: a scene ({}) took control after {} ms of the last leg: the walk stops.", walk, busy, held.count() + )); + return Pokemon::outcome_ok(); + } + continue; + } + const std::optional ended = end_event_look(backend, walk, look, *end, held, end_after); + if (ended){ + return *ended; + } + } + if (!end){ + return Pokemon::outcome_failed( + "walk_no_scene", std::format("{}: control still free after {} ms of the last leg: no scene took it", walk, hold.count()) + ); + } + return Pokemon::outcome_failed("walk_no_end", std::format("{}: no end event within {} ms of the last leg", walk, hold.count())); +} + +Pokemon::Outcome talk_at_prompt(PlzaBackend& backend){ + if (!talk_prompt_settled(backend)){ + return Pokemon::outcome_failed( + "talk_no_prompt", std::format("no A prompt on two looks within {} ms: not at what the talk is for", TALK_PROMPT_WAIT.count()) + ); + } + const AtPromptResult result = a_at_prompt(backend, TALK); + switch (result.end){ + case AtPromptEnd::TAKEN: + backend.log_detail(std::format( + "[PLZA] talk: started {} ms after A {}: {}", result.since_press.count(), result.presses, describe_scene_look(result.look) + )); + return Pokemon::outcome_ok(); + case AtPromptEnd::PROMPT_STAYS: + return Pokemon::outcome_failed("talk_no_scene", std::format( + "the A prompt still up with nothing open after {} As: the game took none", result.presses + )); + case AtPromptEnd::NOTHING: + break; + } + return Pokemon::outcome_failed("talk_no_scene", std::format("nothing started within {} ms of A: the HUD stayed", TALK.wait->count())); +} + +void press_through_prompt(PlzaBackend& backend){ + const AtPromptResult result = a_at_prompt(backend, THROUGH_PROMPT); + if (result.end == AtPromptEnd::PROMPT_STAYS){ + backend.log(std::format( + "[PLZA] press_through_prompt: the prompt is still up {} ms after A {}", result.since_press.count(), result.presses + )); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.h new file mode 100644 index 0000000000..1819b6590f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Actions/PokemonLZA_AutoStoryWalkScreenChecks.h @@ -0,0 +1,91 @@ +/* PokemonLZA AutoStory - Walk screen checks + * + * From: https://github.com/PokemonAutomation/ + * + * What a story walk (run_story_move(), PokemonLZA_AutoStoryMoves.h) reads + * of the screen, since the walk itself reads nothing: before it, that the + * player has free control (check_can_walk()); during its last leg, the + * event it ends on (walk_until()); after it, where it stopped + * (finish_walk()). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryWalkScreenChecks_H +#define PokemonAutomation_PokemonLZA_AutoStoryWalkScreenChecks_H + +#include +#include +#include "CommonTools/ScreenEvents/ScreenEventTypes.h" +#include "Pokemon/AutoStory/Pokemon_StoryOutcome.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryBackend.h" +#include "PokemonLZA_AutoStoryMoves.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Waits for free control: two looks in a row 250 ms apart with a sign of +// control (the party bar, the station platform, the controls hint) and no advance arrow, +// dialog box or fade. The end of a scene is not control: 01.01's scenes +// end ~7 s before it. An A prompt held a second with none of those is +// control too (01.03 has no party, so no party bar). Where no sign shows, +// 10 s with none of those is taken as control. Fails as +// "walk_not_free" when neither comes +// within 30 s: a walk started anywhere else walks blind. +Pokemon::Outcome check_can_walk(PlzaBackend& backend, std::string_view walk); + +// After a walk's legs with no end event: one look, which on a console +// waits for the legs' inputs to run (observe()), so the step ends where +// the walk does, not when its inputs are queued. Logs what the screen +// shows there. +void finish_walk(PlzaBackend& backend, std::string_view walk); + +// Walks `move`, holding its last leg in slices of 250 ms +// (hold_last_leg()) until a look shows `end` (a fade, the box of a talk +// the walk runs into, or the A prompt of what it walks to), the leg's +// time as the upper bound. An A prompt ends it only when it shows again +// on a look 300 ms later, the stick let go; else the leg goes on. A black +// fade ends it too: the scene at the walk's end takes over. A last leg +// that only turns is looked at once, after the turn. Fails as +// "walk_no_end" if neither shows within it. `end` seen before `end_after` +// of the last leg (the plan's measured walk) does not end it: an NPC's A +// prompt beside the path is passed on the way, not the one the walk goes +// to (02.05's Alton). A walk of one straight leg to an A prompt with no +// `end_after` ends before walking on the prompt it starts at, steady +// 300 ms (02.02's street scene can end at the gate). +// With no `end`, the walk ends when a scene takes control: something no +// walk may start on (a text box, text waiting for A, a fade) on two looks +// in a row, for a walk that runs off into a scene with no measured end +// (03.03's drop off the roof). Fails as "walk_no_scene" when control is +// still free at the end of the last leg. +Pokemon::Outcome walk_until( + PlzaBackend& backend, std::string_view walk, const StoryMove& move, std::optional end, Milliseconds end_after +); + +// "talk": on the A prompt of what the player faces (two looks in a row), +// A, then its scene under way (scene_under_way(): Taunie's opens on a +// card, 03.02b) on two looks in a row within 5 s, not one look of the +// press itself. While the prompt stays up with nothing open, A again +// every second, 3 As at most: the game can drop an A (02.05's elevator, +// console B 2026-11-16, then read the overworld 120 s). No prompt on two +// looks in a row within 3 s (it can come up seconds after the walk to it +// ends: 02.05's elevator in German, 2026-11-16): nothing pressed, +// "talk_no_prompt" (the walk stopped short, 03.02, 2026-11-16); nothing +// after the As: "talk_no_scene". +Pokemon::Outcome talk_at_prompt(PlzaBackend& backend); + +// A at the A prompt the player stands at (a door's "A Enter"), then looks +// for up to 3 s for a fade or text; while the prompt stays with neither, +// A again, 3 times at most. The party bar shows under the prompt, so a +// scene machine run right after would take it for control before a late +// fade. A prompt gone with no fade or text, or still up after the last A, +// is left to what runs next. +void press_through_prompt(PlzaBackend& backend); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp new file mode 100644 index 0000000000..066f5dd23b --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.cpp @@ -0,0 +1,134 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/ImageTypes/BinaryImage.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "Kernels/Waterfill/Kernels_Waterfill_Session.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace Kernels::Waterfill; + + +namespace{ + +// How well `blob` (a Waterfill object kept with its pixels) fills a V +// across its bounds: its overlap with the V over their union. Each arm +// runs from 10% in at the top to the middle at the bottom, 22% of the +// width across. +double v_match(const WaterfillObject& blob){ + const double width = double(blob.width()); + const double height = double(blob.height()); + size_t both = 0; + size_t either = 0; + for (size_t y = blob.min_y; y < blob.max_y; y++){ + const double v = (double(y - blob.min_y) + 0.5) / height; + for (size_t x = blob.min_x; x < blob.max_x; x++){ + const double u = (double(x - blob.min_x) + 0.5) / width; + const bool in_v = std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.11; + const bool in_blob = blob.object->get(x, y); + if (in_v && in_blob){ + both++; + } + if (in_v || in_blob){ + either++; + } + } + } + return either == 0 ? 0 : double(both) / double(either); +} + +// The pixels of `mask` outside `blob` within `margin` of its bounds, per +// pixel of the blob. The blob lies inside the window, so they are the +// window's pixels less its own. +double stray_share(const PackedBinaryMatrix& mask, const WaterfillObject& blob, size_t margin){ + const size_t min_x = blob.min_x > margin ? blob.min_x - margin : 0; + const size_t min_y = blob.min_y > margin ? blob.min_y - margin : 0; + const size_t max_x = std::min(blob.max_x + margin, mask.width()); + const size_t max_y = std::min(blob.max_y + margin, mask.height()); + size_t in_window = 0; + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + if (mask.get(x, y)){ + in_window++; + } + } + } + return double(in_window - blob.area) / double(blob.area); +} + +// Whether `blob` of `mask` is the chevron, on a screen `scale` times 1080p: +// 32x19 measured, 30-33 x 18-20 on every frame with it. +bool chevron_blob(const PackedBinaryMatrix& mask, const WaterfillObject& blob, double scale){ + const double width = double(blob.width()) / scale; + const double height = double(blob.height()) / scale; + if (width < 26 || width > 40 || height < 15 || height > 24){ + return false; + } + const double aspect = width / height; + if (aspect < 1.4 || aspect > 2.1){ + return false; + } + return stray_share(mask, blob, size_t(4 * scale + 0.5)) <= 0.09 && v_match(blob) >= 0.45; +} + +// Whether a 4-connected blob of the pixels of `area` in the colour range +// `mins`-`maxs` is the chevron. +bool chevron_in(const ImageViewRGB32& area, double scale, uint32_t mins, uint32_t maxs){ + const PackedBinaryMatrix mask = compress_rgb32_to_binary_range(area, mins, maxs); + PackedBinaryMatrix unfilled = mask.copy(); + const std::unique_ptr session = make_WaterfillSession(unfilled); + const std::unique_ptr blobs = session->make_iterator(1); + WaterfillObject blob; + while (blobs->find_next(blob, true)){ + if (chevron_blob(mask, blob, scale)){ + return true; + } + } + return false; +} + +} // namespace + + +AdvanceArrowDetector::AdvanceArrowDetector(Color color) + : m_color(color) + , m_area(1370.0 / 1920, 930.0 / 1080, 110.0 / 1920, 120.0 / 1080) +{} + +void AdvanceArrowDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_area); +} + +// On 1920x1080 frames saved as JPEG (quality 80) the chevron (on the ad's +// subtitles, on white boxes, on the opening's blue box) fills the V +// 0.51-0.71 with at most 6% stray pixels. The closest other shape in size +// and shape, a dark blade of scenery, fills the V 0.48 but has 13% stray +// pixels, so stray pixels are allowed up to 9%; shapes with nothing +// around them fill the V 0.43 or less and are out of the chevron's size. +// JPEG keeps colour at half resolution, so a white chevron over orange +// (the player's shorts) loses blue below 235 at its edges: white asks +// for 220 in blue (any floor from 190 to 230 reads every frame right). +bool AdvanceArrowDetector::detect(const ImageViewRGB32& screen){ + const double scale = double(screen.height()) / 1080; + const ImageViewRGB32 area = extract_box_reference(screen, m_area); + // White: red and green 235 or more, blue 220 or more; dark: every + // channel 110 or under. + return chevron_in(area, scale, /*mins=*/0x00ebebdc, /*maxs=*/0xffffffff) || chevron_in(area, scale, /*mins=*/0x00000000, /*maxs=*/0xff6e6e6e); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h new file mode 100644 index 0000000000..fb27998a3b --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAdvanceArrowDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Advance arrow detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "press A to go on" chevron at the bottom right of a dialogue box or + * of the boxless subtitles of the ad early in the story. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAdvanceArrowDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The "press A to go on" chevron (a V about 32x19 on 1920x1080) at the +// right end of a dialogue box and of boxless subtitles: dark on a white +// box, white on a blue box and on the ad's subtitles. Found by shape in +// x 1370-1480, y 930-1050: a blob of white (red and green 235+, blue 220+) or dark +// (every channel 110 or under) pixels the size of the chevron, filling a +// V, with no other pixel of its colour around it. Text is not read, so +// any language matches. +class AdvanceArrowDetector : public StaticScreenDetector{ +public: + explicit AdvanceArrowDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_area; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp new file mode 100644 index 0000000000..01b16a77c5 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.cpp @@ -0,0 +1,91 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The icon (white, with a cyan glow, dim or bright), and the strips around +// it (left, above, below) clear of white and bright cyan, `dx` pixels (of +// 1920) right of where the Chinese badge has them. +bool autosave_icon_at( + const ImageViewRGB32& screen, const ImageFloatBox& icon, + std::initializer_list around, double dx +){ + const ImageFloatBox icon_here = shifted(icon, dx); + const double white = white_share(screen, icon_here); + if (white < 0.08 || white > 0.40 || dim_cyan_share(screen, icon_here) < 0.25){ + return false; + } + for (const ImageFloatBox& box : around){ + const ImageFloatBox here = shifted(box, dx); + if (white_share(screen, here) >= 0.05 || cyan_share(screen, here) >= 0.05){ + return false; + } + } + return true; +} + +} // namespace + + +AutosaveDetector::AutosaveDetector(Color color) + : m_color(color) + , m_icon(1603.0 / 1920, 73.0 / 1080, 27.0 / 1920, 31.0 / 1080) + , m_left(1570.0 / 1920, 73.0 / 1080, 28.0 / 1920, 31.0 / 1080) + , m_above(1603.0 / 1920, 56.0 / 1080, 27.0 / 1920, 14.0 / 1080) + , m_below(1603.0 / 1920, 108.0 / 1080, 27.0 / 1920, 14.0 / 1080) +{} + +void AutosaveDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_icon); + items.add(m_color, m_left); + items.add(m_color, m_above); + items.add(m_color, m_below); +} + +// On the badge the icon is 19% white and 48% cyan, with nothing bright +// left of it. A bright cutscene flash passes the icon's bounds too, but is +// 70%+ white and 39%+ cyan left of the icon as well. +// Only the Chinese badge is checked; with a shorter text (English +// "Saving...") a right-aligned badge moves the icon right, so the icon is +// looked for from where it is in Chinese to 240 px right of it. +// Glowing scenery (a lit blade, a bright sky) can match the icon somewhere +// in that span, but glows on past it: the strips 14 px above and below the +// icon are 22%+ white or cyan there and empty around the badge; they must +// stay under 5%. +// The icon's glow is dim over a dark or lamp-lit scene (about 28, 70, 85 +// on 02.04's check-in wall, frame 0075 of a +// run in Japanese, icon at dx 36), so the icon's glow is dim_cyan_share(): +// of the 3569 PLZA captures on the console it finds 11 more +// badges, every one a real badge, and no new false one. +// Both cyan tests take only blue-leaning pixels: a building banner's mint +// (about 96, 203, 147) in the strips hid the badge on 01.01 in English, +// and a green sign's white text with its glow matched the icon. Of 6096 +// console captures and 1173 PokemonLZA test frames this finds the English +// badge, drops 3 green-sign matches, and changes nothing else. +bool AutosaveDetector::detect(const ImageViewRGB32& screen){ + for (double dx = 0; dx <= 240; dx += 4){ + if (autosave_icon_at(screen, m_icon, {m_left, m_above, m_below}, dx)){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h new file mode 100644 index 0000000000..555bb4c1bc --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryAutosaveDetector.h @@ -0,0 +1,46 @@ +/* PokemonLZA AutoStory - Autosave detector + * + * From: https://github.com/PokemonAutomation/ + * + * The autosave badge, a pencil and "Saving...", which shows for 1-3 s + * after the game autosaves. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryAutosaveDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The autosave badge at the top right: a white pencil with a cyan glow at +// x 1603-1630, y 73-104 on 1920x1080 (the glow dim over a dark scene), +// and nothing white or cyan just left of it, above it or below it. Only +// the icon is read, so any language matches. +class AutosaveDetector : public StaticScreenDetector{ +public: + explicit AutosaveDetector(Color color = COLOR_YELLOW); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_icon; + ImageFloatBox m_left; + ImageFloatBox m_above; + ImageFloatBox m_below; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.cpp new file mode 100644 index 0000000000..8472389f5a --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.cpp @@ -0,0 +1,76 @@ +/* PokemonLZA AutoStory - Battle markers detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryBattleMarkersDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr double SLOT_CENTRES_X[] = {405, 432, 459, 486, 513}; +constexpr double SLOT_WIDTH = 16; +constexpr double SLOT_HEIGHT = 12; +constexpr double SLOT_TOP = 800; + +constexpr double TRIANGLE_SHARE = 0.2; +constexpr double NO_TRIANGLE_SHARE = 0.05; +constexpr double DOT_SHARE_MIN = 0.06; +constexpr double DOT_SHARE_MAX = 0.5; +constexpr size_t DOTS_MIN = 3; + +// The triangle's blue: red 70 or less, green 150 to 210, blue 225 or more. +double triangle_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() <= 70 && pixel.green() >= 150 && pixel.green() <= 210 && pixel.blue() >= 225; + }); +} + +} // namespace + + +BattleMarkersDetector::BattleMarkersDetector(Color color) + : m_color(color) +{ + for (double centre : SLOT_CENTRES_X){ + m_slots.emplace_back( + (centre - SLOT_WIDTH / 2) / 1920, SLOT_TOP / 1080, SLOT_WIDTH / 1920, SLOT_HEIGHT / 1080 + ); + } +} + +void BattleMarkersDetector::make_overlays(VideoOverlaySet& items) const{ + for (const ImageFloatBox& slot : m_slots){ + items.add(m_color, slot); + } +} + +bool BattleMarkersDetector::detect(const ImageViewRGB32& screen){ + size_t triangles = 0; + size_t dots = 0; + for (const ImageFloatBox& slot : m_slots){ + const double blue = triangle_share(screen, slot); + if (blue >= TRIANGLE_SHARE){ + triangles++; + continue; + } + const double white = white_share(screen, slot); + if (blue < NO_TRIANGLE_SHARE && white >= DOT_SHARE_MIN && white <= DOT_SHARE_MAX){ + dots++; + } + } + return triangles == 1 && dots >= DOTS_MIN; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.h new file mode 100644 index 0000000000..9db3602acd --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattleMarkersDetector.h @@ -0,0 +1,49 @@ +/* PokemonLZA AutoStory - Battle markers detector + * + * From: https://github.com/PokemonAutomation/ + * + * The row of five markers right of the lead's panel's gender icon in a + * battle: one blue triangle and white dots. The panel itself + * (BattlePanelDetector) shows all through a wild zone, in battle or not + * (console B, 2026-11-16); the markers show only in a battle. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryBattleMarkersDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryBattleMarkersDetector_H + +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The markers' five slots, 16 x 12 on 1920x1080, centred at x 405, 432, +// 459, 486 and 513, y 806. A battle: one slot the triangle's blue (about +// 15, 180, 255) over 20% of its pixels, and 3 or more others a dot's +// white over 6% and under 50% (a white scene fills a slot) with no blue. +// Measured on Taunie's battle (03.02b) and a wild Mareep's (03.08b), +// neither locked on: triangle 0.39 and 0.44, dots 0.18 to 0.57; no +// other of 1403 PLZA frames (CommandLineTests and lab-B's) has them. +class BattleMarkersDetector : public StaticScreenDetector{ +public: + explicit BattleMarkersDetector(Color color = COLOR_BLUE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + std::vector m_slots; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.cpp new file mode 100644 index 0000000000..e5020e54f8 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.cpp @@ -0,0 +1,56 @@ +/* PokemonLZA AutoStory - Battle panel detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryBattlePanelDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The shares of the badge's and the bar's pixels that make the panel. +constexpr double BADGE_SHARE = 0.5; +constexpr double BAR_SHARE = 0.8; + +// An HP bar pixel: its fill (red, yellow or green: channels 90 or more +// apart), or its grey trough (channels within 25, mean 60 to 140). +bool hp_bar_pixel(Color color){ + const int r = color.red(); + const int g = color.green(); + const int b = color.blue(); + const int spread = std::max({r, g, b}) - std::min({r, g, b}); + const int mean = (r + g + b) / 3; + return spread >= 90 || (spread <= 25 && mean >= 60 && mean <= 140); +} + +} // namespace + + +BattlePanelDetector::BattlePanelDetector(Color color) + : m_color(color) + , m_badge(240.0 / 1920, 799.0 / 1080, 80.0 / 1920, 19.0 / 1080) + , m_bar(236.0 / 1920, 882.0 / 1080, 204.0 / 1920, 20.0 / 1080) +{} + +void BattlePanelDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_badge); + items.add(m_color, m_bar); +} + +bool BattlePanelDetector::detect(const ImageViewRGB32& screen){ + return sky_blue_share(screen, m_badge) >= BADGE_SHARE && share_of(screen, m_bar, hp_bar_pixel) >= BAR_SHARE; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.h new file mode 100644 index 0000000000..13fb048562 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryBattlePanelDetector.h @@ -0,0 +1,47 @@ +/* PokemonLZA AutoStory - Battle panel detector + * + * From: https://github.com/PokemonAutomation/ + * + * The player's Pokemon's panel at the bottom left in a battle: its level + * badge ("L. 7") and HP bar. It shows the whole battle, locked on or not; + * the move list and its hints show only while the player is locked on. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryBattlePanelDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryBattlePanelDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The panel's level badge, sky blue (about 140, 210, 235) at x 240-320, +// y 799-818 on 1920x1080, over half its pixels; and its HP bar at x +// 236-440, y 882-902 (left of the "4/24"), over 80% its HP color or its +// grey trough (about 95, 95, 95). Taunie's battle not locked on (03.02b, +// console B 2026-11-16): badge 0.78, bar 0.95; overworld frames: badge +// 0.18 at most, bar 0.20 at most. +class BattlePanelDetector : public StaticScreenDetector{ +public: + explicit BattlePanelDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_badge; + ImageFloatBox m_bar; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp new file mode 100644 index 0000000000..b11af64f96 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.cpp @@ -0,0 +1,98 @@ +/* PokemonLZA AutoStory - Confirm Choices reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "PokemonLZA_AutoStoryHighlightedRow.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +double brightness(const ImageStats& stats){ + return stats.average.sum() / 3; +} + +// The right ends of the "Confirm Choices" rows, right of their text: the +// boxes span x 144 to 742 at 1920 x 1080, the two choices from y 578 and +// 662 (68 high), "Ready!" from y 790 (112 high). Each strip is 100 wide +// from x 620. +ImageFloatBox confirm_row_strip(size_t row){ + switch (row){ + case 0: + return ImageFloatBox(620.0 / 1920, 592.0 / 1080, 100.0 / 1920, 40.0 / 1080); + case 1: + return ImageFloatBox(620.0 / 1920, 676.0 / 1080, 100.0 / 1920, 40.0 / 1080); + default: + return ImageFloatBox(620.0 / 1920, 820.0 / 1080, 100.0 / 1920, 60.0 / 1080); + } +} + +// A "Confirm Choices" row's strip: the highlighted row white, flat; the +// others slate. The highlight's glow pulses over the row next to it: on +// the console's frames in ten languages that row's spread went +// up to 20. +constexpr SlateRows CONFIRM_ROWS_SLATE{ + .min_red = 20, .max_red = 55, + .min_green = 40, .max_green = 80, + .min_blue = 65, .max_blue = 115, + .min_blue_over_red = 25, + .max_spread = 30, +}; + +// The dark blue, dotted background round the rows: between the choices and +// "Ready!", and right of all three. The highlight's blue glow reaches 40 +// pixels out and fills the gap under "Change looks" when that row is +// highlighted: (12-13, 56-59, 102-106), spread up to 51, against the plain +// background's (10-11, 29-37, 63-75), spread up to 31 (the console, ten +// languages). Both are dark and blue with little red; the +// intro's other screens are not. +bool confirm_background_dark(const ImageViewRGB32& frame){ + const ImageFloatBox boxes[] = { + ImageFloatBox(620.0 / 1920, 742.0 / 1080, 100.0 / 1920, 40.0 / 1080), + ImageFloatBox(770.0 / 1920, 580.0 / 1080, 30.0 / 1920, 320.0 / 1080), + }; + for (const ImageFloatBox& box : boxes){ + const ImageStats stats = image_stats(extract_box_reference(frame, box)); + if (stats.count == 0 || brightness(stats) >= 90 || stats.stddev.sum() > 80){ + return false; + } + if (stats.average.r > 30 || stats.average.b < stats.average.r + 40){ + return false; + } + } + return true; +} + +} // namespace + + +// Measured on the console, "Confirm Choices" as it opens: the +// highlighted row 255 white, flat; the others slate, (35-36, 52-61, +// 82-95), spread under 10; the background (10, 29-31, 64-66), spread under +// 25. The intro's language list and look picker fall on the background's +// strips, so neither reads as this screen. +std::optional read_confirm_choices_cursor(const ImageViewRGB32& frame){ + if (!frame || frame.width() < 160 || frame.height() < 90){ + return std::nullopt; + } + if (!confirm_background_dark(frame)){ + return std::nullopt; + } + const std::array strips{confirm_row_strip(0), confirm_row_strip(1), confirm_row_strip(2)}; + return highlighted_row(frame, strips, CONFIRM_ROWS_SLATE); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h new file mode 100644 index 0000000000..7958e110a0 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader.h @@ -0,0 +1,37 @@ +/* PokemonLZA AutoStory - Confirm Choices reader + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's "Confirm Choices" screen, where checkpoint 01.00b starts: + * the PROFILE screen after the name keyboard, with its rows "Change hair + * color", "Change look" and "Ready!". + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_H +#define PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// "Confirm Choices" has three rows: "Change hair color" (0, highlighted as +// it opens), "Change look" and "Ready!", the last. +constexpr size_t CONFIRM_CHOICES_READY_ROW = 2; + +// On "Confirm Choices", the highlighted row, 0 to +// CONFIRM_CHOICES_READY_ROW: the one white box of the three, the others +// slate. Nullopt for any other frame, an empty one included. +std::optional read_confirm_choices_cursor(const ImageViewRGB32& frame); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp new file mode 100644 index 0000000000..b5f8f22f92 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.cpp @@ -0,0 +1,259 @@ +/* PokemonLZA AutoStory - Confirm Choices reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Tests/ImageUnitTest.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr uint32_t BACKGROUND = 0xff0a1d40; +constexpr uint32_t SLATE = 0xff23344f; +constexpr uint32_t WHITE = 0xffffffff; + +// "Confirm Choices" drawn as the console shows it (1920 x 1080): dark blue, +// three slate rows x 144 to 742, the two choices from y 578 and 662 (68 +// high) and "Ready!" from y 790 (112 high), the rows in `highlighted` +// white. +ImageRGB32 confirm_choices_frame(std::initializer_list highlighted){ + ImageRGB32 image(1920, 1080); + image.fill(BACKGROUND); + const size_t tops[] = {578, 662, 790}; + const size_t heights[] = {68, 68, 112}; + for (size_t row = 0; row <= CONFIRM_CHOICES_READY_ROW; row++){ + bool lit = false; + for (const size_t pick : highlighted){ + lit = lit || pick == row; + } + for (size_t y = tops[row]; y < tops[row] + heights[row]; y++){ + for (size_t x = 144; x < 742; x++){ + image.pixel(x, y) = lit ? WHITE : SLATE; + } + } + } + return image; +} + +ImageRGB32 flat_frame(uint32_t color){ + ImageRGB32 image(1920, 1080); + image.fill(color); + return image; +} + +std::string describe_cursor(std::optional cursor){ + return cursor ? std::format("row {}", *cursor) : std::string("no Confirm Choices"); +} + + +struct SyntheticCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + std::optional cursor; // The row read; none: not Confirm Choices. + }; + Expectation expectation; +}; + +const SyntheticCase SYNTHETIC_CASES[] = { + {.name = "row 0 highlighted", .input = {.frame = []{ return confirm_choices_frame({0}); }}, .expectation = {.cursor = 0}}, + {.name = "row 1 highlighted", .input = {.frame = []{ return confirm_choices_frame({1}); }}, .expectation = {.cursor = 1}}, + { + .name = "Ready! highlighted", + .input = {.frame = []{ return confirm_choices_frame({CONFIRM_CHOICES_READY_ROW}); }}, + .expectation = {.cursor = CONFIRM_CHOICES_READY_ROW}, + }, + {.name = "no row highlighted", .input = {.frame = []{ return confirm_choices_frame({}); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "two rows highlighted", .input = {.frame = []{ return confirm_choices_frame({0, 1}); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat black", .input = {.frame = []{ return flat_frame(0xff000000); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat white", .input = {.frame = []{ return flat_frame(WHITE); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat background", .input = {.frame = []{ return flat_frame(BACKGROUND); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "flat slate", .input = {.frame = []{ return flat_frame(SLATE); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "an empty frame", .input = {.frame = []{ return ImageRGB32(); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "a 1 x 1 frame", .input = {.frame = []{ return ImageRGB32(1, 1); }}, .expectation = {.cursor = std::nullopt}}, + {.name = "a 40 x 20 frame", .input = {.frame = []{ return ImageRGB32(40, 20); }}, .expectation = {.cursor = std::nullopt}}, +}; + +class Test_ConfirmChoicesSynthetic : public UnitTest{ +public: + explicit Test_ConfirmChoicesSynthetic(const SyntheticCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Confirm Choices reader: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional cursor = read_confirm_choices_cursor(m_test.input.frame()); + if (cursor != m_test.expectation.cursor){ + return std::format("Expected {}, read {}.", describe_cursor(m_test.expectation.cursor), describe_cursor(cursor)); + } + return true; + } + +private: + const SyntheticCase& m_test; +}; + + +// The folder named by the environment variable `variable`; none if it is +// unset or not a folder. +std::optional env_folder(std::string_view variable){ + const auto* value = std::getenv(std::string(variable).c_str()); + if (value == nullptr){ + return std::nullopt; + } + std::string folder(value); + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return std::nullopt; + } + return folder; +} + +// 01.00b's start check on the console's intro screens, from the folder in +// PLZA_INTRO_SCREENS: it accepts Confirm Choices as it opens, and refuses +// every other intro screen. A file the folder lacks is skipped. +struct CaptureCase{ + std::string name; + struct Input{ + std::string_view file; + }; + Input input; + struct Expectation{ + std::optional cursor; + }; + Expectation expectation; +}; + +const CaptureCase CAPTURE_CASES[] = { + {.name = "Confirm Choices as it opens", .input = {.file = "confirm-choices.png"}, .expectation = {.cursor = 0}}, + {.name = "HOME", .input = {.file = "home.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "a game in front", .input = {.file = "in-game.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the language list", .input = {.file = "language-list.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the look picker", .input = {.file = "look-picker.png"}, .expectation = {.cursor = std::nullopt}}, + {.name = "the name keyboard", .input = {.file = "keyboard.png"}, .expectation = {.cursor = std::nullopt}}, +}; + +class Test_ConfirmChoicesCapture : public UnitTest{ +public: + explicit Test_ConfirmChoicesCapture(const CaptureCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - Confirm Choices reader: {} (PLZA_INTRO_SCREENS)", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::optional folder = env_folder("PLZA_INTRO_SCREENS"); + if (!folder){ + return UnitTestResult(UnitTestResult::SKIPPED, "PLZA_INTRO_SCREENS is not a folder."); + } + const std::string path = std::format("{}/{}", *folder, m_test.input.file); + if (!Filesystem::exists(Filesystem::Path(path))){ + return UnitTestResult(UnitTestResult::SKIPPED, std::format("{} is not in the folder.", m_test.input.file)); + } + const std::optional cursor = read_confirm_choices_cursor(ImageRGB32(path)); + if (cursor != m_test.expectation.cursor){ + return std::format( + "{}: expected {}, read {}.", m_test.input.file, describe_cursor(m_test.expectation.cursor), describe_cursor(cursor) + ); + } + return true; + } + +private: + const CaptureCase& m_test; +}; + + +// "Confirm Choices" on the console in each game language, one frame per +// highlighted row: PokemonLZA/AutoStoryConfirmChoices/ +// _.png, row 0 to CONFIRM_CHOICES_READY_ROW. Each frame +// reads as its row. +struct LanguageFramesCase{ + std::string name; + struct Input{ + std::string_view language; + }; + Input input; + struct Expectation{ + }; + Expectation expectation; +}; + +const LanguageFramesCase LANGUAGE_FRAMES_CASES[] = { + {.name = "English", .input = {.language = "English"}, .expectation = {}}, + {.name = "Spanish", .input = {.language = "Spanish"}, .expectation = {}}, + {.name = "Latin American Spanish", .input = {.language = "SpanishLatAm"}, .expectation = {}}, + {.name = "French", .input = {.language = "French"}, .expectation = {}}, + {.name = "German", .input = {.language = "German"}, .expectation = {}}, + {.name = "Italian", .input = {.language = "Italian"}, .expectation = {}}, + {.name = "Japanese", .input = {.language = "Japanese"}, .expectation = {}}, + {.name = "Korean", .input = {.language = "Korean"}, .expectation = {}}, + {.name = "Simplified Chinese", .input = {.language = "ChineseSimplified"}, .expectation = {}}, + {.name = "Traditional Chinese", .input = {.language = "ChineseTraditional"}, .expectation = {}}, +}; + +std::vector language_frame_paths(std::string_view language){ + std::vector paths; + for (size_t row = 0; row <= CONFIRM_CHOICES_READY_ROW; row++){ + paths.push_back(std::format("PokemonLZA/AutoStoryConfirmChoices/{}_{}.png", language, row)); + } + return paths; +} + +class Test_ConfirmChoicesFrames : public ImageUnitTest{ +public: + explicit Test_ConfirmChoicesFrames(const LanguageFramesCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - Confirm Choices - {}", test.name), + language_frame_paths(test.input.language) + ) + {} + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, const std::vector& images + ) const override{ + for (size_t row = 0; row < images.size(); row++){ + const std::optional cursor = read_confirm_choices_cursor(images[row]); + if (cursor != row){ + return std::format("{}: expected row {}, read {}.", image_paths()[row], row, describe_cursor(cursor)); + } + } + return true; + } +}; + +} // namespace + + +void add_tests_AutoStoryConfirmChoicesReader(UnitTestDatabase& database){ + for (const SyntheticCase& test : SYNTHETIC_CASES){ + database.add(test); + } + for (const CaptureCase& test : CAPTURE_CASES){ + database.add(test); + } + for (const LanguageFramesCase& test : LANGUAGE_FRAMES_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h new file mode 100644 index 0000000000..72f0b5e74f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryConfirmChoicesReader_Tests.h @@ -0,0 +1,28 @@ +/* PokemonLZA AutoStory - Confirm Choices reader Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryConfirmChoicesReader_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the Confirm Choices reader's tests: synthetic frames always; +// the console's frames in each language from the test images +// (PokemonLZA/AutoStoryConfirmChoices/), and the other intro screens from +// the folder in PLZA_INTRO_SCREENS, skipped when unset. +void add_tests_AutoStoryConfirmChoicesReader(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp new file mode 100644 index 0000000000..cffd64f865 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.cpp @@ -0,0 +1,69 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the card at x 946-971, +// y 868-901 with its spine at x 974-976; the "?" at x 953-965, y 876-894. +ControlsHintDetector::ControlsHintDetector(Color color) + : m_color(color) + , m_card(947.0 / 1920, 872.0 / 1080, 24.0 / 1920, 28.0 / 1080) + , m_glyph(955.0 / 1920, 879.0 / 1080, 8.0 / 1920, 7.0 / 1080) + , m_around{ + {928.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {980.0 / 1920, 872.0 / 1080, 14.0 / 1920, 28.0 / 1080}, + {945.0 / 1920, 856.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + {945.0 / 1920, 906.0 / 1080, 30.0 / 1920, 8.0 / 1080}, + } +{} + +void ControlsHintDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_card); + for (const ImageFloatBox& box : m_around){ + items.add(m_color, box); + } +} + +// With the hint up the card is 88%+ sky blue, the "?" at most 48%, and +// nothing around it is. A sky behind the card comes closest (card 76%), +// but then the "?" and the area around it are blue too (100%, 54%). +// The card sits higher the more rows the hint has: with three rows ("Move", +// "Control Camera", "Reset Camera") it is 16-24 px above the two-row one +// the boxes were measured on (English and Italian 01.01, console B +// 2026-11-15), so it is looked for from 60 px above to 20 px below. +bool ControlsHintDetector::detect(const ImageViewRGB32& screen){ + for (double dy = -60; dy <= 20; dy += 2){ + if (card_at(screen, dy)){ + return true; + } + } + return false; +} + +bool ControlsHintDetector::card_at(const ImageViewRGB32& screen, double dy) const{ + if (sky_blue_share(screen, shifted(m_card, 0, dy)) < 0.80 || sky_blue_share(screen, shifted(m_glyph, 0, dy)) > 0.70){ + return false; + } + for (const ImageFloatBox& box : m_around){ + if (sky_blue_share(screen, shifted(box, 0, dy)) > 0.10){ + return false; + } + } + return true; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h new file mode 100644 index 0000000000..fc990cf37e --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryControlsHintDetector.h @@ -0,0 +1,52 @@ +/* PokemonLZA AutoStory - Controls hint detector + * + * From: https://github.com/PokemonAutomation/ + * + * The tutorial's controls hint under the player, a "?" card over + * "(L) Move" and "(R) Control Camera", which shows for ~10 s when the + * player gets control. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryControlsHintDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The controls hint the tutorial shows under the player when they get +// control: its "?" card, sky blue with a dark "?", at x 947-971, y 872-900 +// on 1920x1080 under a two-row hint (higher under more rows), centred, +// with no sky blue around it. Only the card is +// read: the rows under it ("(L) Move", "(R) Control Camera") are centred +// too, so their icons move with the text's length. +class ControlsHintDetector : public StaticScreenDetector{ +public: + explicit ControlsHintDetector(Color color = COLOR_BLUE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + // Whether the card shows `dy` pixels (of 1080) below where the boxes + // are. + bool card_at(const ImageViewRGB32& screen, double dy) const; + + Color m_color; + ImageFloatBox m_card; + ImageFloatBox m_glyph; + ImageFloatBox m_around[4]; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp new file mode 100644 index 0000000000..b87cac3fa7 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.cpp @@ -0,0 +1,728 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Synthetic frames, and console frames (1920x1080 JPEG or PNG) from + * CommandLineTests/PokemonLZA/AutoStoryDetectors/, one folder per scene + * sequence. + * + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_PartyBarDetector.h" +#include "PokemonLZA_AutoStoryAdvanceArrowDetector.h" +#include "PokemonLZA_AutoStoryAutosaveDetector.h" +#include "PokemonLZA_AutoStoryControlsHintDetector.h" +#include "PokemonLZA_AutoStoryDetectors_Tests.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" +#include "PokemonLZA_AutoStoryTutorialCardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr size_t WIDTH = 1920; +constexpr size_t HEIGHT = 1080; + + +// What each detector reads on a frame: true or false, or none when a test +// does not check that detector. +struct DetectorReads{ + std::optional platform = {}; + std::optional autosave = {}; + std::optional controls_hint = {}; + std::optional advance_arrow = {}; + std::optional mission_banner = {}; + std::optional mission_complete = {}; + std::optional party_bar = {}; +}; + +struct DetectorField{ + std::string_view name; + std::optional DetectorReads::* read; +}; +const DetectorField DETECTOR_FIELDS[] = { + {"platform", &DetectorReads::platform}, + {"autosave", &DetectorReads::autosave}, + {"controls hint", &DetectorReads::controls_hint}, + {"advance arrow", &DetectorReads::advance_arrow}, + {"mission banner", &DetectorReads::mission_banner}, + {"mission complete", &DetectorReads::mission_complete}, + {"party bar", &DetectorReads::party_bar}, +}; + +// `reads` with every detector checked: one it leaves unset must not fire. +DetectorReads every_detector(DetectorReads reads){ + for (const DetectorField& field : DETECTOR_FIELDS){ + reads.*field.read = (reads.*field.read).value_or(false); + } + return reads; +} + +// The detectors that fire in `got`, for the log: ", platform, autosave". +std::string firing(const DetectorReads& got){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + if ((got.*field.read).value_or(false)){ + text += std::format(", {}", field.name); + } + } + return text; +} + +// The detectors `want` checks that `got` reads otherwise: " platform +// (expected yes)"; empty when all agree. +std::string mismatches(const DetectorReads& got, const DetectorReads& want){ + std::string text; + for (const DetectorField& field : DETECTOR_FIELDS){ + const std::optional& expected = want.*field.read; + if (expected && got.*field.read != expected){ + text += std::format(" {} (expected {})", field.name, *expected ? "yes" : "no"); + } + } + return text; +} + +// Every detector of this module, one of each. +struct StoryDetectors{ + PlatformArrivalDetector platform; + AutosaveDetector autosave; + ControlsHintDetector hint; + AdvanceArrowDetector arrow; + MainMissionBannerDetector banner; + MissionCompleteDetector complete; + PartyBarDetector party_bar; + + DetectorReads read(const ImageViewRGB32& frame){ + return DetectorReads{ + .platform = platform.detect(frame), .autosave = autosave.detect(frame), + .controls_hint = hint.detect(frame), .advance_arrow = arrow.detect(frame), + .mission_banner = banner.detect(frame), .mission_complete = complete.detect(frame), + .party_bar = party_bar.detect(frame), + }; + } +}; + + +// Synthetic frames. + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +ImageRGB32 solid_frame(uint32_t color){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(color); + return image; +} + +// The controls hint's card drawn on black: sky blue with a dark block for +// the top of its "?". +ImageRGB32 controls_hint_frame(){ + ImageRGB32 image = solid_frame(0xff000000); + fill_box(/*min_x=*/946, /*min_y=*/868, /*max_x=*/972, /*max_y=*/902, /*color=*/0xff7dd2fa, image); + fill_box(/*min_x=*/956, /*min_y=*/879, /*max_x=*/962, /*max_y=*/886, /*color=*/0xff1c7db0, image); + return image; +} + +// The platform reference with the player's area (x 780-1200, y 330-1080), +// which is in no region, painted white. +ImageRGB32 platform_with_player_frame(){ + ImageRGB32 image = platform_reference_image(WIDTH, HEIGHT); + fill_box(/*min_x=*/780, /*min_y=*/330, /*max_x=*/1200, HEIGHT, /*color=*/0xffffffff, image); + return image; +} + +// The advance chevron drawn in `ink` on `background`: a V 32x19 at +// x 1400, y 1004, each arm 18% of the width across. +ImageRGB32 chevron_frame(uint32_t background, uint32_t ink){ + ImageRGB32 image = solid_frame(background); + for (size_t y = 0; y < 19; y++){ + const double v = (double(y) + 0.5) / 19; + for (size_t x = 0; x < 32; x++){ + const double u = (double(x) + 0.5) / 32; + if (std::abs(std::abs(u - 0.5) - (0.40 * (1 - v) + 0.03)) <= 0.09){ + image.pixel(1400 + x, 1004 + y) = ink; + } + } + } + return image; +} + +// A solid white block the chevron's size on the ad's light blue. +ImageRGB32 chevron_sized_block_frame(){ + ImageRGB32 image = solid_frame(0xffc1ecf0); + fill_box(/*min_x=*/1400, /*min_y=*/1004, /*max_x=*/1432, /*max_y=*/1023, /*color=*/0xffffffff, image); + return image; +} + +// The autosave badge's pencil, 48 px right of the Chinese badge's (where +// the shorter Japanese text puts it), over `background`: a white stroke 4 px +// wide in a `glow` band 12 px wide. +ImageRGB32 autosave_badge_frame(uint32_t background, uint32_t glow){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(background); + fill_box(/*min_x=*/1655, /*min_y=*/73, /*max_x=*/1667, /*max_y=*/104, glow, image); + fill_box(/*min_x=*/1659, /*min_y=*/73, /*max_x=*/1663, /*max_y=*/104, /*color=*/0xffffffff, image); + return image; +} + +// autosave_badge_frame() with a bright glow over a dark scene, and bands +// of a building banner's mint green (about 96, 203, 147) 4 px high across +// the strips above and below the icon, as on 01.01 in English. +ImageRGB32 autosave_badge_by_mint_banner_frame(){ + ImageRGB32 image = autosave_badge_frame(/*background=*/0xff101418, /*glow=*/0xff28c8e6); + fill_box(/*min_x=*/1600, /*min_y=*/60, /*max_x=*/1720, /*max_y=*/64, /*color=*/0xff60cb93, image); + fill_box(/*min_x=*/1600, /*min_y=*/112, /*max_x=*/1720, /*max_y=*/116, /*color=*/0xff60cb93, image); + return image; +} + +// How mission_banner_frame() draws the icon. +enum class BannerIcon{ + NONE, + DRAWN, +}; + +// The mission banner drawn on a grey scene: the yellow bar, and with +// BannerIcon::DRAWN the icon's white hexagon (as a block) with the orange +// stroke, `dx` right of where the detector expects it. +ImageRGB32 mission_banner_frame(size_t dx, BannerIcon icon){ + ImageRGB32 image = solid_frame(0xff606060); + fill_box(/*min_x=*/345, /*min_y=*/410, /*max_x=*/1590, /*max_y=*/600, /*color=*/0xfffad228, image); + if (icon == BannerIcon::DRAWN){ + fill_box(700 + dx, /*min_y=*/327, 738 + dx, /*max_y=*/370, /*color=*/0xffffffff, image); + fill_box(716 + dx, /*min_y=*/336, 720 + dx, /*max_y=*/357, /*color=*/0xfff07020, image); + } + return image; +} + +// The white title card after the plaza's white fade ("POKEMON LEGENDS: +// Z-A" centered in black, x 450-1450, y 520-560): 01.01's end, where a +// rerun from 01.01 finds the game before it reloads. Each letter a black +// block 30 px wide, 20 px apart. +ImageRGB32 title_card_frame(){ + ImageRGB32 image = solid_frame(0xffffffff); + for (size_t x = 450; x + 30 <= 1450; x += 50){ + fill_box(x, /*min_y=*/520, x + 30, /*max_y=*/560, /*color=*/0xff000000, image); + } + return image; +} + +// The mission complete card drawn on a grey scene: the yellow bar, and +// "COMPLETE" as green strokes (three columns in four) above it. +ImageRGB32 mission_complete_frame(){ + ImageRGB32 image = solid_frame(0xff606060); + fill_box(/*min_x=*/345, /*min_y=*/390, /*max_x=*/1590, /*max_y=*/585, /*color=*/0xfffad228, image); + for (size_t x = 615; x < 1300; x += 4){ + fill_box(x, /*min_y=*/275, x + 3, /*max_y=*/345, /*color=*/0xffa5f07d, image); + } + return image; +} + +// mission_complete_frame() with green right above the word: a green +// scene, not the card. +ImageRGB32 mission_complete_green_above_frame(){ + ImageRGB32 image = mission_complete_frame(); + fill_box(/*min_x=*/615, /*min_y=*/245, /*max_x=*/1300, /*max_y=*/270, /*color=*/0xffa5f07d, image); + return image; +} + +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t BLACK = 0xff000000; +constexpr uint32_t GRAY = 0xff808080; +constexpr uint32_t AD_LIGHT_BLUE = 0xffc1ecf0; +constexpr uint32_t BOX_WHITE = 0xfffcfcf7; +constexpr uint32_t BOX_TEXT_DARK = 0xff323648; +constexpr uint32_t HINT_SKY_BLUE = 0xff7dd2fa; +constexpr uint32_t BANNER_YELLOW = 0xfffad228; +constexpr uint32_t COMPLETE_GREEN = 0xffa5f07d; + +struct SyntheticFrameCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + DetectorReads reads = {}; + }; + Expectation expectation; +}; + +const SyntheticFrameCase SYNTHETIC_FRAME_CASES[] = { + { + .name = "an empty image", + .input = {.frame = []{ return ImageRGB32(); }}, + .expectation = {.reads = every_detector({})}, + }, + { + .name = "a black frame", + .input = {.frame = []{ return solid_frame(BLACK); }}, + .expectation = {.reads = {.autosave = false, .advance_arrow = false}}, + }, + { + .name = "a white frame", + .input = {.frame = []{ return solid_frame(WHITE); }}, + .expectation = {.reads = {.platform = false, .autosave = false, .advance_arrow = false}}, + }, + { + .name = "the white title card is not the platform", + .input = {.frame = title_card_frame}, + .expectation = {.reads = {.platform = false}}, + }, + { + .name = "a gray frame", + .input = {.frame = []{ return solid_frame(GRAY); }}, + .expectation = {.reads = {.platform = false}}, + }, + { + .name = "the platform reference", + .input = {.frame = []{ return platform_reference_image(WIDTH, HEIGHT); }}, + .expectation = {.reads = {.platform = true}}, + }, + { + .name = "the platform reference with the player's area painted", + .input = {.frame = platform_with_player_frame}, + .expectation = {.reads = {.platform = true}}, + }, + // The autosave badge's glow is bright over a dark sky and dim over a + // lamp-lit wall (about 28, 70, 85: 02.04's check-in, + // frame 0075, which the badge detector missed). + { + .name = "the autosave badge with a bright glow", + .input = {.frame = []{ return autosave_badge_frame(0xff101418, 0xff28c8e6); }}, + .expectation = {.reads = {.autosave = true}}, + }, + { + .name = "the autosave badge with a dim glow over a dark wall", + .input = {.frame = []{ return autosave_badge_frame(0xff252722, 0xff1c4655); }}, + .expectation = {.reads = {.autosave = true}}, + }, + { + .name = "the autosave badge's white stroke with no glow", + .input = {.frame = []{ return autosave_badge_frame(0xff252722, 0xff252722); }}, + .expectation = {.reads = {.autosave = false}}, + }, + // Mint and sign green have green over blue; the badge's glow does not. + { + .name = "the autosave badge with mint banner bands above and below", + .input = {.frame = autosave_badge_by_mint_banner_frame}, + .expectation = {.reads = {.autosave = true}}, + }, + { + .name = "a white stroke with a green sign's glow", + .input = {.frame = []{ return autosave_badge_frame(0xff252722, 0xff3cf06e); }}, + .expectation = {.reads = {.autosave = false}}, + }, + { + .name = "the controls hint card", + .input = {.frame = controls_hint_frame}, + .expectation = {.reads = {.controls_hint = true}}, + }, + { + .name = "a frame all in the hint card's blue (a sky)", + .input = {.frame = []{ return solid_frame(HINT_SKY_BLUE); }}, + .expectation = {.reads = {.controls_hint = false}}, + }, + { + .name = "a white chevron on the ad's light blue", + .input = {.frame = []{ return chevron_frame(AD_LIGHT_BLUE, WHITE); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a dark chevron on a white box", + .input = {.frame = []{ return chevron_frame(BOX_WHITE, BOX_TEXT_DARK); }}, + .expectation = {.reads = {.advance_arrow = true}}, + }, + { + .name = "a solid block the chevron's size", + .input = {.frame = chevron_sized_block_frame}, + .expectation = {.reads = {.advance_arrow = false}}, + }, + { + .name = "the mission banner", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true}}, + }, + { + .name = "the mission banner with its icon 40 px right", + .input = {.frame = []{ return mission_banner_frame(40, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_banner = true}}, + }, + { + .name = "the mission banner's bar without the icon", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::NONE); }}, + .expectation = {.reads = {.mission_banner = false}}, + }, + { + .name = "a frame all in the bar's yellow", + .input = {.frame = []{ return solid_frame(BANNER_YELLOW); }}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = false}}, + }, + { + .name = "the mission complete card", + .input = {.frame = mission_complete_frame}, + .expectation = {.reads = {.mission_banner = false, .mission_complete = true}}, + }, + { + .name = "a frame all in the card's green", + .input = {.frame = []{ return solid_frame(COMPLETE_GREEN); }}, + .expectation = {.reads = {.mission_complete = false}}, + }, + { + .name = "the card's word with green right above it", + .input = {.frame = mission_complete_green_above_frame}, + .expectation = {.reads = {.mission_complete = false}}, + }, + { + .name = "the mission banner is not the complete card", + .input = {.frame = []{ return mission_banner_frame(0, BannerIcon::DRAWN); }}, + .expectation = {.reads = {.mission_complete = false}}, + }, +}; + +UnitTestResult check_synthetic_frame(const SyntheticFrameCase& test, Logger& logger){ + StoryDetectors detectors; + const DetectorReads got = detectors.read(test.input.frame()); + logger.log(std::format("{}: {}{}", test.name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, test.expectation.reads); + if (!wrong.empty()){ + return std::format("Wrong on {}:{}", test.name, wrong); + } + return true; +} + + +// Console frames, from folders in CommandLineTests. + +// The number of `digits` digits at `offset` in a frame's file name; none +// if they are not all digits. +std::optional frame_number(std::string_view name, size_t offset, size_t digits){ + if (name.size() < offset + digits){ + return std::nullopt; + } + const std::string_view text = name.substr(offset, digits); + int number = 0; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), number); + if (error != std::errc() || end != text.data() + text.size()){ + return std::nullopt; + } + return number; +} + +// What a frame shows, from the readings in its name +// ("0001-nothing+platform+controls_hint.jpg"); a reading of a detector +// not in this module is ignored. +DetectorReads as_named(std::string_view name){ + const auto has = [name](std::string_view reading){ + return name.find(reading) != std::string_view::npos; + }; + return DetectorReads{ + .platform = has("+platform"), .autosave = has("+autosave"), + .controls_hint = has("+controls_hint"), .advance_arrow = has("+advance_arrow"), + .mission_banner = has("+mission_banner"), .mission_complete = has("+mission_complete"), + }; +} + +// The opening cutscene up to the platform: none is the platform. +std::optional opening(std::string_view){ + return DetectorReads{}; +} + +// The platform (as named) with the controls hint (0001-0006), the walk, +// the white fade, the title card, the text-free flyover, the starters, the tourist choice and the start of the +// ad: no autosave badge, the advance chevron only on the one box with its +// text done (0040). +std::optional arrival_to_ad(std::string_view name){ + const std::optional index = frame_number(name, /*offset=*/0, /*digits=*/4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, .advance_arrow = *index == 40, + }; +} + +// As arrival_to_ad(), on to the ad's subtitles (0090-0115). The advance +// chevron on the boxes with their text done (0044, 0049, 0053, 0078) and +// on the subtitles once typed (0093-0115). +std::optional arrival_to_ad_subtitles(std::string_view name){ + const std::optional index = frame_number(name, /*offset=*/0, /*digits=*/4); + if (!index){ + return std::nullopt; + } + const bool box_arrow = *index == 44 || *index == 49 || *index == 53 || *index == 78; + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = *index <= 6, + .advance_arrow = box_arrow || (*index >= 93 && *index <= 115), + }; +} + +// As arrival_to_ad_subtitles(), on through the ad and the bag scene. The +// platform, the controls hint and the advance chevron as named; the +// mission banner on 0181 only, the last frame. +std::optional arrival_to_mission_banner(std::string_view name){ + const std::optional index = frame_number(name, /*offset=*/0, /*digits=*/4); + if (!index){ + return std::nullopt; + } + return DetectorReads{ + .platform = name.find("+platform") != std::string_view::npos, + .controls_hint = name.find("+controls_hint") != std::string_view::npos, + .advance_arrow = name.find("+advance_arrow") != std::string_view::npos, + .mission_banner = *index == 181, + }; +} + +// Every detector as named: the banner on 0181-0189, the three-row +// controls hint from 0198. The autosave badge on 0197 too: it is named +// "nothing" but shows the badge +// (Traditional Chinese, top right), right after the black fade. +std::optional arrival_past_mission_banner(std::string_view name){ + DetectorReads want = as_named(name); + if (name.starts_with("0197-")){ + want.autosave = true; + } + return want; +} + +// The end of Main Mission 01 to Main Mission 02 (01.03's end): as named, +// and the complete card on 0045 and 0046 (named "nothing"); the next +// mission's banner (0133-0139) is not the card. +std::optional mission_complete_to_next_banner(std::string_view name){ + DetectorReads want = as_named(name); + want.mission_complete = name.starts_with("0045-") || name.starts_with("0046-"); + return want; +} + +struct FrameFolderCase{ + std::string name; + struct Input{ + std::string_view folder; // In CommandLineTests/PokemonLZA/AutoStoryDetectors/. + }; + Input input; + struct Expectation{ + size_t frames = 0; // How many frames of the folder `reads` has an expectation for. + // Each frame's reads, by file name; none for a file not in the set. + // A detector it leaves unset must not fire. + std::function(std::string_view name)> reads = {}; + }; + Expectation expectation; +}; + +const FrameFolderCase FRAME_FOLDER_CASES[] = { + { + .name = "the opening cutscene", + .input = {.folder = "Opening"}, + .expectation = {.frames = 56, .reads = opening}, + }, + { + .name = "the arrival to the ad", + .input = {.folder = "ArrivalToAd"}, + .expectation = {.frames = 89, .reads = arrival_to_ad}, + }, + { + .name = "the arrival to the ad's subtitles", + .input = {.folder = "ArrivalToAdSubtitles"}, + .expectation = {.frames = 115, .reads = arrival_to_ad_subtitles}, + }, + { + .name = "the arrival to the mission banner", + .input = {.folder = "ArrivalToMissionBanner"}, + .expectation = {.frames = 181, .reads = arrival_to_mission_banner}, + }, + { + .name = "the arrival past the mission banner", + .input = {.folder = "ArrivalPastMissionBanner"}, + .expectation = {.frames = 221, .reads = arrival_past_mission_banner}, + }, + { + .name = "the mission complete card to the next mission's banner", + .input = {.folder = "MissionCompleteToNextBanner"}, + .expectation = {.frames = 139, .reads = mission_complete_to_next_banner}, + }, + { + // console B, 2026-11-15, 1280x720: Italian's longer header moves + // the banner's icon 96 px left; the three-row controls hint puts + // its card ~20 px higher. + .name = "01.01's banner and controls hint in English and Italian", + .input = {.folder = "Arrival0101Languages"}, + .expectation = {.frames = 7, .reads = as_named}, + }, +}; + +UnitTestResult check_frame_folder(const FrameFolderCase& test, Logger& logger){ + const Filesystem::Path folder(std::format("{}PokemonLZA/AutoStoryDetectors/{}/", UNIT_TEST_RESOURCE_PATH(), test.input.folder)); + if (!Filesystem::is_directory(folder)){ + return std::format("{} is not a folder.", folder.string()); + } + StoryDetectors detectors; + size_t frames = 0; + std::string failures; + for (const Filesystem::Path& path : Filesystem::list_directory(folder)){ + const std::string name = path.filename().string(); + if (!name.ends_with(".jpg") && !name.ends_with(".png")){ + continue; + } + const std::optional want = test.expectation.reads(name); + if (!want){ + continue; + } + const DetectorReads got = detectors.read(ImageRGB32(path.string())); + logger.log(std::format("{}: {}{}", name, to_string(detectors.platform.last_distances()), firing(got))); + const std::string wrong = mismatches(got, every_detector(*want)); + if (!wrong.empty()){ + failures += std::format(" {}:{};", name, wrong); + } + frames++; + } + if (frames != test.expectation.frames){ + return std::format("Expected {} frames in {}, found {}.", test.expectation.frames, folder.string(), frames); + } + if (!failures.empty()){ + return std::format("Wrong on:{}", failures); + } + return true; +} + + +struct TutorialCardFrameCase{ + std::string name; + struct Input{ + std::string_view path; // In CommandLineTests/PokemonLZA/. + }; + Input input; + struct Expectation{ + bool card = false; + }; + Expectation expectation; +}; + +const TutorialCardFrameCase TUTORIAL_CARD_FRAME_CASES[] = { + { + // console B, 2026-11-16 (03.02b run 5): the "Map & Missions" card, + // page 1 of 2, over the map app's blue screen, not a dark one. + .name = "the Map & Missions card over the map app (Italian)", + .input = {.path = "AutoStoryScreenEvents/it_0302b_map_missions_card.png"}, + .expectation = {.card = true}, + }, + { + .name = "no card in a battle", + .input = {.path = "AutoStoryScreenEvents/it_0302b_battle_no_lock_on.png"}, + .expectation = {.card = false}, + }, + { + .name = "no card at the A Talk prompt", + .input = {.path = "AutoStoryScreenEvents/it_0102_parla_talk_prompt.png"}, + .expectation = {.card = false}, + }, + { + // console B, 2026-11-16: a save file's first X opens the menu under + // the "Salvare il gioco" card (save by R in the main menu); A + // closes it. + .name = "the Save card over the main menu (Italian)", + .input = {.path = "AutoStoryScreenEvents/it_menu_save_tutorial_card.png"}, + .expectation = {.card = true}, + }, + { + .name = "no card on the main menu", + .input = {.path = "MainMenuDetector/it_menu_after_save_tutorial_True.png"}, + .expectation = {.card = false}, + }, +}; + +UnitTestResult check_tutorial_card_frame(const TutorialCardFrameCase& test){ + const std::string path = std::format("{}PokemonLZA/{}", UNIT_TEST_RESOURCE_PATH(), test.input.path); + const bool card = tutorial_card_shows(ImageRGB32(path)); + if (card != test.expectation.card){ + return std::format("{}: expected card {}, got {}", test.input.path, test.expectation.card ? "yes" : "no", card ? "yes" : "no"); + } + return true; +} + + +class Test_SyntheticFrame : public UnitTest{ +public: + explicit Test_SyntheticFrame(const SyntheticFrameCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - synthetic frame: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_synthetic_frame(m_test, logger); + } + +private: + const SyntheticFrameCase& m_test; +}; + +class Test_TutorialCardFrame : public UnitTest{ +public: + explicit Test_TutorialCardFrame(const TutorialCardFrameCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - tutorial card frame: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_tutorial_card_frame(m_test); + } + +private: + const TutorialCardFrameCase& m_test; +}; + +class Test_FrameFolder : public UnitTest{ +public: + explicit Test_FrameFolder(const FrameFolderCase& test) + : UnitTest(std::format("PokemonLZA::AutoStoryDetectors - console frames: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check_frame_folder(m_test, logger); + } + +private: + const FrameFolderCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryDetectors(UnitTestDatabase& database){ + for (const SyntheticFrameCase& test : SYNTHETIC_FRAME_CASES){ + database.add(test); + } + for (const FrameFolderCase& test : FRAME_FOLDER_CASES){ + database.add(test); + } + for (const TutorialCardFrameCase& test : TUTORIAL_CARD_FRAME_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h new file mode 100644 index 0000000000..bda582d5d1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDetectors_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - Detectors Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDetectors_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the tests of the AutoStory detectors (Detectors/): synthetic +// frames, and console frames from CommandLineTests. +void add_tests_AutoStoryDetectors(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp new file mode 100644 index 0000000000..482b3cd397 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.cpp @@ -0,0 +1,179 @@ +/* PokemonLZA AutoStory - Dialog choice box + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// Rows read from the bottom up before the reader stops looking. +constexpr size_t MAX_CHOICE_ROWS = 6; +// A menu has two rows at least. One white strip alone is not one: light +// pavement reads as a highlighted row (ArrivalToAd 0038). +constexpr size_t MIN_CHOICE_ROWS = 2; + +// The menu's geometry at 1920 x 1080. Measured on the console on the +// tourist's two rows (Traditional Chinese), the Research Lab elevator's +// three (Japanese) and Villy's two at 01.02 (Italian, French). The panel +// is anchored at its bottom right: it ends at x 1728, y 808 whatever its +// rows, and grows left to fit its longest row (x 1408 in Japanese, 1343 +// in Italian, 1198 in French). The highlight runs from 10 right of the +// panel's left to x 1715; the bottom row's from y 732 to 800, each row +// above 80 higher. The rows' text starts 35 right of the panel's left +// and ends at x 1690 at most (Italian's and French's longest rows: 1688). +constexpr double ROW_PITCH = 80.0 / 1080; +constexpr double STRIP_TOP = 748.0 / 1080; +constexpr double STRIP_HEIGHT = 40.0 / 1080; +constexpr double TEXT_END_X = 1690.0 / 1920; +// Where the leftward scan for the highlight's left end starts, and the +// bottom row's highlight 4 below its top and 4 above its bottom: few +// glyphs reach both. +constexpr double HIGHLIGHT_SCAN_X = 1700.0 / 1920; +constexpr double HIGHLIGHT_TOP_BAND_Y = 736.0 / 1080; +constexpr double HIGHLIGHT_BOTTOM_BAND_Y = 796.0 / 1080; +// A plain row's share of bright text pixels: 0.07 to 0.2 on the console, +// 0 on the elevator's wall above its top row. +constexpr double MIN_TEXT_SHARE = 0.005; + +// Right of any row's text, inside the highlight. +ImageFloatBox right_strip(size_t from_bottom){ + return ImageFloatBox(1698.0 / 1920, STRIP_TOP - ROW_PITCH * from_bottom, 14.0 / 1920, STRIP_HEIGHT); +} + +// Where a row's text can be, from 10 right of the highlight's left end +// (`highlight_left`, a share of the width) to the text's right limit. +ImageFloatBox text_band(size_t from_bottom, double highlight_left){ + const double left = highlight_left + 10.0 / 1920; + return ImageFloatBox(left, STRIP_TOP - ROW_PITCH * from_bottom, TEXT_END_X - left, STRIP_HEIGHT); +} + +bool bright(uint32_t pixel, uint32_t min_channel){ + const uint32_t red = (pixel >> 16) & 0xff; + const uint32_t green = (pixel >> 8) & 0xff; + const uint32_t blue = pixel & 0xff; + return red >= min_channel && green >= min_channel && blue >= min_channel; +} + +enum class ChoiceSlot{ + HIGHLIGHTED, + PANEL, + OTHER, +}; + +// A slot's right strip: the highlighted row 255 white, flat; a plain row +// the panel's dark blue, (35-45, 60-73, 108-120), spread 3-7. The wall +// above the elevator's top row is as dark and blue, (23-24, 58-62, +// 102-107); `row_has_text` tells it from a row. +ChoiceSlot classify_slot(const ImageViewRGB32& frame, size_t from_bottom){ + const ImageStats right = image_stats(extract_box_reference(frame, right_strip(from_bottom))); + if (right.count == 0 || right.stddev.sum() > 30){ + return ChoiceSlot::OTHER; + } + const FloatPixel& average = right.average; + if (average.r >= 200 && average.g >= 200 && average.b >= 200){ + return ChoiceSlot::HIGHLIGHTED; + } + const bool dark_blue = + average.r <= 70 && average.g <= 100 && + average.b >= 80 && average.b <= 150 && + average.b >= average.r + 40; + return dark_blue ? ChoiceSlot::PANEL : ChoiceSlot::OTHER; +} + +// The highlight's left end on the row `from_bottom` slots up, as a share +// of the width: scanned left from inside its right end while its top or +// bottom band is white. +double highlight_left(const ImageViewRGB32& frame, size_t from_bottom){ + const double pitch = ROW_PITCH * from_bottom; + const size_t top_y = (size_t)((HIGHLIGHT_TOP_BAND_Y - pitch) * frame.height()); + const size_t bottom_y = (size_t)((HIGHLIGHT_BOTTOM_BAND_Y - pitch) * frame.height()); + size_t x = (size_t)(HIGHLIGHT_SCAN_X * frame.width()); + while (x > 0 && (bright(frame.pixel(x - 1, top_y), 200) || bright(frame.pixel(x - 1, bottom_y), 200))){ + x--; + } + return (double)x / frame.width(); +} + +// Whether a plain row's text band has text: white glyphs on the panel, +// whatever the language or the row's length. +bool row_has_text(const ImageViewRGB32& frame, size_t from_bottom, double highlight_left){ + const ImageViewRGB32 band = extract_box_reference(frame, text_band(from_bottom, highlight_left)); + const size_t area = band.width() * band.height(); + if (area == 0){ + return false; + } + size_t bright_pixels = 0; + for (size_t y = 0; y < band.height(); y++){ + for (size_t x = 0; x < band.width(); x++){ + if (bright(band.pixel(x, y), 180)){ + bright_pixels++; + } + } + } + return (double)bright_pixels / area >= MIN_TEXT_SHARE; +} + +} // namespace + + +// The selection arrow sits left of the panel: from 13 left of it to 27 +// right (x 1185 to 1225 on French's panel at 1198). French's 01.02 rows, +// 35 characters, take 455 of the panel's 530; the box starts at x 960, +// room for a panel at x 973, rows of about 54 such characters. +const ImageFloatBox& dialog_choice_box(){ + static const ImageFloatBox box{960.0 / 1920, 0.481553, 0.949837 - 960.0 / 1920, 0.312621}; + return box; +} + +std::optional read_dialog_choice_rows(const ImageViewRGB32& frame){ + if (!frame || frame.width() < 192 || frame.height() < 108){ + return std::nullopt; + } + // The menu's slots from the bottom up, highlighted or panel blue. + std::vector slots; + std::optional first_highlighted; + while (slots.size() < MAX_CHOICE_ROWS){ + const ChoiceSlot slot = classify_slot(frame, slots.size()); + if (slot == ChoiceSlot::OTHER){ + break; + } + if (slot == ChoiceSlot::HIGHLIGHTED && !first_highlighted){ + first_highlighted = slots.size(); + } + slots.push_back(slot); + } + if (!first_highlighted){ + return std::nullopt; + } + // The rows end at the first blue slot with no text. + const double left = highlight_left(frame, *first_highlighted); + size_t count = 0; + size_t highlighted = 0; + for (; count < slots.size(); count++){ + if (slots[count] == ChoiceSlot::HIGHLIGHTED){ + highlighted++; + }else if (!row_has_text(frame, count, left)){ + break; + } + } + if (highlighted != 1 || count < MIN_CHOICE_ROWS){ + return std::nullopt; + } + return DialogChoiceRows{.count = count, .selected = count - 1 - *first_highlighted}; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h new file mode 100644 index 0000000000..780833bb4c --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox.h @@ -0,0 +1,53 @@ +/* PokemonLZA AutoStory - Dialog choice box + * + * From: https://github.com/PokemonAutomation/ + * + * Where a dialogue's choices are drawn, for the selection arrow's + * detector (the screen events' choice box), and a reader for the choice + * rows: how many there are and which one is highlighted (the backend's + * choice readings, which the "choose_N" steps press by). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_H + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Where a dialogue's selection arrow can be: the restaurant +// receptionist's box (RestaurantFarmer), widened left to x 960 for the +// panels of long languages (French's 01.02 arrow is at x 1185). +const ImageFloatBox& dialog_choice_box(); + +// A dialogue's choice menu as read: how many rows it has, and the +// highlighted one, 0 the top. +struct DialogChoiceRows{ + size_t count = 0; + size_t selected = 0; +}; + +// Reads the choice menu on `frame`. The menu's panel is anchored at its +// bottom right: its last row sits at the same place whatever the row +// count, each further row adds one row pitch on top, and the panel grows +// left to fit its longest row, which differs by language. So the rows are +// counted from the bottom up, each either highlighted (white right of any +// text) or plain (dark blue there, with bright text somewhere between the +// highlight's left end and the text's right limit), until a slot is +// neither; the selected row is then counted from the top. None for fewer +// than two rows, or if no row, or not exactly one, is highlighted. +std::optional read_dialog_choice_rows(const ImageViewRGB32& frame); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp new file mode 100644 index 0000000000..eed12874c0 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.cpp @@ -0,0 +1,297 @@ +/* PokemonLZA AutoStory - Dialog choice box Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The choice rows reader on synthetic menus drawn as the console shows + * them, and on console frames (1920x1080) from CommandLineTests/PokemonLZA/. + * + * Not here: PokemonLZA/AutoStoryScreenEvents/za2_m3w_s2.png, the old + * chain's choice box frame for the screen events, is not in + * CommandLineTests yet; ArrivalToAd's 0065 stands in for it. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStoryScreenEvents.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox.h" +#include "PokemonLZA_AutoStoryDialogChoiceBox_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr uint32_t PAVEMENT = 0xffc8c3be; +constexpr uint32_t BLUE_WALL = 0xff18375c; +constexpr uint32_t PANEL = 0xff243c6c; +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t DARK_TEXT = 0xff1e1e28; + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +// How wide a choice menu's panel is and how far its rows' text runs: the +// panel grows to the left to fit its longest row. +struct MenuWidth{ + size_t panel_left = 1408; + size_t text_end = 1540; +}; + +// The menus whose rows read on the console in Japanese and Chinese. +constexpr MenuWidth NARROW_MENU{}; +// The Italian 01.02 menu's panel (x 1343), with every row's text running +// to x 1688, near the highlight's right end. +constexpr MenuWidth WIDE_MENU_LONG_ROWS{.panel_left = 1343, .text_end = 1688}; +// The same panel with rows as short as "Si": no text past x 1400. +constexpr MenuWidth WIDE_MENU_SHORT_ROWS{.panel_left = 1343, .text_end = 1400}; + +// A choice menu of `rows` rows drawn as the console shows it (1920 x 1080) +// on `background`: the panel from x `width.panel_left` to 1728, from +// y 808 up, one 80 pitch per row; row k from the bottom highlighted white +// at y 732 - 80k to 800 - 80k, from 10 right of the panel's left to +// x 1715. Each row's text runs from 35 right of the panel's left to +// `width.text_end`, in strokes one column in three: dark on the +// highlighted rows (`highlighted`), white on the others. +ImageRGB32 choice_menu_frame( + uint32_t background, size_t rows, std::initializer_list highlighted, + MenuWidth width = NARROW_MENU +){ + ImageRGB32 image(1920, 1080); + image.fill(background); + fill_box(width.panel_left, 808 - 80 * rows - 4, /*max_x=*/1728, /*max_y=*/808, PANEL, image); + for (size_t row = 0; row < rows; row++){ + const size_t from_bottom = rows - 1 - row; + const size_t top = 732 - 80 * from_bottom; + bool lit = false; + for (const size_t pick : highlighted){ + lit = lit || pick == row; + } + if (lit){ + fill_box(width.panel_left + 10, top, /*max_x=*/1715, top + 68, WHITE, image); + } + for (size_t x = width.panel_left + 35; x < width.text_end; x += 3){ + fill_box(x, top + 16, x + 1, top + 52, lit ? DARK_TEXT : WHITE, image); + } + } + return image; +} + +ImageRGB32 flat_frame(uint32_t color){ + ImageRGB32 image(1920, 1080); + image.fill(color); + return image; +} + +std::string describe(const std::optional& rows){ + if (!rows){ + return "no choice menu"; + } + return std::format("row {} of {}", rows->selected, rows->count); +} + +bool same(const std::optional& a, const std::optional& b){ + if (!a || !b){ + return a.has_value() == b.has_value(); + } + return a->count == b->count && a->selected == b->selected; +} + + +struct SyntheticCase{ + std::string name; + struct Input{ + std::function frame; + }; + Input input; + struct Expectation{ + std::optional rows; // None: no choice menu read. + }; + Expectation expectation; +}; + +const SyntheticCase SYNTHETIC_CASES[] = { + { + .name = "two rows, the top highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/2, /*highlighted=*/{0}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}}, + }, + { + .name = "two rows, the bottom highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/2, /*highlighted=*/{1}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 1}}, + }, + { + .name = "three rows on a blue wall, the middle highlighted", + .input = {.frame = []{ return choice_menu_frame(BLUE_WALL, /*rows=*/3, /*highlighted=*/{1}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 3, .selected = 1}}, + }, + { + .name = "four rows, the bottom highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/4, /*highlighted=*/{3}); }}, + .expectation = {.rows = DialogChoiceRows{.count = 4, .selected = 3}}, + }, + { + .name = "two long rows on a wide panel, the top highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/2, /*highlighted=*/{0}, WIDE_MENU_LONG_ROWS); }}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}}, + }, + { + .name = "three short rows on a wide panel, on a blue wall, the bottom highlighted", + .input = {.frame = []{ return choice_menu_frame(BLUE_WALL, /*rows=*/3, /*highlighted=*/{2}, WIDE_MENU_SHORT_ROWS); }}, + .expectation = {.rows = DialogChoiceRows{.count = 3, .selected = 2}}, + }, + { + .name = "one highlighted row is no menu", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/1, /*highlighted=*/{0}); }}, + .expectation = {.rows = std::nullopt}, + }, + { + .name = "three rows, none highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/3, /*highlighted=*/{}); }}, + .expectation = {.rows = std::nullopt}, + }, + { + .name = "three rows, two highlighted", + .input = {.frame = []{ return choice_menu_frame(PAVEMENT, /*rows=*/3, /*highlighted=*/{0, 2}); }}, + .expectation = {.rows = std::nullopt}, + }, + {.name = "flat panel blue", .input = {.frame = []{ return flat_frame(PANEL); }}, .expectation = {.rows = std::nullopt}}, + {.name = "flat white", .input = {.frame = []{ return flat_frame(WHITE); }}, .expectation = {.rows = std::nullopt}}, + {.name = "an empty frame", .input = {.frame = []{ return ImageRGB32(); }}, .expectation = {.rows = std::nullopt}}, + {.name = "a 1 x 1 frame", .input = {.frame = []{ return ImageRGB32(1, 1); }}, .expectation = {.rows = std::nullopt}}, +}; + +UnitTestResult check_synthetic(const SyntheticCase& test){ + const std::optional rows = read_dialog_choice_rows(test.input.frame()); + if (!same(rows, test.expectation.rows)){ + return std::format("Expected {}, read {}.", describe(test.expectation.rows), describe(rows)); + } + return true; +} + + +// The console's frames, with the menus' rows as they open (the top one +// highlighted). +struct FrameCase{ + std::string name; + struct Input{ + std::string_view file; // In CommandLineTests/PokemonLZA/. + }; + Input input; + struct Expectation{ + std::optional rows; // None: no choice menu read. + // The screen events' CHOICE_BOX (the selection arrow); none: not + // checked. + std::optional choice_box = std::nullopt; + }; + Expectation expectation; +}; + +const FrameCase FRAME_CASES[] = { + { + .name = "the tourist's yes or yes (2 rows, Traditional Chinese)", + .input = {.file = "AutoStoryDetectors/ArrivalToAd/0065-text_box+choice_box.jpg"}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}, .choice_box = true}, + }, + { + .name = "the bag's \"not really\" (2 rows, Traditional Chinese)", + .input = {.file = "AutoStoryDetectors/ArrivalToMissionBanner/0173-text_box.jpg"}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}}, + }, + { + .name = "the Research Lab elevator's floors (3 rows, Japanese)", + .input = {.file = "AutoStoryDialogChoices/elevator_3rows_row0.png"}, + .expectation = {.rows = DialogChoiceRows{.count = 3, .selected = 0}}, + }, + { + .name = "Villy's \"I'll get it back\" or \"battle\" at 01.02 (2 rows, Italian: a wider panel)", + .input = {.file = "AutoStoryDialogChoices/it_0102_villy_2rows_row0.png"}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}, .choice_box = true}, + }, + { + .name = "Villy's \"I'll get it back\" or \"battle\" at 01.02 (2 rows, French: the arrow left of x 1256)", + .input = {.file = "AutoStoryDialogChoices/fr_0102_villy_2rows_row0.png"}, + .expectation = {.rows = DialogChoiceRows{.count = 2, .selected = 0}, .choice_box = true}, + }, + { + .name = "light pavement, no menu", + .input = {.file = "AutoStoryDetectors/ArrivalToAd/0038-nothing+autosave.jpg"}, + .expectation = {.rows = std::nullopt}, + }, +}; + +UnitTestResult check_frame(const FrameCase& test){ + const Filesystem::Path path(std::format("{}PokemonLZA/{}", UNIT_TEST_RESOURCE_PATH(), test.input.file)); + if (!Filesystem::exists(path)){ + return std::format("{} is missing.", path.string()); + } + const ImageRGB32 frame(path.string()); + const std::optional rows = read_dialog_choice_rows(frame); + if (!same(rows, test.expectation.rows)){ + return std::format("{}: expected {}, read {}.", test.input.file, describe(test.expectation.rows), describe(rows)); + } + if (!test.expectation.choice_box){ + return true; + } + PlzaScreenObserver observer; + const bool choice_box = observer.read(frame).has(ScreenEvent::CHOICE_BOX); + if (choice_box != *test.expectation.choice_box){ + return std::format("{}: expected the choice box {}.", test.input.file, *test.expectation.choice_box ? "on" : "off"); + } + return true; +} + + +template +class ChoiceBoxTest : public UnitTest{ +public: + using Check = UnitTestResult (*)(const Case&); + + ChoiceBoxTest(std::string_view suite, const Case& test, Check check) + : UnitTest(std::format("PokemonLZA::AutoStory - Dialog choice rows, {}: {}", suite, test.name)) + , m_test(test) + , m_check(check) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_check(m_test); + } + +private: + const Case& m_test; + Check m_check; +}; + +} // namespace + + +void add_tests_AutoStoryDialogChoiceBox(UnitTestDatabase& database){ + for (const SyntheticCase& test : SYNTHETIC_CASES){ + database.add>("synthetic", test, check_synthetic); + } + for (const FrameCase& test : FRAME_CASES){ + database.add>("console frame", test, check_frame); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h new file mode 100644 index 0000000000..ded4339d3d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogChoiceBox_Tests.h @@ -0,0 +1,27 @@ +/* PokemonLZA AutoStory - Dialog choice box Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogChoiceBox_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the tests of the choice rows reader on synthetic menus and on +// console frames from CommandLineTests, and of the screen events' choice +// box on a console frame. +void add_tests_AutoStoryDialogChoiceBox(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp new file mode 100644 index 0000000000..bceec91675 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.cpp @@ -0,0 +1,37 @@ +/* PokemonLZA AutoStory - Dialog Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Color.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryDialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +StoryDialogDetector::StoryDialogDetector(StoryDialogKinds kinds) + : m_kinds(kinds) + , m_white(COLOR_RED) + , m_blue(COLOR_BLUE) + , m_teal(COLOR_GREEN) +{} + +void StoryDialogDetector::make_overlays(VideoOverlaySet& items) const{ + m_white.make_overlays(items); + m_blue.make_overlays(items); + m_teal.make_overlays(items); +} + +bool StoryDialogDetector::detect(const ImageViewRGB32& screen){ + const bool white = m_kinds.white == WhiteDialogPart::BOX_ONLY ? m_white.detect_box(screen) : m_white.detect(screen); + return white || m_blue.detect(screen) || m_teal.detect(screen); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h new file mode 100644 index 0000000000..b479d6b7a5 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector.h @@ -0,0 +1,50 @@ +/* PokemonLZA AutoStory - Dialog Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Any of the story's dialogue boxes, from the upstream dialog detectors. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_H + +#include "CommonTools/VisualDetector.h" +#include "VideoGames/PokemonLZA/Inference/PokemonLZA_DialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// How much of a white dialogue box StoryDialogDetector needs. +enum class WhiteDialogPart{ + BOX_AND_ARROW, // FlatWhiteDialogDetector: the box with its advance arrow + BOX_ONLY, // its bottom edge only: the box while the text still types +}; + +// Which dialogue boxes StoryDialogDetector counts, besides blue and teal. +struct StoryDialogKinds{ + WhiteDialogPart white = WhiteDialogPart::BOX_AND_ARROW; +}; + +// Any dialogue box the story shows: white, blue or teal. +class StoryDialogDetector : public StaticScreenDetector{ +public: + explicit StoryDialogDetector(StoryDialogKinds kinds = {}); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + StoryDialogKinds m_kinds; + FlatWhiteDialogDetector m_white; + BlueDialogDetector m_blue; + TealDialogDetector m_teal; +}; + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.cpp new file mode 100644 index 0000000000..438a6915a1 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.cpp @@ -0,0 +1,181 @@ +/* PokemonLZA AutoStory - Dialog Detector Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonLZA_AutoStoryDialogDetector.h" +#include "PokemonLZA_AutoStoryDialogDetector_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr size_t WIDTH = 1920; +constexpr size_t HEIGHT = 1080; +constexpr uint32_t WHITE = 0xffffffff; +constexpr uint32_t DARK_GRAY = 0xff303030; + +void fill_box(size_t min_x, size_t min_y, size_t max_x, size_t max_y, uint32_t color, ImageRGB32& image){ + for (size_t y = min_y; y < max_y; y++){ + for (size_t x = min_x; x < max_x; x++){ + image.pixel(x, y) = color; + } + } +} + +ImageRGB32 solid_frame(uint32_t color){ + ImageRGB32 image(WIDTH, HEIGHT); + image.fill(color); + return image; +} + +// A white dialogue box's bottom edge (x 500-1410, y 990-1040) on a dark +// scene, with no advance arrow: the box while its text still types. +ImageRGB32 box_without_arrow_frame(){ + ImageRGB32 image = solid_frame(DARK_GRAY); + fill_box(/*min_x=*/500, /*min_y=*/990, /*max_x=*/1410, /*max_y=*/1040, WHITE, image); + return image; +} + +// A console frame from CommandLineTests/PokemonLZA/. +ImageRGB32 console_frame(std::string_view path){ + return ImageRGB32(std::format("{}PokemonLZA/{}", UNIT_TEST_RESOURCE_PATH(), path)); +} + +// The console frame `path` with the advance arrow's area (x 1395-1470, +// y 935-1030) painted white, as the box shows before its text is done. +ImageRGB32 console_frame_without_arrow(std::string_view path){ + ImageRGB32 image = console_frame(path); + fill_box(/*min_x=*/1395, /*min_y=*/935, /*max_x=*/1470, /*max_y=*/1030, WHITE, image); + return image; +} + + +struct DialogCase{ + std::string name; + struct Input{ + std::function frame; + WhiteDialogPart white; + }; + Input input; + struct Expectation{ + bool dialog; + }; + Expectation expectation; +}; + +const DialogCase DIALOG_CASES[] = { + { + .name = "a black frame, box and arrow", + .input = {.frame = []{ return solid_frame(0xff000000); }, .white = WhiteDialogPart::BOX_AND_ARROW}, + .expectation = {.dialog = false}, + }, + { + .name = "a black frame, box only", + .input = {.frame = []{ return solid_frame(0xff000000); }, .white = WhiteDialogPart::BOX_ONLY}, + .expectation = {.dialog = false}, + }, + { + .name = "a white box without its arrow, box and arrow", + .input = {.frame = box_without_arrow_frame, .white = WhiteDialogPart::BOX_AND_ARROW}, + .expectation = {.dialog = false}, + }, + { + .name = "a white box without its arrow, box only", + .input = {.frame = box_without_arrow_frame, .white = WhiteDialogPart::BOX_ONLY}, + .expectation = {.dialog = true}, + }, + { + .name = "a white box on the console, box and arrow", + .input = { + .frame = []{ return console_frame("FlatWhiteDialogDetector/mac_restaurant_True.png"); }, + .white = WhiteDialogPart::BOX_AND_ARROW, + }, + .expectation = {.dialog = true}, + }, + { + .name = "a white box on the console, box only", + .input = { + .frame = []{ return console_frame("FlatWhiteDialogDetector/mac_restaurant_True.png"); }, + .white = WhiteDialogPart::BOX_ONLY, + }, + .expectation = {.dialog = true}, + }, + { + .name = "a white box on the console in French, box and arrow", + .input = { + .frame = []{ return console_frame("FlatWhiteDialogDetector/french_fossil_1_True.png"); }, + .white = WhiteDialogPart::BOX_AND_ARROW, + }, + .expectation = {.dialog = true}, + }, + { + .name = "a white box on the console with its arrow painted out, box and arrow", + .input = { + .frame = []{ return console_frame_without_arrow("FlatWhiteDialogDetector/mac_restaurant_True.png"); }, + .white = WhiteDialogPart::BOX_AND_ARROW, + }, + .expectation = {.dialog = false}, + }, + { + .name = "a white box on the console with its arrow painted out, box only", + .input = { + .frame = []{ return console_frame_without_arrow("FlatWhiteDialogDetector/mac_restaurant_True.png"); }, + .white = WhiteDialogPart::BOX_ONLY, + }, + .expectation = {.dialog = true}, + }, + { + .name = "a blue box on the console", + .input = { + .frame = []{ return console_frame("BlueDialogDetector/mac_cannotFastTravel_True.png"); }, + .white = WhiteDialogPart::BOX_AND_ARROW, + }, + .expectation = {.dialog = true}, + }, +}; + +class Test_StoryDialog : public UnitTest{ +public: + explicit Test_StoryDialog(const DialogCase& test) + : UnitTest(std::format("PokemonLZA::AutoStory - StoryDialogDetector: {}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + StoryDialogDetector detector({.white = m_test.input.white}); + const bool dialog = detector.detect(m_test.input.frame()); + if (dialog != m_test.expectation.dialog){ + return std::format("Expected {}, read {}.", m_test.expectation.dialog ? "a dialogue box" : "none", dialog ? "a dialogue box" : "none"); + } + return true; + } + +private: + const DialogCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryDialogDetector(UnitTestDatabase& database){ + for (const DialogCase& test : DIALOG_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.h new file mode 100644 index 0000000000..d025f5b28f --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryDialogDetector_Tests.h @@ -0,0 +1,26 @@ +/* PokemonLZA AutoStory - Dialog Detector Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryDialogDetector_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the tests of StoryDialogDetector: synthetic frames, and the +// upstream dialog detectors' frames in CommandLineTests. +void add_tests_AutoStoryDialogDetector(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.cpp new file mode 100644 index 0000000000..a73d7e0618 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.cpp @@ -0,0 +1,67 @@ +/* PokemonLZA AutoStory - Highlighted row + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryHighlightedRow.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +enum class RowBox{ + HIGHLIGHTED, + PLAIN, + OTHER, +}; + +RowBox classify_row(const ImageStats& stats, const SlateRows& slate){ + if (stats.count == 0){ + return RowBox::OTHER; + } + const FloatPixel& average = stats.average; + const double spread = stats.stddev.sum(); + if (average.r >= 200 && average.g >= 200 && average.b >= 200 && spread <= 30){ + return RowBox::HIGHLIGHTED; + } + const bool plain = + average.r >= slate.min_red && average.r <= slate.max_red && + average.g >= slate.min_green && average.g <= slate.max_green && + average.b >= slate.min_blue && average.b <= slate.max_blue && + average.b >= average.r + slate.min_blue_over_red && + spread <= slate.max_spread; + return plain ? RowBox::PLAIN : RowBox::OTHER; +} + +} // namespace + + +std::optional highlighted_row(const ImageViewRGB32& frame, std::span strips, const SlateRows& slate){ + std::optional highlighted; + for (size_t row = 0; row < strips.size(); row++){ + switch (classify_row(image_stats(extract_box_reference(frame, strips[row])), slate)){ + case RowBox::HIGHLIGHTED: + if (highlighted){ + return std::nullopt; + } + highlighted = row; + break; + case RowBox::PLAIN: + break; + case RowBox::OTHER: + return std::nullopt; + } + } + return highlighted; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.h new file mode 100644 index 0000000000..b656857111 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryHighlightedRow.h @@ -0,0 +1,48 @@ +/* PokemonLZA AutoStory - Highlighted row + * + * From: https://github.com/PokemonAutomation/ + * + * The intro's menus (the language list, "Confirm Choices") highlight their + * cursor's row white among slate rows. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryHighlightedRow_H +#define PokemonAutomation_PokemonLZA_AutoStoryHighlightedRow_H + +#include +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A menu's plain rows: each channel's average within its bounds, blue over +// red by `min_blue_over_red`, the channels' spreads summing to +// `max_spread` at most. +struct SlateRows{ + double min_red; + double max_red; + double min_green; + double max_green; + double min_blue; + double max_blue; + double min_blue_over_red; + double max_spread; +}; + +// The index of the one strip of `strips` on `frame` that is white and flat +// (every channel 200 or more, spreads summing to 30 at most), the others +// `slate`. Nullopt when none or two are white, or a strip is neither. +std::optional highlighted_row(const ImageViewRGB32& frame, std::span strips, const SlateRows& slate); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.cpp new file mode 100644 index 0000000000..bd092a1592 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.cpp @@ -0,0 +1,346 @@ +/* PokemonLZA AutoStory - New game intro screens + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/Images/ImageFilter.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "PokemonLZA_AutoStoryHighlightedRow.h" +#include "PokemonLZA_AutoStoryIntroScreens.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +constexpr size_t LANGUAGE_LIST_ROWS = 5; +constexpr size_t LANGUAGE_LIST_SIZE = 2 * LANGUAGE_LIST_ROWS; +constexpr size_t LOOK_COLUMNS = 4; +constexpr size_t LOOK_COUNT = 2 * LOOK_COLUMNS; + +bool usable(const ImageViewRGB32& frame){ + return frame && frame.width() >= 160 && frame.height() >= 90; +} + +double brightness(const ImageStats& stats){ + return stats.average.sum() / 3; +} + +ImageStats box_stats(const ImageViewRGB32& frame, const ImageFloatBox& box){ + return image_stats(extract_box_reference(frame, box)); +} + +// A strip at the left end of entry `index`'s box, left of its text and +// right of the cursor's arrow. The boxes are 576 x 84 at 1920 x 1080, the +// columns' left edges at x 347 and 996, the rows' tops 108 apart from +// y 342; the strip is 65 x 54 from 38 right of the edge and 15 down. +ImageFloatBox language_box_strip(size_t index){ + const size_t column = index / LANGUAGE_LIST_ROWS; + const size_t row = index % LANGUAGE_LIST_ROWS; + return ImageFloatBox(column == 0 ? 0.2005 : 0.5365, 0.331 + 0.1 * (double)row, 0.034, 0.05); +} + +// The highlighted box is 255 white, flat; the others slate blue, about +// (54, 60-76, 82-107), each channel's spread under 3. +constexpr SlateRows LANGUAGE_LIST_ROWS_SLATE{ + .min_red = 35, .max_red = 90, + .min_green = 45, .max_green = 110, + .min_blue = 65, .max_blue = 140, + .min_blue_over_red = 15, + .max_spread = 20, +}; + +// The dark blue background between the columns (about (9, 24, 56)). +bool column_gap_dark(const ImageViewRGB32& frame){ + const ImageStats gap = box_stats(frame, ImageFloatBox(0.4896, 0.331, 0.0208, 0.447)); + return gap.count > 0 && brightness(gap) < 70; +} + +// The look picker's tiles are 240 x 240 at 1920 x 1080, their left edges +// at x 373, 685, 997 and 1309, their tops at y 324 and 609. +double look_tile_left(size_t look){ + return (373.0 + 312.0 * (double)((look - 1) % LOOK_COLUMNS)) / 1920; +} +double look_tile_top(size_t look){ + return (look <= LOOK_COLUMNS ? 324.0 : 609.0) / 1080; +} + +// Corner `corner` of look `look`'s tile (0 top left, 1 top right, 2 bottom +// left, 3 bottom right): 12 x 12, 10 in from both edges. The highlighted +// look's white card reaches about 12 past the tile on every side, so the +// patches stay on it while it is enlarged. +ImageFloatBox look_corner_patch(size_t look, size_t corner){ + const double dx = (corner % 2 == 0 ? 10.0 : 218.0) / 1920; + const double dy = (corner < 2 ? 10.0 : 218.0) / 1080; + return ImageFloatBox(look_tile_left(look) + dx, look_tile_top(look) + dy, 12.0 / 1920, 12.0 / 1080); +} + +// The face in the middle of look `look`'s tile: 60 x 60 from 90 right of +// its edge and 100 down. +ImageFloatBox look_face_patch(size_t look){ + return ImageFloatBox(look_tile_left(look) + 90.0 / 1920, look_tile_top(look) + 100.0 / 1080, 60.0 / 1920, 60.0 / 1080); +} + +enum class LookCorner{ + WHITE, + DARK, + OTHER, +}; + +// Measured on the console's look pickers (67 frames in English, German +// and Japanese): the highlighted card's corners (240-248, 245-250, +// 246-251), spread under 17; the others slate or, past a tile's rounded +// corner, the background: (21-34, 29-57, 56-91), spread under 32. +LookCorner classify_look_corner(const ImageStats& stats){ + if (stats.count == 0){ + return LookCorner::OTHER; + } + const FloatPixel& average = stats.average; + const double spread = stats.stddev.sum(); + if (average.r >= 200 && average.g >= 200 && average.b >= 200 && spread <= 40){ + return LookCorner::WHITE; + } + if (brightness(stats) < 90 && average.r <= 50 && average.b >= average.r + 20 && spread <= 50){ + return LookCorner::DARK; + } + return LookCorner::OTHER; +} + +// How many of look `look`'s 4 corners are white; nullopt if the tile is +// not a look picker tile: a corner neither white nor dark, or no face in +// its middle (on the same frames brightness 108-196, spread over 117). +std::optional look_white_corners(const ImageViewRGB32& frame, size_t look){ + const ImageStats face = box_stats(frame, look_face_patch(look)); + if (face.count == 0 || brightness(face) < 80 || face.stddev.sum() < 60){ + return std::nullopt; + } + size_t white = 0; + for (size_t corner = 0; corner < 4; corner++){ + switch (classify_look_corner(box_stats(frame, look_corner_patch(look, corner)))){ + case LookCorner::WHITE: + white++; + break; + case LookCorner::DARK: + break; + case LookCorner::OTHER: + return std::nullopt; + } + } + return white; +} + +// The look picker as read from `frame`: whether it shows, and the look +// highlighted, the one with 4 white corners while the other 7 have none. +struct LookPickerReading{ + bool shows = false; + std::optional highlighted; +}; + +LookPickerReading read_look_picker(const ImageViewRGB32& frame){ + if (!usable(frame)){ + return {}; + } + std::optional card; + size_t lit_tiles = 0; + for (size_t look = 1; look <= LOOK_COUNT; look++){ + const std::optional white = look_white_corners(frame, look); + if (!white){ + return {}; + } + if (*white > 0){ + lit_tiles++; + } + if (*white == 4){ + card = look; + } + } + LookPickerReading reading{.shows = true}; + if (lit_tiles == 1 && card){ + reading.highlighted = card; + } + return reading; +} + +// The number at `text[pos]`, moving `pos` past it; nullopt when no digit +// is there. +std::optional read_digits(std::string_view text, size_t& pos){ + const size_t start = pos; + size_t value = 0; + while (pos < text.size() && text[pos] >= '0' && text[pos] <= '9' && pos - start < 4){ + value = value * 10 + (size_t)(text[pos] - '0'); + pos++; + } + if (pos == start){ + return std::nullopt; + } + return value; +} + +void skip_spaces(std::string_view text, size_t& pos){ + while (pos < text.size() && (text[pos] == ' ' || text[pos] == '\n' || text[pos] == '\r')){ + pos++; + } +} + +// The limit N in a counter's text "0/N"; nullopt for any other text. +std::optional parse_counter(std::string_view text){ + size_t pos = 0; + skip_spaces(text, pos); + if (!read_digits(text, pos)){ + return std::nullopt; + } + skip_spaces(text, pos); + if (pos >= text.size() || text[pos] != '/'){ + return std::nullopt; + } + pos++; + skip_spaces(text, pos); + const std::optional limit = read_digits(text, pos); + skip_spaces(text, pos); + if (!limit || *limit == 0 || pos != text.size()){ + return std::nullopt; + } + return limit; +} + +// `box` of `frame` as black text on white for OCR: the pixels with every +// channel at least `min_channel` are the text. +std::string read_light_text(const ImageViewRGB32& frame, const ImageFloatBox& box, Language language, uint8_t min_channel){ + const uint32_t mins = 0xff000000 | ((uint32_t)min_channel << 16) | ((uint32_t)min_channel << 8) | min_channel; + const ImageRGB32 filtered = to_blackwhite_rgb32_range( + extract_box_reference(frame, box), /*in_range_black=*/true, mins, /*maxs=*/0xffffffff + ); + return OCR::ocr_read(language, filtered); +} + +} // namespace + + +// The title line runs y 215-266 at 1920 x 1080, from the highlighted +// tile's left edge; the tile row's cyan border starts at y 277. The text +// is about (10, 212, 224) on (241, 241, 241). +bool is_home_tile_plza(const ImageViewRGB32& frame){ + if (!usable(frame)){ + return false; + } + const ImageRGB32 filtered = to_blackwhite_rgb32_range( + extract_box_reference(frame, ImageFloatBox(0.02, 205.0 / 1080, 0.96, 67.0 / 1080)), + /*in_range_black=*/true, /*mins=*/0xff009696, /*maxs=*/0xff7fffff + ); + std::string letters; + for (const char ch : OCR::ocr_read(Language::English, filtered)){ + if (std::isalnum((unsigned char)ch)){ + letters += (char)std::tolower((unsigned char)ch); + } + } + return letters.ends_with("za"); +} + +std::optional read_language_list_cursor(const ImageViewRGB32& frame){ + if (!usable(frame) || !column_gap_dark(frame)){ + return std::nullopt; + } + std::array strips; + for (size_t index = 0; index < LANGUAGE_LIST_SIZE; index++){ + strips[index] = language_box_strip(index); + } + return highlighted_row(frame, strips, LANGUAGE_LIST_ROWS_SLATE); +} + +std::optional read_look_picker_cursor(const ImageViewRGB32& frame){ + return read_look_picker(frame).highlighted; +} + +bool is_look_picker(const ImageViewRGB32& frame){ + return read_look_picker(frame).shows; +} + +// The question area (x 1056-1824, y 54-184 at 1920 x 1080) is dark and +// flat: 27, spread 3 on all ten languages' keyboards; the intro's other +// screens are 46 or brighter there, or busy. The keys' bottom row +// (y 929-950) averages 205-211; elsewhere 160 at most. In the console's +// dark theme (Basic Black) the question area is navy (5, 28, 48: 27, spread +// 4) and the keys are dark grey (111 on that row), so the bar below them +// (y 1010-1042) tells it instead: flat grey 76, where the intro's other +// screens are 56 or darker and HOME's dark theme is 51. +bool is_name_keyboard(const ImageViewRGB32& frame){ + if (!usable(frame)){ + return false; + } + const ImageStats question = box_stats(frame, ImageFloatBox(0.55, 0.05, 0.40, 0.12)); + if (question.count == 0 || brightness(question) > 40 || question.stddev.sum() > 15){ + return false; + } + const ImageStats keys = box_stats(frame, ImageFloatBox(0.03, 0.86, 0.80, 0.02)); + if (keys.count > 0 && brightness(keys) >= 185){ + return true; + } + const ImageStats bar = box_stats(frame, ImageFloatBox(0.10, 0.935, 0.35, 0.03)); + return bar.count > 0 && brightness(bar) >= 65 && brightness(bar) <= 90 && bar.stddev.sum() < 10; +} + +// The counter is grey (about 128) on the dark question area: with a +// 12-character field ("0/12") at x 1645-1725, y 325-355; with a 6-character +// one ("0/6") at x 1275-1335, y 355-385 (Japanese) or 340-370 (Korean, +// Chinese, whose keyboards are taller). The field's underline runs just +// above each, so each layout gets its own box. +std::optional read_name_keyboard_limit(const ImageViewRGB32& frame){ + if (!usable(frame)){ + return std::nullopt; + } + const ImageFloatBox boxes[] = { + ImageFloatBox(1625.0 / 1920, 312.0 / 1080, 120.0 / 1920, 53.0 / 1080), + ImageFloatBox(1258.0 / 1920, 345.0 / 1080, 115.0 / 1920, 50.0 / 1080), + ImageFloatBox(1258.0 / 1920, 328.0 / 1080, 115.0 / 1920, 50.0 / 1080), + }; + for (const ImageFloatBox& box : boxes){ + const std::optional limit = parse_counter(read_light_text(frame, box, Language::English, /*min_channel=*/90)); + if (limit){ + return limit; + } + } + return std::nullopt; +} + +// The values right of the "Language" and "Name" labels: white, from x 345; +// the language's row y 400-450, the name's 468-522. +std::string read_confirm_choices_name(const ImageViewRGB32& frame){ + if (!usable(frame)){ + return ""; + } + const std::string text = read_light_text( + frame, ImageFloatBox(340.0 / 1920, 468.0 / 1080, 760.0 / 1920, 54.0 / 1080), Language::English, /*min_channel=*/150 + ); + std::string name; + for (const char ch : text){ + if (ch != ' ' && ch != '\n' && ch != '\r'){ + name += ch; + } + } + return name; +} + +std::string read_confirm_choices_language(const ImageViewRGB32& frame, Language language){ + if (!usable(frame)){ + return ""; + } + return read_light_text( + frame, ImageFloatBox(340.0 / 1920, 400.0 / 1080, 760.0 / 1920, 50.0 / 1080), language, /*min_channel=*/150 + ); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h new file mode 100644 index 0000000000..9bc37d9a69 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens.h @@ -0,0 +1,69 @@ +/* PokemonLZA AutoStory - New game intro screens + * + * From: https://github.com/PokemonAutomation/ + * + * Readers of the new game's screens before "Confirm Choices" (whose + * highlight is in PokemonLZA_AutoStoryConfirmChoicesReader.h): the + * language list, the look picker and the Switch's name keyboard, and the + * name and language "Confirm Choices" shows. Measured on the console's + * frames in ten languages (2026-11-13). + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryIntroScreens_H +#define PokemonAutomation_PokemonLZA_AutoStoryIntroScreens_H + +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Language.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On the Switch HOME, whether the highlighted tile is PLZA: the title the +// HOME prints above the tile row (cyan) ends in "Z-A", read with OCR +// (English; measured on an English system). False for any other tile or +// frame. +bool is_home_tile_plza(const ImageViewRGB32& frame); + +// On "Which language will you play in?", the highlighted entry (numbered +// down the left column, then down the right one): the one white box of +// the 10, the others slate. Nullopt for any other frame, an empty one +// included. +std::optional read_language_list_cursor(const ImageViewRGB32& frame); + +// On the look picker ("LOOKS"), the look (1 to 8) whose portrait is +// highlighted: the one tile whose 4 corners are on the white card. Nullopt +// while none is (the screen opens so), while the card moves, and for any +// other frame. +std::optional read_look_picker_cursor(const ImageViewRGB32& frame); + +// The look picker's 2 x 4 portraits on dark blue, highlighted or not. +bool is_look_picker(const ImageViewRGB32& frame); + +// The Switch's name keyboard: the dark question area on top, the light +// key panel below. Any of its layouts (Latin, Hangul, Chinese). +bool is_name_keyboard(const ImageViewRGB32& frame); + +// The name keyboard's character limit, read with OCR from its "0/N" +// counter right under the text field (it sits further right with a +// 12-character field). Nullopt when no counter reads as one. +std::optional read_name_keyboard_limit(const ImageViewRGB32& frame); + +// The player name "Confirm Choices" shows, read with OCR (English), its +// spaces dropped. Empty when nothing reads. +std::string read_confirm_choices_name(const ImageViewRGB32& frame); + +// The language "Confirm Choices" shows, read with OCR in `language`. +std::string read_confirm_choices_language(const ImageViewRGB32& frame, Language language); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.cpp new file mode 100644 index 0000000000..2649fbee37 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.cpp @@ -0,0 +1,322 @@ +/* PokemonLZA AutoStory - New game intro screens Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "Tests/ImageUnitTest.h" +#include "PokemonLZA_AutoStoryConfirmChoicesReader.h" +#include "PokemonLZA_AutoStoryIntroScreens.h" +#include "PokemonLZA_AutoStoryIntroScreens_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +std::string describe(std::optional value){ + return value ? std::format("{}", *value) : std::string("none"); +} + +// One console frame and what each reader makes of it. +struct FrameCase{ + std::string name; + struct Input{ + std::string file; // Under the test images' PokemonLZA/. + }; + Input input; + struct Expectation{ + std::optional language_entry = std::nullopt; + bool look_picker = false; + std::optional look = std::nullopt; + bool keyboard = false; + // The counter's limit; nullopt: not checked (the field holds text). + std::optional name_limit = std::nullopt; + bool home_plza = false; + }; + Expectation expectation; +}; + +const FrameCase FRAME_CASES[] = { + {.name = "language list on English", .input = {.file = "AutoStoryLanguageList/0_en.png"}, .expectation = {.language_entry = 0}}, + {.name = "language list on Espanol (Espana)", .input = {.file = "AutoStoryLanguageList/1_es-ES.png"}, .expectation = {.language_entry = 1}}, + {.name = "language list on Espanol (Latinoamerica)", .input = {.file = "AutoStoryLanguageList/2_es-419.png"}, .expectation = {.language_entry = 2}}, + {.name = "language list on Francais", .input = {.file = "AutoStoryLanguageList/3_fr.png"}, .expectation = {.language_entry = 3}}, + {.name = "language list on Deutsch", .input = {.file = "AutoStoryLanguageList/4_de.png"}, .expectation = {.language_entry = 4}}, + {.name = "language list on Italiano", .input = {.file = "AutoStoryLanguageList/5_it.png"}, .expectation = {.language_entry = 5}}, + {.name = "language list on Japanese", .input = {.file = "AutoStoryLanguageList/6_ja.png"}, .expectation = {.language_entry = 6}}, + {.name = "language list on Korean", .input = {.file = "AutoStoryLanguageList/7_ko.png"}, .expectation = {.language_entry = 7}}, + {.name = "language list on Simplified Chinese", .input = {.file = "AutoStoryLanguageList/8_zh-Hans.png"}, .expectation = {.language_entry = 8}}, + {.name = "language list on Traditional Chinese", .input = {.file = "AutoStoryLanguageList/9_zh-Hant.png"}, .expectation = {.language_entry = 9}}, + {.name = "look picker as it opens", .input = {.file = "AutoStoryLookPicker/English_1_0.png"}, .expectation = {.look_picker = true}}, + {.name = "look picker on look 3", .input = {.file = "AutoStoryLookPicker/English_2_3.png"}, .expectation = {.look_picker = true, .look = 3}}, + {.name = "look picker on look 7", .input = {.file = "AutoStoryLookPicker/German_2_7.png"}, .expectation = {.look_picker = true, .look = 7}}, + {.name = "look picker on look 6 in Japanese", .input = {.file = "AutoStoryLookPicker/Japanese_2_6.png"}, .expectation = {.look_picker = true, .look = 6}}, + {.name = "keyboard in German", .input = {.file = "AutoStoryNameKeyboard/de_12.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "keyboard in Spanish (Spain)", .input = {.file = "AutoStoryNameKeyboard/es-ES_12.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "keyboard in Latin American Spanish", .input = {.file = "AutoStoryNameKeyboard/es-419_12.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "keyboard in French", .input = {.file = "AutoStoryNameKeyboard/fr_12.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "keyboard in Italian", .input = {.file = "AutoStoryNameKeyboard/it_12.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "keyboard in Japanese", .input = {.file = "AutoStoryNameKeyboard/ja_6.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "keyboard in Korean", .input = {.file = "AutoStoryNameKeyboard/ko_6.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "keyboard in Simplified Chinese", .input = {.file = "AutoStoryNameKeyboard/zh-Hans_6.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "keyboard in Traditional Chinese", .input = {.file = "AutoStoryNameKeyboard/zh-Hant_6.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "Confirm Choices is none of them", .input = {.file = "AutoStoryConfirmChoices/German_0.png"}, .expectation = {}}, + // The console's frames of 2026-11-15, one launch per language. + {.name = "2026-11-15: the language list opens on English", .input = {.file = "AutoStoryNewGame/2026-11-15/en_langlist.png"}, .expectation = {.language_entry = 0}}, + {.name = "2026-11-15: the language list on Espanol (Espana)", .input = {.file = "AutoStoryNewGame/2026-11-15/esES_langsel.png"}, .expectation = {.language_entry = 1}}, + {.name = "2026-11-15: the language list on Espanol (Latinoamerica)", .input = {.file = "AutoStoryNewGame/2026-11-15/esLA_langsel.png"}, .expectation = {.language_entry = 2}}, + {.name = "2026-11-15: the language list on Francais", .input = {.file = "AutoStoryNewGame/2026-11-15/fr_langsel.png"}, .expectation = {.language_entry = 3}}, + {.name = "2026-11-15: the language list on Deutsch", .input = {.file = "AutoStoryNewGame/2026-11-15/de_langsel.png"}, .expectation = {.language_entry = 4}}, + {.name = "2026-11-15: the language list on Italiano", .input = {.file = "AutoStoryNewGame/2026-11-15/it_langsel.png"}, .expectation = {.language_entry = 5}}, + {.name = "2026-11-15: the language list on Japanese", .input = {.file = "AutoStoryNewGame/2026-11-15/ja_langsel.png"}, .expectation = {.language_entry = 6}}, + {.name = "2026-11-15: the language list on Korean", .input = {.file = "AutoStoryNewGame/2026-11-15/ko_langsel.png"}, .expectation = {.language_entry = 7}}, + {.name = "2026-11-15: the language list on Simplified Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_langsel.png"}, .expectation = {.language_entry = 8}}, + {.name = "2026-11-15: the language list on Traditional Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_langsel.png"}, .expectation = {.language_entry = 9}}, + {.name = "2026-11-15: the look picker before the stick moves", .input = {.file = "AutoStoryNewGame/2026-11-15/en_look.png"}, .expectation = {.look_picker = true}}, + {.name = "2026-11-15: the look picker after the first tilt in English", .input = {.file = "AutoStoryNewGame/2026-11-15/en_look_hl.png"}, .expectation = {.look_picker = true, .look = 6}}, + {.name = "2026-11-15: the look picker after the first tilt in Espanol (Espana)", .input = {.file = "AutoStoryNewGame/2026-11-15/esES_look_hl.png"}, .expectation = {.look_picker = true, .look = 5}}, + {.name = "2026-11-15: the look picker after the first tilt in Espanol (Latinoamerica)", .input = {.file = "AutoStoryNewGame/2026-11-15/esLA_look_hl.png"}, .expectation = {.look_picker = true, .look = 6}}, + {.name = "2026-11-15: the look picker after the first tilt in Francais", .input = {.file = "AutoStoryNewGame/2026-11-15/fr_look_hl.png"}, .expectation = {.look_picker = true, .look = 1}}, + {.name = "2026-11-15: the look picker after the first tilt in Deutsch", .input = {.file = "AutoStoryNewGame/2026-11-15/de_look_hl.png"}, .expectation = {.look_picker = true, .look = 5}}, + {.name = "2026-11-15: the look picker after the first tilt in Italiano", .input = {.file = "AutoStoryNewGame/2026-11-15/it_look_hl.png"}, .expectation = {.look_picker = true, .look = 1}}, + {.name = "2026-11-15: the look picker after the first tilt in Japanese", .input = {.file = "AutoStoryNewGame/2026-11-15/ja_look_hl.png"}, .expectation = {.look_picker = true, .look = 7}}, + {.name = "2026-11-15: the look picker after the first tilt in Korean", .input = {.file = "AutoStoryNewGame/2026-11-15/ko_look_hl.png"}, .expectation = {.look_picker = true, .look = 2}}, + {.name = "2026-11-15: the look picker after the first tilt in Simplified Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_look_hl.png"}, .expectation = {.look_picker = true, .look = 1}}, + {.name = "2026-11-15: the look picker after the first tilt in Traditional Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_look_hl.png"}, .expectation = {.look_picker = true, .look = 8}}, + {.name = "2026-11-15: the keyboard in English", .input = {.file = "AutoStoryNewGame/2026-11-15/en_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Espanol (Espana)", .input = {.file = "AutoStoryNewGame/2026-11-15/esES_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Espanol (Latinoamerica)", .input = {.file = "AutoStoryNewGame/2026-11-15/esLA_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Francais", .input = {.file = "AutoStoryNewGame/2026-11-15/fr_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Deutsch", .input = {.file = "AutoStoryNewGame/2026-11-15/de_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Italiano", .input = {.file = "AutoStoryNewGame/2026-11-15/it_kb.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-15: the keyboard in Japanese", .input = {.file = "AutoStoryNewGame/2026-11-15/ja_kb.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "2026-11-15: the keyboard in Korean", .input = {.file = "AutoStoryNewGame/2026-11-15/ko_kb.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "2026-11-15: the keyboard in Simplified Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_kb.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "2026-11-15: the keyboard in Traditional Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_kb.png"}, .expectation = {.keyboard = true, .name_limit = 6}}, + {.name = "2026-11-15: Simplified Chinese keeps the keyboard after the first +", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_plus0.png"}, .expectation = {.keyboard = true, .name_limit = std::nullopt}}, + // HOME's tiles in last-played order: FireRed first, Z-A second. + {.name = "2026-11-15: HOME on FireRed's tile", .input = {.file = "AutoStoryNewGame/2026-11-15/home_firered.png"}, .expectation = {}}, + {.name = "2026-11-15: HOME on Z-A's tile", .input = {.file = "AutoStoryNewGame/2026-11-15/home_plza.png"}, .expectation = {.home_plza = true}}, + // The console in its dark theme (Basic Black), 2026-11-18. + {.name = "2026-11-18 dark theme: the keyboard in English", .input = {.file = "AutoStoryNewGame/2026-11-18/en_kb_dark.png"}, .expectation = {.keyboard = true, .name_limit = 12}}, + {.name = "2026-11-18 dark theme: HOME on Pokemon HOME's tile", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_tile0.png"}, .expectation = {}}, + {.name = "2026-11-18 dark theme: HOME on Z-A's tile", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_tile1.png"}, .expectation = {.home_plza = true}}, + {.name = "2026-11-18 dark theme: HOME on FireRed's tile", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_tile2.png"}, .expectation = {}}, + {.name = "2026-11-18 dark theme: HOME on Brilliant Diamond's tile", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_tile3.png"}, .expectation = {}}, + {.name = "2026-11-18 dark theme: HOME on Violet's tile", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_tile4.png"}, .expectation = {}}, + {.name = "2026-11-18 dark theme: HOME dimmed", .input = {.file = "AutoStoryNewGame/2026-11-18/home_dark_dim.png"}, .expectation = {}}, +}; + +class Test_IntroScreenFrame : public ImageUnitTest{ +public: + explicit Test_IntroScreenFrame(const FrameCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - intro screens: {}", test.name), + {std::format("PokemonLZA/{}", test.input.file)} + ) + , m_test(test) + {} + virtual UnitTestResult run_on_images(Logger&, CancellableScope&, const std::vector& images) const override{ + const ImageRGB32& frame = images[0]; + const FrameCase::Expectation& expected = m_test.expectation; + const std::optional entry = read_language_list_cursor(frame); + if (entry != expected.language_entry){ + return std::format("Language list: expected entry {}, read {}.", describe(expected.language_entry), describe(entry)); + } + if (is_look_picker(frame) != expected.look_picker){ + return std::format("Look picker: expected {}, read {}.", expected.look_picker, !expected.look_picker); + } + const std::optional look = read_look_picker_cursor(frame); + if (look != expected.look){ + return std::format("Look picker: expected look {}, read {}.", describe(expected.look), describe(look)); + } + if (is_name_keyboard(frame) != expected.keyboard){ + return std::format("Name keyboard: expected {}, read {}.", expected.keyboard, !expected.keyboard); + } + if (expected.keyboard && expected.name_limit){ + const std::optional limit = read_name_keyboard_limit(frame); + if (limit != expected.name_limit){ + return std::format("Name keyboard: expected a limit of {}, read {}.", describe(expected.name_limit), describe(limit)); + } + } + if (is_home_tile_plza(frame) != expected.home_plza){ + return std::format("HOME tile: expected PLZA {}, read {}.", expected.home_plza, !expected.home_plza); + } + return true; + } + +private: + const FrameCase& m_test; +}; + + +// Confirm Choices' name and language, read with OCR. The console's frames +// were taken with the name "q". +class Test_ConfirmChoicesProfile : public ImageUnitTest{ +public: + Test_ConfirmChoicesProfile() + : ImageUnitTest( + "PokemonLZA::AutoStory - intro screens: Confirm Choices shows the name and language", + {"PokemonLZA/AutoStoryConfirmChoices/English_0.png"} + ) + {} + virtual UnitTestResult run_on_images(Logger&, CancellableScope&, const std::vector& images) const override{ + const std::string name = read_confirm_choices_name(images[0]); + if (name != "q"){ + return std::format("Expected the name \"q\", read \"{}\".", name); + } + const std::string language = read_confirm_choices_language(images[0], Language::English); + if (language.find("English") == std::string::npos){ + return std::format("Expected the language \"English\", read \"{}\".", language); + } + return true; + } +}; + + +// The name keyboard as the Switch's keyboard typer (type_on_system_keyboard()) +// reads it when it opens: the cursor on "1". Every language the program +// types in (Korean is not one). +struct SystemKeyboardCase{ + std::string name; + struct Input{ + std::string file; // Under the test images' PokemonLZA/. + }; + Input input; +}; + +const SystemKeyboardCase SYSTEM_KEYBOARD_CASES[] = { + {.name = "English", .input = {.file = "AutoStoryNewGame/2026-11-15/en_kb.png"}}, + {.name = "Espanol (Espana)", .input = {.file = "AutoStoryNewGame/2026-11-15/esES_kb.png"}}, + {.name = "Espanol (Latinoamerica)", .input = {.file = "AutoStoryNewGame/2026-11-15/esLA_kb.png"}}, + {.name = "Francais", .input = {.file = "AutoStoryNewGame/2026-11-15/fr_kb.png"}}, + {.name = "Deutsch", .input = {.file = "AutoStoryNewGame/2026-11-15/de_kb.png"}}, + {.name = "Italiano", .input = {.file = "AutoStoryNewGame/2026-11-15/it_kb.png"}}, + {.name = "Japanese", .input = {.file = "AutoStoryNewGame/2026-11-15/ja_kb.png"}}, + {.name = "Simplified Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_kb.png"}}, + {.name = "Traditional Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_kb.png"}}, + {.name = "English, dark theme", .input = {.file = "AutoStoryNewGame/2026-11-18/en_kb_dark.png"}}, +}; + +class Test_SystemKeyboardOnNameKeyboard : public ImageUnitTest{ +public: + explicit Test_SystemKeyboardOnNameKeyboard(const SystemKeyboardCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - intro screens: the system keyboard reader in {}", test.name), + {std::format("PokemonLZA/{}", test.input.file)} + ) + {} + virtual UnitTestResult run_on_images(Logger&, CancellableScope&, const std::vector& images) const override{ + const ImageRGB32& frame = images[0]; + if (!SystemKeyboardDetector().detect(frame)){ + return std::string("The system keyboard was not detected."); + } + const std::optional cursor = read_system_keyboard_cursor(frame); + if (cursor != SystemKeyboardKey{.row = 0, .col = 0}){ + return std::format( + "Expected the cursor on \"1\" (0, 0), read {}.", + cursor ? std::format("({}, {})", cursor->row, cursor->col) : std::string("none") + ); + } + return true; + } +}; + + +// Confirm Choices after the name "1" was typed and submitted (the console's +// frames of 2026-11-15): its highlight, the name, and for the languages +// English OCR reads, a part of the language's name. +struct ConfirmCase{ + std::string name; + struct Input{ + std::string file; // Under the test images' PokemonLZA/. + }; + Input input; + struct Expectation{ + std::string name; + std::optional language; + }; + Expectation expectation; +}; + +const ConfirmCase CONFIRM_CASES[] = { + {.name = "English", .input = {.file = "AutoStoryNewGame/2026-11-15/en_confirm.png"}, .expectation = {.name = "1", .language = "English"}}, + {.name = "Espanol (Espana)", .input = {.file = "AutoStoryNewGame/2026-11-15/esES_confirm.png"}, .expectation = {.name = "1", .language = "Espa"}}, + {.name = "Espanol (Latinoamerica)", .input = {.file = "AutoStoryNewGame/2026-11-15/esLA_confirm.png"}, .expectation = {.name = "1", .language = "Latino"}}, + {.name = "Francais", .input = {.file = "AutoStoryNewGame/2026-11-15/fr_confirm.png"}, .expectation = {.name = "1", .language = "Fran"}}, + {.name = "Deutsch", .input = {.file = "AutoStoryNewGame/2026-11-15/de_confirm.png"}, .expectation = {.name = "1", .language = "Deutsch"}}, + {.name = "Italiano", .input = {.file = "AutoStoryNewGame/2026-11-15/it_confirm.png"}, .expectation = {.name = "1", .language = "Italiano"}}, + {.name = "Japanese", .input = {.file = "AutoStoryNewGame/2026-11-15/ja_confirm.png"}, .expectation = {.name = "1", .language = std::nullopt}}, + {.name = "Korean", .input = {.file = "AutoStoryNewGame/2026-11-15/ko_confirm.png"}, .expectation = {.name = "1", .language = std::nullopt}}, + {.name = "Simplified Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHans_confirm.png"}, .expectation = {.name = "1", .language = std::nullopt}}, + {.name = "Traditional Chinese", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_confirm.png"}, .expectation = {.name = "1", .language = std::nullopt}}, + {.name = "German, one + after the name", .input = {.file = "AutoStoryNewGame/2026-11-15/de_plus0.png"}, .expectation = {.name = "1", .language = "Deutsch"}}, + {.name = "Traditional Chinese, two + after the name", .input = {.file = "AutoStoryNewGame/2026-11-15/zhHant_plus1.png"}, .expectation = {.name = "1", .language = std::nullopt}}, +}; + +class Test_ConfirmChoicesFrame : public ImageUnitTest{ +public: + explicit Test_ConfirmChoicesFrame(const ConfirmCase& test) + : ImageUnitTest( + std::format("PokemonLZA::AutoStory - intro screens: Confirm Choices in {}", test.name), + {std::format("PokemonLZA/{}", test.input.file)} + ) + , m_test(test) + {} + virtual UnitTestResult run_on_images(Logger&, CancellableScope&, const std::vector& images) const override{ + const ImageRGB32& frame = images[0]; + if (!read_confirm_choices_cursor(frame)){ + return std::string("Confirm Choices' highlight did not read."); + } + if (is_name_keyboard(frame)){ + return std::string("Read as the name keyboard."); + } + const std::string name = read_confirm_choices_name(frame); + if (name != m_test.expectation.name){ + return std::format("Expected the name \"{}\", read \"{}\".", m_test.expectation.name, name); + } + if (m_test.expectation.language){ + const std::string language = read_confirm_choices_language(frame, Language::English); + if (language.find(*m_test.expectation.language) == std::string::npos){ + return std::format("Expected the language to have \"{}\", read \"{}\".", *m_test.expectation.language, language); + } + } + return true; + } + +private: + const ConfirmCase& m_test; +}; + +} // namespace + + +void add_tests_AutoStoryIntroScreens(UnitTestDatabase& database){ + for (const FrameCase& test : FRAME_CASES){ + database.add(test); + } + database.add(); + for (const SystemKeyboardCase& test : SYSTEM_KEYBOARD_CASES){ + database.add(test); + } + for (const ConfirmCase& test : CONFIRM_CASES){ + database.add(test); + } +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.h new file mode 100644 index 0000000000..803a8cb509 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryIntroScreens_Tests.h @@ -0,0 +1,27 @@ +/* PokemonLZA AutoStory - New game intro screens Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryIntroScreens_Tests_H +#define PokemonAutomation_PokemonLZA_AutoStoryIntroScreens_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// Registers the intro screen readers' tests on the console's frames +// (PokemonLZA/AutoStoryLanguageList/, AutoStoryLookPicker/, +// AutoStoryNameKeyboard/ and AutoStoryConfirmChoices/ in the test images). +void add_tests_AutoStoryIntroScreens(UnitTestDatabase& database); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp new file mode 100644 index 0000000000..cac3dd7117 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.cpp @@ -0,0 +1,77 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMainMissionBannerDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the bar's yellow fills its ends, x 430-600 and +// x 1320-1500 at y 425-585, which no mission name reaches; none of it 15 +// px above or below the bar. With "MAIN MISSION 01" the icon's hexagon is +// white at x 700-710 and 724-734, y 337-358, with the "!"'s orange stroke +// at x 716-719 between them. +MainMissionBannerDetector::MainMissionBannerDetector(Color color) + : m_color(color) + , m_bar_left(430.0 / 1920, 425.0 / 1080, 170.0 / 1920, 160.0 / 1080) + , m_bar_right(1320.0 / 1920, 425.0 / 1080, 180.0 / 1920, 160.0 / 1080) + , m_above(430.0 / 1920, 375.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_below(430.0 / 1920, 615.0 / 1080, 1070.0 / 1920, 25.0 / 1080) + , m_icon_left(700.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_right(724.0 / 1920, 337.0 / 1080, 10.0 / 1920, 21.0 / 1080) + , m_icon_stroke(716.0 / 1920, 337.0 / 1080, 3.0 / 1920, 11.0 / 1080) +{} + +void MainMissionBannerDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_icon_left); + items.add(m_color, m_icon_right); + items.add(m_color, m_icon_stroke); +} + +// On the banner both bar boxes are 100% yellow, the strips around the bar +// 0-1%, and the icon's three boxes 100% at dx 0. Other screens have the +// bar boxes 8% yellow at most and the icon's weakest box 27% at most (a +// city flyover). The header is centred with the icon left of its text, so +// the header's length moves the icon: Italian's "MISSIONE PRINCIPALE 01" +// has it at dx -96 (console B, 2026-11-15). It is looked for across the +// header's left half, x 400-960, so any language fits. +bool MainMissionBannerDetector::detect(const ImageViewRGB32& screen){ + if (banner_yellow_share(screen, m_bar_left) < 0.6 || banner_yellow_share(screen, m_bar_right) < 0.6){ + return false; + } + if (banner_yellow_share(screen, m_above) > 0.3 || banner_yellow_share(screen, m_below) > 0.3){ + return false; + } + const auto bright = [](Color pixel){ + return pixel.red() > 225 && pixel.green() > 225 && pixel.blue() > 225; + }; + const auto orange = [](Color pixel){ + return pixel.red() > 200 && pixel.blue() < 150; + }; + for (double dx = -300; dx <= 226; dx += 2){ + if (share_of(screen, shifted(m_icon_left, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_right, dx), bright) >= 0.7 && + share_of(screen, shifted(m_icon_stroke, dx), orange) >= 0.7 + ){ + return true; + } + } + return false; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h new file mode 100644 index 0000000000..585e776fdd --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMainMissionBannerDetector.h @@ -0,0 +1,52 @@ +/* PokemonLZA AutoStory - Main mission banner detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "MAIN MISSION 01" banner across the middle when a main mission + * starts. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMainMissionBannerDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The banner a main mission starts with: a see-through yellow bar at +// x 345-1590, y 410-600 on 1920x1080 with the mission's name in the +// middle, and above it, left of "MAIN MISSION 01", a white hexagon with +// an orange "!" at about x 700-738, y 327-370. The bar is read at its +// ends, with no yellow just above or below it, and the icon is looked for +// across the header's left half (the header's text is centred). Text is +// not read, so any language matches. +class MainMissionBannerDetector : public StaticScreenDetector{ +public: + explicit MainMissionBannerDetector(Color color = COLOR_ORANGE); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_icon_left; + ImageFloatBox m_icon_right; + ImageFloatBox m_icon_stroke; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp new file mode 100644 index 0000000000..5485103435 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.cpp @@ -0,0 +1,53 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPixelShares.h" +#include "PokemonLZA_AutoStoryMissionCompleteDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// On 1920x1080: the word's green fills x 615-1300, +// y 271-349, none of it 20 px above or below; the bar's yellow fills +// y 390-585, read left and right of the name as for the banner. +MissionCompleteDetector::MissionCompleteDetector(Color color) + : m_color(color) + , m_word(640.0 / 1920, 285.0 / 1080, 640.0 / 1920, 55.0 / 1080) + , m_above(620.0 / 1920, 245.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_below(620.0 / 1920, 358.0 / 1080, 680.0 / 1920, 20.0 / 1080) + , m_bar_left(430.0 / 1920, 440.0 / 1080, 390.0 / 1920, 140.0 / 1080) + , m_bar_right(1100.0 / 1920, 440.0 / 1080, 400.0 / 1920, 140.0 / 1080) +{} + +void MissionCompleteDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_word); + items.add(m_color, m_above); + items.add(m_color, m_below); + items.add(m_color, m_bar_left); + items.add(m_color, m_bar_right); +} + +// On the card the word is 70% green, nothing around it is green and both +// bar boxes are 100% yellow. Other screens have the word 11% green at +// most; the main mission banners, which have the same bar, 2% at most. +bool MissionCompleteDetector::detect(const ImageViewRGB32& screen){ + if (complete_green_share(screen, m_word) < 0.40){ + return false; + } + if (complete_green_share(screen, m_above) > 0.10 || complete_green_share(screen, m_below) > 0.10){ + return false; + } + return banner_yellow_share(screen, m_bar_left) >= 0.6 && banner_yellow_share(screen, m_bar_right) >= 0.6; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h new file mode 100644 index 0000000000..e337d5ff41 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryMissionCompleteDetector.h @@ -0,0 +1,49 @@ +/* PokemonLZA AutoStory - Mission complete detector + * + * From: https://github.com/PokemonAutomation/ + * + * The "COMPLETE" card when a main mission ends, which waits for A. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryMissionCompleteDetector_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The card a main mission ends with, which waits for A: a big green +// "COMPLETE" at x 615-1300, y 271-349 on 1920x1080 over the mission's +// yellow bar (x 345-1590, y 390-585, "MAIN MISSION 01" and the mission's +// name), in a yellow hexagon with the mission's item. The word is read +// with nothing green just above or below it, and the bar left and right +// of the name. The word is the same in every language seen (the header +// stays English in Chinese), so its text is not read. +class MissionCompleteDetector : public StaticScreenDetector{ +public: + explicit MissionCompleteDetector(Color color = COLOR_GREEN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_word; + ImageFloatBox m_above; + ImageFloatBox m_below; + ImageFloatBox m_bar_left; + ImageFloatBox m_bar_right; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp new file mode 100644 index 0000000000..28b439265d --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.cpp @@ -0,0 +1,63 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "PokemonLZA_AutoStoryPixelShares.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > pixel.red() + 50 && pixel.green() > pixel.red() + 30 && pixel.blue() > 120 && + pixel.blue() + 20 >= pixel.green(); + }); +} + +double dim_cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > pixel.red() + 40 && pixel.green() > pixel.red() + 25 && pixel.blue() > 70 && + pixel.blue() + 20 >= pixel.green(); + }); +} + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 200 && pixel.blue() > 200; + }); +} + +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.blue() > 200 && pixel.green() > 160 && pixel.blue() > pixel.red() + 60; + }); +} + +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.red() > 200 && pixel.green() > 140 && pixel.blue() < 130 && + pixel.red() > pixel.blue() + 110; + }); +} + +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box){ + return share_of(screen, box, [](Color pixel){ + return pixel.green() > 180 && pixel.green() > pixel.red() + 40 && pixel.green() > pixel.blue() + 70; + }); +} + +ImageFloatBox shifted(ImageFloatBox box, double dx, double dy){ + box.x += dx / 1920; + box.y += dy / 1080; + return box; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h new file mode 100644 index 0000000000..6dfbfc07b6 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPixelShares.h @@ -0,0 +1,81 @@ +/* PokemonLZA AutoStory - Pixel shares + * + * From: https://github.com/PokemonAutomation/ + * + * How much of a box's pixels are of a color: what the badge detectors + * (Detectors/) read. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H +#define PokemonAutomation_PokemonLZA_AutoStoryPixelShares_H + +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The share of `box`'s pixels that `matches` (a Color -> bool). +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches); + +// The share of `box`'s pixels that glow cyan: blue over red by 50, green +// over red by 30, blue over 120, and blue at least green less 20. The last +// keeps out mint and sign green (about 96, 203, 147 on a building banner), +// which have green far over blue. +double cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels that glow cyan, dim too: blue over red by +// 40, green over red by 25, blue over 70, blue at least green less 20. +// The autosave badge's glow over a +// dark or warm scene (a lamp-lit wall: about 28, 70, 85) is too dim for +// cyan_share(). +double dim_cyan_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels whose every channel is above 200. +double white_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the controls hint card's sky blue (about +// 125, 210, 250): blue over 200, green over 160, blue over red by 60. +double sky_blue_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission banner's yellow (about 250, +// 210, 40 over a scene): red over 200, green over 140, blue under 130, +// red over blue by 110. +double banner_yellow_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// The share of `box`'s pixels in the mission complete card's green (about +// 165, 240, 125): green over 180, over red by 40 and over blue by 70. +double complete_green_share(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// `box` moved `dx` pixels (of 1920) right and `dy` pixels (of 1080) down. +ImageFloatBox shifted(ImageFloatBox box, double dx, double dy = 0); + + +template +double share_of(const ImageViewRGB32& screen, const ImageFloatBox& box, Matches&& matches){ + const ImageViewRGB32 area = extract_box_reference(screen, box); + const size_t total = area.width() * area.height(); + if (total == 0){ + return 0; + } + size_t count = 0; + for (size_t y = 0; y < area.height(); y++){ + for (size_t x = 0; x < area.width(); x++){ + if (matches(Color(area.pixel(x, y)))){ + count++; + } + } + } + return (double)count / (double)total; +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp new file mode 100644 index 0000000000..dde0a61d92 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.cpp @@ -0,0 +1,170 @@ +/* PokemonLZA AutoStory - Opening detectors + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonLZA_AutoStoryPlatformArrivalDetector.h" +#include "PokemonLZA_AutoStoryReferenceCells.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// A region of the frame and the cells it is averaged into. On 1920x1080: +// the top band y 0-300 across; the left +// side x 0-640 and the right side x 1280-1920, both y 300-1080. The player +// (x 780-1200, y 330-1080) is in none of them. Each cell is 30x30 (top) or +// 40x39 pixels (sides). +struct PlatformRegion{ + std::string_view name; + ImageFloatBox box; + size_t cols; + size_t rows; +}; + +const PlatformRegion PLATFORM_REGIONS[] = { + {"top", {0.0, 0.0, 1.0, 300.0 / 1080}, 64, 10}, + {"left", {0.0, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, + {"right", {1280.0 / 1920, 300.0 / 1080, 640.0 / 1920, 780.0 / 1080}, 16, 20}, +}; + +// The bounds of platform_matches(), from this code's cell averages on +// 1920x1080, against the platform reference: +// - the closest cutscene scene, the train's interior just before the +// platform: top 112.8, left 82.6, right 71.9; the next, city flyovers: +// top 134.7+, left 95.6+; +// - the reference against itself moved 16 px: top 42.2, left 20.8, +// right 21.2; 10% darker: top 24.6, left 19.9, right 16.4; +// - the platform after loading its autosave (camera turned a little): +// top 67.9-68.1, left 25.8-41.7, right 16.5-16.9. +// So the bounds sit between the loaded view and the train's interior: +// top 90 (23 below it), left 55 (27 below). The right side holds a walking +// NPC and the companion, so it only has to be roughly the platform (120: +// a menu or another place is 140+). +constexpr double PLATFORM_TOP_MAX = 90.0; +constexpr double PLATFORM_LEFT_MAX = 55.0; +constexpr double PLATFORM_RIGHT_MAX = 120.0; + +// The largest distance: every channel of every cell 255 apart, sqrt(3) * 255. +constexpr double MAX_PLATFORM_DISTANCE = 441.7; + +using Cells = ReferenceCells; + +// `region` of `screen`, averaged into its cells, row by row. +Cells region_cells(const ImageViewRGB32& screen, const PlatformRegion& region){ + return grid_cells(screen, region.box, region.cols, region.rows); +} + +// The reference's cells of `region` from ReferenceData/plza-platform.json: +// "regions": {: {"cols": n, "rows": ["rrggbb...", ...]}}. +Cells reference_cells(const JsonObject& regions, const PlatformRegion& region){ + const std::string source = "plza-platform.json"; + const JsonObject& entry = regions.get_object_throw(std::string(region.name), source); + return read_cell_grid(entry, region.cols, region.rows, std::format("{} region {}", source, region.name)); +} + +// The reference's cells, per PLATFORM_REGIONS entry. +const std::vector& platform_reference(){ + static const std::vector reference = []{ + for (const auto& [name, text] : autostory_reference_json()){ + if (name != "plza-platform"){ + continue; + } + const JsonValue json = parse_json(text); + const JsonObject& regions = json.to_object_throw("plza-platform.json").get_object_throw("regions"); + std::vector cells; + for (const PlatformRegion& region : PLATFORM_REGIONS){ + cells.push_back(reference_cells(regions, region)); + } + return cells; + } + throw FileException(nullptr, PA_CURRENT_FUNCTION, "The platform reference is not built in.", "plza-platform.json"); + }(); + return reference; +} + +// The regions of a frame against the platform reference. +PlatformDistances platform_distances(const ImageViewRGB32& screen){ + const std::vector& reference = platform_reference(); + double distance[std::size(PLATFORM_REGIONS)] = {}; + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ + distance[index] = cells_rmsd(reference[index], region_cells(screen, PLATFORM_REGIONS[index])); + } + return PlatformDistances{.top = distance[0], .left = distance[1], .right = distance[2]}; +} + +// Whether `distances` are the platform: top and left within their bounds, +// and the right side within a looser one (the companion and the NPCs there +// move). +bool platform_matches(const PlatformDistances& distances){ + return distances.top <= PLATFORM_TOP_MAX + && distances.left <= PLATFORM_LEFT_MAX + && distances.right <= PLATFORM_RIGHT_MAX; +} + +} // namespace + + +ImageRGB32 platform_reference_image(size_t width, size_t height){ + ImageRGB32 image(width, height); + image.fill(0xff000000); + const std::vector& reference = platform_reference(); + for (size_t index = 0; index < std::size(PLATFORM_REGIONS); index++){ + const PlatformRegion& region = PLATFORM_REGIONS[index]; + const ImagePixelBox box = floatbox_to_pixelbox(width, height, region.box); + for (size_t y = box.min_y; y < box.max_y; y++){ + for (size_t x = box.min_x; x < box.max_x; x++){ + const size_t col = (x - box.min_x) * region.cols / (box.max_x - box.min_x); + const size_t row = (y - box.min_y) * region.rows / (box.max_y - box.min_y); + const FloatPixel& cell = reference[index][row * region.cols + col]; + image.pixel(x, y) = (uint32_t)Color((uint8_t)std::lround(cell.r), (uint8_t)std::lround(cell.g), (uint8_t)std::lround(cell.b)); + } + } + } + return image; +} + +std::string to_string(const PlatformDistances& distances){ + return std::format("top {:.1f}, left {:.1f}, right {:.1f}", distances.top, distances.left, distances.right); +} + + +PlatformArrivalDetector::PlatformArrivalDetector(Color color) + : m_color(color) +{} + +void PlatformArrivalDetector::make_overlays(VideoOverlaySet& items) const{ + for (const PlatformRegion& region : PLATFORM_REGIONS){ + items.add(m_color, region.box); + } +} + +bool PlatformArrivalDetector::detect(const ImageViewRGB32& screen){ + if (screen.width() == 0 || screen.height() == 0){ + // No frame: as far from the platform as a frame can be. + m_last = PlatformDistances{.top = MAX_PLATFORM_DISTANCE, .left = MAX_PLATFORM_DISTANCE, .right = MAX_PLATFORM_DISTANCE}; + return false; + } + m_last = platform_distances(screen); + return platform_matches(m_last); +} + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h new file mode 100644 index 0000000000..53a15e3e23 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryPlatformArrivalDetector.h @@ -0,0 +1,68 @@ +/* PokemonLZA AutoStory - Platform arrival detector + * + * From: https://github.com/PokemonAutomation/ + * + * The station platform once the new game's opening cutscene ends and the + * player has control (PlatformArrivalDetector): the frame's top and sides + * against a reference of that first frame, with the player cropped out. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryPlatformArrivalDetector_H + +#include +#include "Common/Cpp/Color.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// How far a frame's regions are from the platform reference: each region +// is averaged into cells, and the distance is the RMSD over its cells of +// the summed squared channel differences (0 = the same, at most 441). +struct PlatformDistances{ + double top; + double left; + double right; +}; + +// The reference as a `width` x `height` frame: each cell painted its +// color, the player's area black. A frame the detector sees as the +// platform, for tests. +// code-health: allow test_only_api -- the tests' platform frame needs the reference cells that only this .cpp reads +ImageRGB32 platform_reference_image(size_t width, size_t height); + +// "top 12.3, left 4.5, right 30.1", for the log. +std::string to_string(const PlatformDistances& distances); + + +// The station platform with free control, right after the opening +// cutscene, at the default camera. +class PlatformArrivalDetector : public StaticScreenDetector{ +public: + explicit PlatformArrivalDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The distances of the last detect(). + const PlatformDistances& last_distances() const{ + return m_last; + } + +private: + Color m_color; + PlatformDistances m_last{.top = 0, .left = 0, .right = 0}; +}; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.cpp new file mode 100644 index 0000000000..372c430d44 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.cpp @@ -0,0 +1,114 @@ +/* PokemonLZA AutoStory - Reference cells + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonArray.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryReferenceCells.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +namespace{ + +// The "rrggbb" at `offset` of `hex` as a color. Throws ParseException +// naming `source` for a character that is not a hex digit. +FloatPixel hex_pixel(std::string_view hex, size_t offset, std::string_view source){ + const auto channel = [&](size_t index){ + const std::string_view digits = hex.substr(offset + index * 2, 2); + int value = 0; + const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), value, 16); + if (error != std::errc() || end != digits.data() + digits.size()){ + throw ParseException(std::format("{}: \"{}\" is not a hex color channel.", source, digits)); + } + return (double)value; + }; + return FloatPixel(channel(0), channel(1), channel(2)); +} + +} // namespace + + +ReferenceCells grid_cells(const ImageViewRGB32& screen, const ImageFloatBox& box, size_t cols, size_t rows){ + ReferenceCells cells; + cells.reserve(cols * rows); + const double cell_width = box.width / (double)cols; + const double cell_height = box.height / (double)rows; + for (size_t row = 0; row < rows; row++){ + for (size_t col = 0; col < cols; col++){ + const ImageFloatBox cell( + box.x + cell_width * (double)col, box.y + cell_height * (double)row, + cell_width, cell_height + ); + cells.push_back(image_stats(extract_box_reference(screen, cell)).average); + } + } + return cells; +} + +ReferenceCells read_cell_grid(const JsonObject& grid, size_t cols, size_t rows, std::string_view source){ + const std::string where(source); + const JsonArray& hex_rows = grid.get_array_throw("rows", where); + if ((size_t)grid.get_integer_throw("cols", where) != cols || hex_rows.size() != rows){ + throw ParseException(std::format("{}: a grid is not {}x{} cells.", source, cols, rows)); + } + ReferenceCells cells; + cells.reserve(cols * rows); + for (const JsonValue& row : hex_rows){ + const std::string& hex = row.to_string_throw(where); + if (hex.size() != cols * 6){ + throw ParseException(std::format("{}: a row of a grid is not {} colors.", source, cols)); + } + for (size_t col = 0; col < cols; col++){ + cells.push_back(hex_pixel(hex, col * 6, source)); + } + } + return cells; +} + +JsonObject write_cell_grid(const ReferenceCells& cells, size_t cols){ + const auto channel = [](double value){ + return (unsigned)std::lround(std::clamp(value, 0.0, 255.0)); + }; + JsonArray hex_rows; + for (size_t start = 0; start < cells.size(); start += cols){ + std::string hex; + for (size_t index = start; index < std::min(start + cols, cells.size()); index++){ + hex += std::format("{:02x}{:02x}{:02x}", channel(cells[index].r), channel(cells[index].g), channel(cells[index].b)); + } + hex_rows.push_back(JsonValue(std::move(hex))); + } + JsonObject grid; + grid["cols"] = JsonValue((int64_t)cols); + grid["rows"] = JsonValue(std::move(hex_rows)); + return grid; +} + +double cells_rmsd(const ReferenceCells& reference, const ReferenceCells& cells){ + double sum = 0; + for (size_t index = 0; index < cells.size(); index++){ + const double r = reference[index].r - cells[index].r; + const double g = reference[index].g - cells[index].g; + const double b = reference[index].b - cells[index].b; + sum += r * r + g * g + b * b; + } + return std::sqrt(sum / (double)cells.size()); +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.h new file mode 100644 index 0000000000..80e5e5a0b2 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryReferenceCells.h @@ -0,0 +1,60 @@ +/* PokemonLZA AutoStory - Reference cells + * + * From: https://github.com/PokemonAutomation/ + * + * A part of the screen averaged into a grid of cells, and the checked-in + * references (the JSON files in ReferenceData/) the detectors compare such grids to. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryReferenceCells_H +#define PokemonAutomation_PokemonLZA_AutoStoryReferenceCells_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonObject.h" +#include "CommonFramework/ImageTools/FloatPixel.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// The checked-in reference files, ReferenceData/*.json, built into the +// program (cmake/EmbedTextFiles.cmake): (name, text), by name. +// plza-platform holds the platform reference's cell colors per region; +// plza-minimap- a minimap reference (PokemonLZA_AutoStoryMinimapMatch.h). +const std::vector>& autostory_reference_json(); + + +// Cell colors, row by row. +using ReferenceCells = std::vector; + +// `box` of `screen` averaged into `cols` x `rows` cells, row by row. +ReferenceCells grid_cells(const ImageViewRGB32& screen, const ImageFloatBox& box, size_t cols, size_t rows); + +// A grid of a reference file, {"cols": n, "rows": ["rrggbb...", ...]}, +// that must be `cols` x `rows` cells. Throws ParseException naming +// `source` when it is not. +ReferenceCells read_cell_grid(const JsonObject& grid, size_t cols, size_t rows, std::string_view source); + +// `cells`, `cols` to a row, as a grid read_cell_grid() reads: each +// channel rounded to the nearest of 0 to 255. +JsonObject write_cell_grid(const ReferenceCells& cells, size_t cols); + +// How far `cells` are from `reference`, cell for cell: the RMSD over the +// cells of the summed squared channel differences (0 = the same, at most +// 441.7). `cells` is as long as `reference`. +double cells_rmsd(const ReferenceCells& reference, const ReferenceCells& cells); + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp new file mode 100644 index 0000000000..caeaf61408 --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.cpp @@ -0,0 +1,39 @@ +/* PokemonLZA AutoStory - Tutorial card detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "PokemonLZA_AutoStoryTutorialCardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A white panel in the middle over a dark, blurred, even screen. Measured +// on 5 cards and 9 other screens (map, Pokedex, dialogs, overworld, night) +// on console B, 2026-11-10, at full resolution: cards' panel 241-246 +// bright (its text spreads it to 40-48) and edges 61-76 with spread under +// 7; the others' panel at most 185, or edges brighter or busy. +bool tutorial_card_shows(const ImageViewRGB32& frame){ + const ImageStats panel = image_stats(extract_box_reference(frame, ImageFloatBox(0.32, 0.62, 0.36, 0.20))); + if (panel.average.sum() / 3 < 225){ + return false; + } + for (const ImageFloatBox& side : {ImageFloatBox(0.03, 0.30, 0.15, 0.40), ImageFloatBox(0.82, 0.30, 0.15, 0.40)}){ + const ImageStats edge = image_stats(extract_box_reference(frame, side)); + if (edge.average.sum() / 3 > 100 || edge.stddev.sum() / 3 > 12){ + return false; + } + } + return true; +} + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h new file mode 100644 index 0000000000..83af11612b --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/Detectors/PokemonLZA_AutoStoryTutorialCardDetector.h @@ -0,0 +1,36 @@ +/* PokemonLZA AutoStory - Tutorial card detector + * + * From: https://github.com/PokemonAutomation/ + * + * The tutorial cards the story opens (Map & Missions, Main Missions, + * Pokedex, ...). Only A turns their pages (B turns back), so a scene mash + * checks for one before it presses its text button. + * + */ + +#ifndef PokemonAutomation_PokemonLZA_AutoStoryTutorialCardDetector_H +#define PokemonAutomation_PokemonLZA_AutoStoryTutorialCardDetector_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + + +// A tutorial card on screen: a bright panel in the middle over dark, even +// edges. +bool tutorial_card_shows(const ImageViewRGB32& frame); + +// The story backend's reading set to 1 while a tutorial card shows. Only +// A turns its pages; a catch in the tutorial ends on "Battling wild +// Pokemon" cards (03.07). +constexpr std::string_view TUTORIAL_CARD_READING = "scene.tutorial_card"; + + + +} // namespace PokemonLZA +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStory.cpp b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStory.cpp new file mode 100644 index 0000000000..5b4a7bde0a --- /dev/null +++ b/SerialPrograms/Source/VideoGames/PokemonLZA/Programs/AutoStory/PokemonLZA_AutoStory.cpp @@ -0,0 +1,274 @@ +/* PokemonLZA AutoStory + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Time.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/Notifications/ProgramNotifications.h" +#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" +#include "Pokemon/AutoStory/Engine/Pokemon_StoryEngine.h" +#include "Pokemon/Pokemon_Strings.h" +#include "Actions/PokemonLZA_AutoStoryConfirmChoices.h" +#include "Actions/PokemonLZA_AutoStoryNewGame.h" +#include "PokemonLZA_AutoStoryConsoleBackend.h" +#include "PokemonLZA_AutoStoryConsoleGame.h" +#include "PokemonLZA_AutoStoryGame.h" +#include "PokemonLZA_AutoStoryStarts.h" +#include "PokemonLZA_AutoStoryStoryFiles.h" +#include "PokemonLZA_AutoStory.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonLZA{ + +using namespace Pokemon; +using namespace std::chrono_literals; + + +AutoStory_Descriptor::AutoStory_Descriptor() + : SingleSwitchProgramDescriptor( + "PokemonLZA:AutoStory", + std::format("{} LZA", STRING_POKEMON), "Auto Story", + "Programs/PokemonLZA/AutoStory.html", + "Play the main story of Pokemon Legends: Z-A. Prototype: from a new game to the start of Main Mission 01. " + "Checkpoint 01.00a starts a new game (delete the save first) from the HOME screen (its tile among the first 8; a running game is closed) " + "and makes the language, look and name below, up to Confirm Choices. 01.00b starts on Confirm Choices (made by the program or by hand), " + "checks them and confirms. " + "Later checkpoints load the latest save first, unless that is turned off.", + PanelQualityStanding::DEVELOPER, + ProgramControllerClass::StandardController_NoRestrictions, + FeedbackType::REQUIRED, + AllowCommandsWhenRunning::DISABLE_COMMANDS + ) +{} + + +namespace{ + +// The start and end dropdowns' entries: "01.00a New game choices", by id. +// The options hold a reference to it. +const StringSelectDatabase& checkpoint_database(){ + static const StringSelectDatabase database = []{ + StringSelectDatabase entries; + for (const StoryCheckpoint& row : builtin_story_data().checkpoints){ + entries.add_entry(StringSelectEntry(row.id, std::format("{} {}", row.id, row.title))); + } + return entries; + }(); + return database; +} + +// Throws UserSetupError unless rows `first` to `last` are in a table of +// `rows` rows, in order. +void check_checkpoint_range(ConsoleHandle& console, size_t first, size_t last, size_t rows){ + if (last >= rows){ + throw UserSetupError(console, "The checkpoint selection is out of range: pick the start and end checkpoints again."); + } + if (first > last){ + throw UserSetupError(console, "The start checkpoint must not come after the end checkpoint."); + } +} + +// The game language dropdown's entries, in the language list's order. +const EnumDropdownDatabase& new_game_language_database(){ + static const EnumDropdownDatabase database{ + {NewGameLanguage::English, "en", "English"}, + {NewGameLanguage::SpanishSpain, "es-ES", "Espa\xc3\xb1ol (Espa\xc3\xb1" "a)"}, + {NewGameLanguage::SpanishLatinAmerica, "es-419", "Espa\xc3\xb1ol (Latinoam\xc3\xa9rica)"}, + {NewGameLanguage::French, "fr", "Fran\xc3\xa7" "ais"}, + {NewGameLanguage::German, "de", "Deutsch"}, + {NewGameLanguage::Italian, "it", "Italiano"}, + {NewGameLanguage::Japanese, "ja", "Japanese"}, + {NewGameLanguage::Korean, "ko", "Korean (choices by hand, start at 01.00b)"}, + {NewGameLanguage::ChineseSimplified, "zh-Hans", "Chinese (Simplified)"}, + {NewGameLanguage::ChineseTraditional, "zh-Hant", "Chinese (Traditional)"}, + }; + return database; +} + +// Throws UserSetupError, before any press, if the run of `rows` plays a +// new game that `choices` cannot: choices no new game takes +// (new_game_problem()), or ones 01.00a cannot make (new_game_run_problem()). +void check_new_game_choices(ConsoleHandle& console, std::span rows, const NewGameChoices& choices){ + if (rows.empty() || !story_row_starts_without_save(rows.front())){ + return; + } + if (const std::optional problem = new_game_problem(choices)){ + throw UserSetupError(console, *problem); + } + if (const std::optional problem = new_game_run_problem(rows, choices)){ + throw UserSetupError(console, *problem); + } +} + +// Connects the controller for a run from `first_row`. A new game's row +// starts on a screen that proves the save is fresh (01.00a: HOME, 01.00b: +// Confirm Choices), so that screen is checked before any press, the +// connect press included. +void connect_controller(ConsoleHandle& console, ProControllerContext& context, const StoryCheckpoint& first_row){ + if (story_row_makes_new_game_choices(first_row)){ + require_new_game_home(console, context); + }else if (story_row_starts_without_save(first_row)){ + require_confirm_choices(console, context); + } + require_player(console, context, BUTTON_L); +} + +constexpr std::string_view ATTEMPTS_PER_CHECKPOINT_LABEL = + "Attempts per Checkpoint:
A failed checkpoint that started from a save reloads it and tries again, up to this many times in all."; + +// Loads the latest save for `row` with `game`: a run that starts after +// the new game starts from it (01.01: the platform autosave), not from +// wherever the game is. Throws OperationFailedException, with an error +// report (the screenshot and the stream history's clip), if that save +// does not load. +void load_first_save(ConsoleHandle& console, const StoryGame& game, PlzaBackend& backend, const StoryCheckpoint& row){ + const Outcome loaded = load_checkpoint_start(game, backend, row); + if (!loaded.succeeded()){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::format("AutoStory: could not load the save for {}: {}", row.id, to_string(loaded)), + console + ); + } + console.log(std::format("AutoStory: loaded the latest save for {}.", row.id)); +} + +// Where saving_frame_before() saves its frames. +std::string start_minimaps_folder(){ + return std::format("{}AutoStoryDev/", USER_FILE_PATH()); +} + + + +// `reload`, after the frame on screen is saved in start_minimaps_folder() +// as plza-before-reload-