diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml new file mode 100644 index 0000000000..cb577266da --- /dev/null +++ b/.github/workflows/code-health.yml @@ -0,0 +1,53 @@ +name: C++ Code Health +# Stub: the linter (devtools/code_health) and its self-tests live on the fork's agents branch. +# This checks that branch out next to the code and runs it against the code checkout. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + inputs: + base-ref: + description: Revision to compare with (e.g. origin/main); empty = whole tree + required: false + +permissions: + contents: read + +jobs: + code-health: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/checkout@v7 + with: + repository: dayfine/Arduino-Source + ref: agents + path: .agents-tools + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: .agents-tools/devtools/code_health/requirements.txt + - run: python -m pip install -r .agents-tools/devtools/code_health/requirements.txt + - name: Linter self-tests + working-directory: .agents-tools + env: + CODE_HEALTH_REPO: ${{ github.workspace }} + run: python -m unittest discover -s devtools/code_health/tests -v + - name: Check changed C++ files + working-directory: .agents-tools + env: + PR_BASE: ${{ github.event.pull_request.base.sha }} + INPUT_BASE: ${{ inputs.base-ref }} + PUSH_BASE: ${{ github.event.before }} + CODE: ${{ github.workspace }} + shell: bash + run: | + base="${PR_BASE:-${INPUT_BASE:-$PUSH_BASE}}" + if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]]; then + base=$(git -C "$CODE" hash-object -w -t tree --stdin /Resources/Tesseract`. +- `SerialProgramsTests` (built by default next to `SerialPrograms`; `-DTEST_RUNNER_EXE=OFF` + turns it off) runs the same tests without the GUI, settings or device scans, so it can run + while SerialPrograms is open: `ninja SerialProgramsTests`, then + `SerialProgramsTests --filter ` (repeatable) or `--list`. Its exit code is the number + of failed tests. Same requirements as above. - Test framework: test classes derive `UnitTest` (`Common/Cpp/TestRunners/UnitTest.h`) and are collected via per-module `add_tests(UnitTestDatabase&)` — see `SerialPrograms/Source/Tests/CommandLineTests.h`. diff --git a/Common/Cpp/Filesystem/Filesystem.h b/Common/Cpp/Filesystem/Filesystem.h index 21d350f014..b6cedfd202 100644 --- a/Common/Cpp/Filesystem/Filesystem.h +++ b/Common/Cpp/Filesystem/Filesystem.h @@ -22,6 +22,7 @@ #ifndef PokemonAutomation_Filesystem_Filesystem_H #define PokemonAutomation_Filesystem_Filesystem_H +#include #include #include #include "FilePath.h" @@ -115,11 +116,11 @@ Path application_install_path(); Path application_scratch_path(); -// Set a profile for program settings (/UserSettings/PROFILE_NAME/) on MacOS. -// Have to run the program with command-line argument "open -n PATH_TO_APP --args --profile PROFILE_NAME" to set the profile and launch a new window. -// This allows multiple instances of the program to run since settings are no longer shared. #if defined(__APPLE__) -void set_startup_profile(int& argc, char* argv[]); +// Set a validated macOS settings profile before any settings paths are cached. +// GUI launch uses --profile NAME or the build-bound PA_LAB_PROFILE identity. +void set_startup_profile(std::string_view profile); +// Return the identity used for the window title and scratch directory. const std::string& STARTUP_PROFILE(); #endif diff --git a/Common/Cpp/Filesystem/Filesystem_Mac.h b/Common/Cpp/Filesystem/Filesystem_Mac.h index a9dbc281ab..472233e0cc 100644 --- a/Common/Cpp/Filesystem/Filesystem_Mac.h +++ b/Common/Cpp/Filesystem/Filesystem_Mac.h @@ -21,22 +21,9 @@ namespace Filesystem{ static std::string g_startup_profile; -void set_startup_profile(int& argc, char* argv[]){ - for (int i = 1; i + 1 < argc; i++){ - if (strcmp(argv[i], "--profile") == 0){ - std::string profile = argv[i + 1]; - for (char c : profile){ - if (!(std::isalpha(c) || std::isdigit(c)) && c != u'_' && c != u'-') c = u'_'; - } - g_startup_profile = std::move(profile); - // Shift everything after --profile down by 2. - for (int j = i; j + 2 < argc; j++){ - argv[j] = argv[j + 2]; - } - argc -= 2; - return; - } - } +// Store the already-validated identity before any scratch/settings path is cached. +void set_startup_profile(std::string_view profile){ + g_startup_profile = profile; } const std::string& STARTUP_PROFILE(){ diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d7fc0c3e76..539f6ec13a 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -66,6 +66,8 @@ find_package(Threads REQUIRED) # Define a toggle option (ON by default) option(COMMAND_LINE_EXE "Build the optional executable" ON) +# Build SerialProgramsTests, which runs the unit tests without the GUI. +option(TEST_RUNNER_EXE "Build the unit test executable, SerialProgramsTests" ON) # Build the _pa_core Python (pybind11) module; see Source/PythonBindings/README.md. # OFF by default because it needs a Python interpreter with development headers # and pybind11 (a system package or `pip install pybind11`). @@ -236,6 +238,27 @@ if (APPLE) # https://stackoverflow.com/questions/53560288/how-to-create-a-macos-app-bundle-with-cmake # https://cmake.org/cmake/help/latest/command/add_executable.html add_executable(SerialPrograms MACOSX_BUNDLE ${SerialPrograms_ICON} ${EXECUTABLE_SOURCES}) + # Compiled identity survives Finder/TCC relaunch without command-line arguments. + set(PA_LAB_PROFILE "" CACHE STRING "Bind this lab binary to one macOS settings profile") + if(NOT PA_LAB_PROFILE STREQUAL "") + if(NOT PA_LAB_PROFILE MATCHES "^[A-Za-z0-9_][A-Za-z0-9_-]*$") + message(FATAL_ERROR "PA_LAB_PROFILE must contain only ASCII letters, digits, '_' or '-'.") + endif() + target_compile_definitions(SerialPrograms PRIVATE PA_LAB_PROFILE="${PA_LAB_PROFILE}") + endif() + + # Receipt binds packaging to the linked binary, not just a CMake cache value. + set_property(TARGET SerialPrograms APPEND PROPERTY LINK_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LabBuildReceipt.cmake") + add_custom_command(TARGET SerialPrograms POST_BUILD + COMMAND "${CMAKE_COMMAND}" + "-DBINARY=$" + "-DPROFILE=${PA_LAB_PROFILE}" + "-DSOURCE=${CMAKE_CURRENT_SOURCE_DIR}/.." + -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LabBuildReceipt.cmake" + VERBATIM + ) + set_target_properties(SerialPrograms PROPERTIES BUNDLE True MACOSX_BUNDLE_GUI_IDENTIFIER PokemonAutomation.SerialPrograms @@ -368,6 +391,10 @@ if(NOT EXISTS "${DEST_DIR}/Scripts") ) endif() +if(TEST_RUNNER_EXE) + include(Source/Tests/TestRunnerExecutable.cmake) +endif() + if (WIN32) if (PA_SDL) add_library(SDL3_lib IMPORTED UNKNOWN) @@ -944,6 +971,15 @@ if(COMMAND_LINE_EXE) include(Source/CommandLine/CommandLineExecutable.cmake) endif() +# The registered cross-process tests use this GUI-free native runner as their child. +add_subdirectory(Source/CommonFramework/Devices) +if(WIN32) + set_target_properties(LabDeviceAccessTests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${WIN_DEPLOY_DIR}") +else() + set_target_properties(LabDeviceAccessTests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") +endif() +add_dependencies(SerialPrograms LabDeviceAccessTests) + if(PA_PYTHON_BINDINGS) # Python bindings (GUI-free) from subdirectory include(Source/PythonBindings/PythonBindings.cmake) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp index 8327b7e48a..b2a1cfad58 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp @@ -11,6 +11,8 @@ #include "Sources/AudioSource_File.h" #include "Sources/AudioSource_Device.h" #include "AudioOption.h" +#include "AudioPipelineOptions.h" +#include "CommonFramework/GlobalSettingsPanel.h" //#include //using std::cout; @@ -55,6 +57,7 @@ std::string AudioOption::audioDisplayTypeToString(AudioOption::AudioDisplayType AudioOption::AudioOption() : m_input_source(new AudioSourceDescriptor_Null()) + , m_display_type(GlobalSettings::instance().AUDIO_PIPELINE->DEFAULT_DISPLAY) {} void AudioOption::set_input(std::shared_ptr descriptor){ diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h index 73e002f2a8..c63b0a83ee 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h @@ -72,7 +72,7 @@ class AudioOption{ AudioInputChoice m_input_choice; AudioDeviceInfo m_output_device; - AudioDisplayType m_display_type = AudioDisplayType::FREQ_BARS; + AudioDisplayType m_display_type; // the global default until load_json() sets one double m_volume = 1.0; // Volume Range: [0, 1.0] }; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h index 5f4a889b8c..d0f7e3bd73 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h @@ -11,6 +11,8 @@ #include "Common/Cpp/Options/BooleanCheckBoxOption.h" #include "Common/Cpp/Options/SimpleIntegerOption.h" #include "Common/Cpp/Options/FloatingPointOption.h" +#include "Common/Cpp/Options/EnumDropdownOption.h" +#include "CommonFramework/AudioPipeline/AudioOption.h" #include "CommonFramework/StaticGlobals.h" namespace PokemonAutomation{ @@ -26,6 +28,17 @@ class AudioPipelineOptions : public GroupOption{ true, false ) + , DEFAULT_DISPLAY( + "Default Audio Display:
" + "What a program's console shows for its audio until you pick a display in that console's settings.", + { + {AudioOption::AudioDisplayType::NO_DISPLAY, "none", "No Display"}, + {AudioOption::AudioDisplayType::FREQ_BARS, "freq-bars", "Frequency Bars"}, + {AudioOption::AudioDisplayType::SPECTROGRAM, "spectrogram", "Spectrogram"}, + }, + LockMode::UNLOCK_WHILE_RUNNING, + AudioOption::AudioDisplayType::FREQ_BARS + ) , FILE_VOLUME_SCALE( "Audio File Input Volume Scale:
" "Multiply audio file playback by this factor. (This is linear scale. So each factor of 10 is 20dB.)", @@ -58,6 +71,7 @@ class AudioPipelineOptions : public GroupOption{ 5 ) { + PA_ADD_OPTION(DEFAULT_DISPLAY); PA_ADD_OPTION(FILE_VOLUME_SCALE); PA_ADD_OPTION(DEVICE_VOLUME_SCALE); PA_ADD_OPTION(USE_KAISER_WINDOW); @@ -68,6 +82,7 @@ class AudioPipelineOptions : public GroupOption{ } public: + EnumDropdownOption DEFAULT_DISPLAY; FloatingPointOption FILE_VOLUME_SCALE; FloatingPointOption DEVICE_VOLUME_SCALE; BooleanCheckBoxOption USE_KAISER_WINDOW; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index 4bb1f84370..51bfcf9936 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -406,16 +406,12 @@ void AudioPassthroughPairQt::reset( auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this, format, descriptor, output, output_volume]{ auto scope_check = m_sanitizer.check_scope(); + close_audio_source(); std::unique_ptr source; if (descriptor){ source = descriptor->make_AudioSource(m_logger, format); } WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_reader){ - teardown_tree(); - m_sink_device.reset(); - m_reader.reset(); - } m_input_choice = format; m_output_device = output; m_output_volume = output_volume; @@ -427,6 +423,15 @@ void AudioPassthroughPairQt::reset( } }); } +// Called on this object's Qt thread before constructing a replacement input. +void AudioPassthroughPairQt::close_audio_source(){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + if (m_reader){ + teardown_tree(); + m_sink_device.reset(); + m_reader.reset(); + } +} void AudioPassthroughPairQt::clear_audio_source(){ auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this]{ @@ -447,16 +452,12 @@ void AudioPassthroughPairQt::set_audio_source( auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this, descriptor, format]{ auto scope_check = m_sanitizer.check_scope(); + close_audio_source(); std::unique_ptr source; if (descriptor){ source = descriptor->make_AudioSource(m_logger, format); } WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_reader){ - teardown_tree(); - m_sink_device.reset(); - m_reader.reset(); - } m_input_choice = format; if (source){ m_reader = std::move(source); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h index 014b9c2da7..872904e33f 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h @@ -97,6 +97,7 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair // thread only ever sees them through the node listener sets, where // ListenerSet::remove blocks on in-flight pushes so rebuilds never // race a live push. + void close_audio_source(); void teardown_tree(); void teardown_sink_branch(); void build_tree(float input_volume); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp index 758ccc70b8..0795682ce4 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp @@ -53,6 +53,7 @@ AudioInputDevice::AudioInputDevice( RAW_BUFFER_CAPACITY ) , ObjectStreamListener(raw_output_frame_bytes(format, reinterpret_96k_mono_as_stereo)) + , m_device_lease(acquire_lab_capture(logger, LabDeviceKind::AUDIO, device.id().toHex().toStdString())) , m_sample_type(sample_format_of(format)) , m_sample_rate(format.sampleRate()) , m_channels(format.channelCount()) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h index 2ef2c7365b..fba2d91614 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h @@ -17,6 +17,7 @@ using NativeAudioSource = QAudioSource; #include "Common/Cpp/StreamConverters.h" #include "Kernels/AudioPipeline/Kernels_AudioPipeline.h" #include "CommonFramework/AudioPipeline/AudioFormat.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" class QAudioFormat; @@ -87,6 +88,7 @@ class AudioInputDevice final private: + std::unique_ptr m_device_lease; std::unique_ptr m_source; Kernels::AudioPipeline::SampleFormat m_sample_type; size_t m_sample_rate; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp index 18706767e7..afca55d3ba 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp @@ -68,7 +68,11 @@ std::unique_ptr AudioSourceDescriptor_Device::make_AudioSource( return nullptr; } // cout << "format = " << (int)format << endl; - return std::make_unique(logger, m_info, format); + try{ + return std::make_unique(logger, m_info, format); + }catch (UserSetupError&){ + return nullptr; // Refusal is logged; never throw across the Qt audio callback. + } } diff --git a/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt b/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt new file mode 100644 index 0000000000..dba4f55b10 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt @@ -0,0 +1,20 @@ +# Focused native tests, without configuring/linking the GUI application: +# cmake -S SerialPrograms/Source/CommonFramework/Devices -B /path/to/test-build +# cmake --build /path/to/test-build -j4 +# ctest --test-dir /path/to/test-build --output-on-failure +cmake_minimum_required(VERSION 3.24) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(LabDeviceAccessTests LANGUAGES CXX) + enable_testing() + find_package(Qt6 REQUIRED COMPONENTS Core) +endif() +add_executable(LabDeviceAccessTests LabDeviceAccessQt.cpp LabDeviceAccessQt_Tests.cpp) +target_compile_features(LabDeviceAccessTests PRIVATE cxx_std_23) +target_compile_definitions(LabDeviceAccessTests PRIVATE PA_LAB_DEVICE_STANDALONE_TESTS) +target_link_libraries(LabDeviceAccessTests PRIVATE Qt6::Core) +if(MSVC) + target_compile_options(LabDeviceAccessTests PRIVATE /W4 /WX) +else() + target_compile_options(LabDeviceAccessTests PRIVATE -Wall -Wextra -Werror) +endif() +add_test(NAME LabDeviceAccess COMMAND LabDeviceAccessTests) diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp new file mode 100644 index 0000000000..d420befadf --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp @@ -0,0 +1,153 @@ +/* Lab Device Access (Qt process-lock seam) + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "LabDeviceAccessQt.h" + +namespace PokemonAutomation{ +namespace{ + +QString text(std::string_view value){ + return QString::fromUtf8(value.data(), (qsizetype)value.size()); +} + +std::set read_ids(const QJsonObject& object, const char* key){ + if (!object.value(key).isArray()){ + throw std::runtime_error(std::string("Expected device array: ") + key); + } + std::set ids; + for (const QJsonValue& value : object.value(key).toArray()){ + const std::string id = value.toString().toStdString(); + if (!value.isString() || id.empty() || id.find('\0') != std::string::npos || !ids.emplace(id).second){ + throw std::runtime_error(std::string("Empty, invalid or duplicate device ID in ") + key); + } + } + return ids; +} + +bool permitted(const LabDevicePolicy& policy, const LabDeviceIdentity& device){ + switch (device.kind){ + case LabDeviceKind::SERIAL: return policy.serial.contains(device.id); + case LabDeviceKind::VIDEO: return policy.video.contains(device.id); + case LabDeviceKind::AUDIO: return policy.audio.contains(device.id); + case LabDeviceKind::PROFILE: return false; + } + return false; +} + +// Capture IDs are allowlisted as hex; show the readable ID beside it so a +// refusal names the device the settings selected. +std::string readable_capture_id(const std::string& hex){ + const bool is_hex = hex.size() % 2 == 0 && std::ranges::all_of(hex, [](char c){ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + }); + if (!is_hex){ + return hex; + } + const QByteArray bytes = QByteArray::fromHex(QByteArray(hex.data(), (qsizetype)hex.size())); + return bytes.toStdString() + " (allowlist ID " + hex + ")"; +} + +std::string describe(const LabDeviceIdentity& device){ + switch (device.kind){ + case LabDeviceKind::SERIAL: return "Serial device " + device.id; + case LabDeviceKind::VIDEO: return "Video input " + readable_capture_id(device.id); + case LabDeviceKind::AUDIO: return "Audio input " + readable_capture_id(device.id); + case LabDeviceKind::PROFILE: return "Profile " + device.id; + } + return device.id; +} + +std::string normalize_serial_path(std::string_view path){ + if (path.starts_with("/dev/tty.")){ + return "/dev/cu." + std::string(path.substr(9)); + } + return std::string(path); +} + +} // namespace + +LabDevicePolicy parse_lab_device_policy(std::string_view json, std::string_view profile){ + QJsonParseError error; + const QJsonDocument document = QJsonDocument::fromJson(QByteArray(json.data(), (qsizetype)json.size()), &error); + const QJsonObject object = document.object(); + const std::set keys{"schema", "profile", "serial", "video", "audio"}; + for (auto it = object.begin(); it != object.end(); ++it){ + if (!keys.contains(it.key())){ + throw std::runtime_error("Unknown field in lab device policy."); + } + } + if (error.error != QJsonParseError::NoError || !document.isObject() || object.size() != 5 || + !object.value("schema").isDouble() || object.value("schema").toDouble() != 1 || + profile.empty() || object.value("profile").toString() != text(profile)){ + throw std::runtime_error("Missing, malformed or mismatched lab device policy."); + } + return {std::string(profile), read_ids(object, "serial"), read_ids(object, "video"), read_ids(object, "audio")}; +} + +std::string lab_serial_identity(unsigned vendor, unsigned product, std::string_view serial){ + if (vendor > 0xffff || product > 0xffff || serial.empty() || serial.find('\0') != std::string_view::npos){ + throw std::runtime_error("A unique USB serial, vendor and product identity are required."); + } + return std::format("{:04x}:{:04x}:{}", vendor, product, serial); +} + +std::string resolve_lab_serial_identity(std::span devices, std::string_view port){ + const std::string path = normalize_serial_path(port); + const auto selected = std::ranges::find_if(devices, [&](const LabSerialDevice& device){ + return normalize_serial_path(device.port) == path; + }); + if (selected == devices.end()){ + throw std::runtime_error("Controller has no current USB identity; hardware access is disabled."); + } + const std::string identity = lab_serial_identity(selected->vendor, selected->product, selected->serial); + for (const LabSerialDevice& other : devices){ + if (normalize_serial_path(other.port) != path && !other.serial.empty() && + lab_serial_identity(other.vendor, other.product, other.serial) == identity){ + throw std::runtime_error("Two adapters report the same USB identity; refusing ambiguous ownership."); + } + } + return identity; +} + +std::string lab_device_lock_path(std::string_view directory, const LabDeviceIdentity& device){ + if (directory.empty() || device.id.empty()){ + throw std::runtime_error("Device lease directory and identity must be nonempty."); + } + const QByteArray key = QByteArray::number((int)device.kind) + ':' + QByteArray(device.id.data(), (qsizetype)device.id.size()); + const QByteArray digest = QCryptographicHash::hash(key, QCryptographicHash::Sha256).toHex(); + return QDir(text(directory)).filePath(QString::fromLatin1(digest) + ".lock").toStdString(); +} + +LabDeviceLease::LabDeviceLease(std::string_view path) + : m_lock(text(path)) +{ + // Age alone must never evict an active multi-hour lab run. + m_lock.setStaleLockTime(std::chrono::milliseconds::zero()); + if (!m_lock.tryLock(std::chrono::milliseconds::zero())){ + throw std::runtime_error("Device/profile is already leased, or its lock cannot be created. Nothing was opened."); + } +} + +std::unique_ptr acquire_lab_device( + const LabDevicePolicy& policy, std::string_view directory, const LabDeviceIdentity& device +){ + if (!permitted(policy, device)){ + throw std::runtime_error(describe(device) + " is not allowlisted for profile " + policy.profile + ". Nothing was opened."); + } + if (!QDir().mkpath(text(directory))){ + throw std::runtime_error("Cannot create shared device lease directory. Nothing was opened."); + } + return std::make_unique(lab_device_lock_path(directory, device)); +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h new file mode 100644 index 0000000000..669ce69f9d --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h @@ -0,0 +1,72 @@ +/* Lab Device Access (Qt process-lock seam) + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceAccessQt_H +#define PokemonAutomation_LabDeviceAccessQt_H + +#include +#include +#include +#include +#include +#include + +namespace PokemonAutomation{ + +// Separate namespaces for serial, video, audio and whole-profile reservations. +enum class LabDeviceKind{ SERIAL, VIDEO, AUDIO, PROFILE }; + +// Stable hardware identity, never a display alias, ordinal or USB port pathname. +struct LabDeviceIdentity{ + LabDeviceKind kind; + std::string id; +}; + +// Immutable allowlist loaded once for the selected lab profile. +struct LabDevicePolicy{ + std::string profile; + std::set serial; + std::set video; + std::set audio; +}; + +// One currently enumerated USB port; tty/cu aliases may describe the same adapter. +struct LabSerialDevice{ + std::string port; + unsigned vendor; + unsigned product; + std::string serial; +}; + +// Resolve a selected current port, rejecting missing or ambiguous physical identity. +std::string resolve_lab_serial_identity(std::span devices, std::string_view port); + +// Parse the strict schema; malformed/mismatched input throws std::runtime_error. +LabDevicePolicy parse_lab_device_policy(std::string_view json, std::string_view profile); + +// Normalize VID/PID and a nonempty USB serial; empty identifiers are refused. +std::string lab_serial_identity(unsigned vendor, unsigned product, std::string_view serial); + +// Compute a shared lock path independent of profile, executable and display names. +std::string lab_device_lock_path(std::string_view directory, const LabDeviceIdentity& device); + +// A noncopyable process lease. Destruction releases it; dead owners are recovered by Qt. +// Keep this alive until after the native device has closed. Acquisition never waits. +class LabDeviceLease{ +public: + explicit LabDeviceLease(std::string_view path); + LabDeviceLease(const LabDeviceLease&) = delete; + LabDeviceLease& operator=(const LabDeviceLease&) = delete; +private: + QLockFile m_lock; +}; + +// Check the allowlist before acquiring a lease. A denial never touches hardware. +// Throws std::runtime_error on denied access, contention or filesystem failure. +std::unique_ptr acquire_lab_device( + const LabDevicePolicy& policy, std::string_view directory, const LabDeviceIdentity& device +); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp new file mode 100644 index 0000000000..83f5613e96 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp @@ -0,0 +1,328 @@ +/* Lab Device Access Tests + * + * From: https://github.com/PokemonAutomation/ + * + * PR #226 startup incident: a Console B test build opened Console A's controller. + * These tests never enumerate or open physical devices, or load real profiles. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "LabDeviceAccessQt.h" +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +#include "LabDeviceAccessQt_Tests.h" +#endif + +namespace PokemonAutomation{ +namespace{ + +struct PolicyInput{ + std::string_view name; + std::string_view json; + LabDeviceIdentity requested; +}; + +struct PolicyExpectation{ + int opens = 0; + std::string_view refusal = {}; // The refusal message starts with this. +}; + +struct PolicyCase{ + PolicyInput input; + PolicyExpectation expectation; +}; + +struct SerialInput{ + std::string_view name; + std::vector devices; + std::string_view port; +}; + +struct SerialExpectation{ + std::optional identity = std::nullopt; +}; + +struct SerialCase{ + SerialInput input; + SerialExpectation expectation; +}; + +const SerialCase SERIAL_CASES[] = { + {.input = {.name = "USB serial identity", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "A"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:A"}}, + {.input = {.name = "tty and cu share a key", .devices = {{"/dev/tty.A", 0x10c4, 0xea60, "A"}, {"/dev/cu.A", 0x10c4, 0xea60, "A"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:A"}}, + {.input = {.name = "two adapters with duplicate serial", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "same"}, {"/dev/cu.B", 0x10c4, 0xea60, "same"}}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "missing USB serial", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, ""}}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "unplugged adapter", .devices = {}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "path reused by another adapter", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "B"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:B"}}, +}; + +std::string check_serial(const SerialCase& test){ + std::optional identity; + try{ + identity = resolve_lab_serial_identity(test.input.devices, test.input.port); + }catch (const std::runtime_error&){} + if (identity != test.expectation.identity){ + return "Expected identity " + test.expectation.identity.value_or("") + "; read " + identity.value_or(""); + } + return {}; +} + +const PolicyCase POLICY_CASES[] = { + {.input = {.name = "matching serial", .json = R"({"schema":1,"profile":"lab-B","serial":["10c4:ea60:B"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "10c4:ea60:B"}}, .expectation = {.opens = 1}}, + {.input = {.name = "wrong console serial", .json = R"({"schema":1,"profile":"lab-B","serial":["10c4:ea60:B"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "10c4:ea60:A"}}, .expectation = {.refusal = "Serial device 10c4:ea60:A is not allowlisted for profile lab-B."}}, + {.input = {.name = "camera allowed", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":["766964656f42"],"audio":[]})", .requested = {LabDeviceKind::VIDEO, "766964656f42"}}, .expectation = {.opens = 1}}, + {.input = {.name = "audio allowed", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":[],"audio":["617564696f42"]})", .requested = {LabDeviceKind::AUDIO, "617564696f42"}}, .expectation = {.opens = 1}}, + // lab-B, 2026-11-08: a panel selected console A's audio input, and the refusal didn't say which device it was. + {.input = {.name = "wrong console audio", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":[],"audio":["617564696f42"]})", .requested = {LabDeviceKind::AUDIO, "617564696f41"}}, .expectation = {.refusal = "Audio input audioA (allowlist ID 617564696f41) is not allowlisted for profile lab-B."}}, + {.input = {.name = "camera not audio", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":["same-id"],"audio":[]})", .requested = {LabDeviceKind::AUDIO, "same-id"}}, .expectation = {}}, + {.input = {.name = "mismatched profile", .json = R"({"schema":1,"profile":"lab-A","serial":["port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "missing policy", .json = "", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "malformed policy", .json = "{", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "unsupported schema", .json = R"({"schema":2,"profile":"lab-B","serial":["port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "duplicate identity", .json = R"({"schema":1,"profile":"lab-B","serial":["port","port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "missing audio list", .json = R"({"schema":1,"profile":"lab-B","serial":["port"],"video":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "empty identity", .json = R"({"schema":1,"profile":"lab-B","serial":[""],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, ""}}, .expectation = {}}, + {.input = {.name = "unknown policy field", .json = R"({"schema":1,"profile":"lab-B","serial":["port"],"video":[],"audio":[],"allow_all":true})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, +}; + +std::string check_policy(const PolicyCase& test){ + QTemporaryDir directory; + if (!directory.isValid()){ + return "Cannot create temporary lease directory."; + } + int opens = 0; + std::string refusal; + try{ + const auto policy = parse_lab_device_policy(test.input.json, "lab-B"); + auto lease = acquire_lab_device(policy, directory.path().toStdString(), test.input.requested); + opens++; // Scripted native open: no real port, camera or microphone exists. + }catch (const std::runtime_error& error){ + refusal = error.what(); + } + if (opens != test.expectation.opens){ + return "Expected opens " + std::to_string(test.expectation.opens) + "; read " + std::to_string(opens); + } + if (!refusal.starts_with(test.expectation.refusal)){ + return "Expected a refusal starting \"" + std::string(test.expectation.refusal) + "\"; read \"" + refusal + "\""; + } + return {}; +} + +enum class LeaseScenario{ SAME_PROCESS, DIFFERENT_DEVICE, DIFFERENT_KIND, RELEASE, FAILED_OPEN, LONG_RUNNING, LIVE_CHILD, DEAD_CHILD, PROFILE, EARLY_CHILD }; + +struct LeaseInput{ + std::string_view name; + LeaseScenario scenario; +}; + +struct LeaseExpectation{ + bool second_opens; +}; + +struct LeaseCase{ + LeaseInput input; + LeaseExpectation expectation; +}; + +const LeaseCase LEASE_CASES[] = { + {.input = {.name = "same device two profiles", .scenario = LeaseScenario::SAME_PROCESS}, .expectation = {.second_opens = false}}, + {.input = {.name = "different consoles", .scenario = LeaseScenario::DIFFERENT_DEVICE}, .expectation = {.second_opens = true}}, + {.input = {.name = "video and audio inputs", .scenario = LeaseScenario::DIFFERENT_KIND}, .expectation = {.second_opens = true}}, + {.input = {.name = "disconnect then reconnect", .scenario = LeaseScenario::RELEASE}, .expectation = {.second_opens = true}}, + {.input = {.name = "failed native open releases lease", .scenario = LeaseScenario::FAILED_OPEN}, .expectation = {.second_opens = true}}, +#if !defined(_WIN32) + // Windows' live lock prevents editing this timestamp fixture; exercise it on POSIX. + {.input = {.name = "age never steals live lease", .scenario = LeaseScenario::LONG_RUNNING}, .expectation = {.second_opens = false}}, +#endif + {.input = {.name = "live second process", .scenario = LeaseScenario::LIVE_CHILD}, .expectation = {.second_opens = false}}, + {.input = {.name = "profile reserved before Qt startup", .scenario = LeaseScenario::EARLY_CHILD}, .expectation = {.second_opens = false}}, + {.input = {.name = "killed owner recovery", .scenario = LeaseScenario::DEAD_CHILD}, .expectation = {.second_opens = true}}, + {.input = {.name = "duplicate profile", .scenario = LeaseScenario::PROFILE}, .expectation = {.second_opens = false}}, +}; + +std::string test_helper_path(){ + QString name = "LabDeviceAccessTests"; +#if defined(_WIN32) + name += ".exe"; +#endif + return QDir(QCoreApplication::applicationDirPath()).filePath(name).toStdString(); +} + +void start_lease_child(QProcess& child, const std::string& path, LeaseScenario scenario){ + const QString flag = scenario == LeaseScenario::EARLY_CHILD ? "--lease-before-qt" : "--lease-child"; + child.start(QString::fromStdString(test_helper_path()), {flag, QString::fromStdString(path)}); + if (!child.waitForStarted(5000) || !child.waitForReadyRead(5000) || child.readAllStandardOutput().trimmed() != "LOCKED"){ + throw std::runtime_error("Lease test child did not confirm ownership."); + } +} + +std::unique_ptr prepare_owner(LeaseScenario scenario, const std::string& path, QProcess& child){ + if (scenario == LeaseScenario::LIVE_CHILD || scenario == LeaseScenario::DEAD_CHILD || scenario == LeaseScenario::EARLY_CHILD){ + start_lease_child(child, path, scenario); + if (scenario == LeaseScenario::DEAD_CHILD){ + child.kill(); + if (!child.waitForFinished(5000)){ + throw std::runtime_error("Child did not exit after kill."); + } + } + return nullptr; + } + auto owner = std::make_unique(path); + if (scenario == LeaseScenario::FAILED_OPEN){ + try{ + auto opening = std::move(owner); + throw std::runtime_error("Scripted native open failed."); + }catch (const std::runtime_error&){} + } + if (scenario == LeaseScenario::RELEASE){ + owner.reset(); + } + if (scenario == LeaseScenario::LONG_RUNNING){ + QFile lock(QString::fromStdString(path)); + if (!lock.open(QIODevice::ReadOnly) || !lock.setFileTime(QDateTime::currentDateTimeUtc().addDays(-2), QFileDevice::FileModificationTime)){ + throw std::runtime_error("Could not age the lock fixture."); + } + } + return owner; +} + +std::string check_lease(const LeaseCase& test){ + QTemporaryDir directory; + if (!directory.isValid()){ + return "Cannot create temporary lease directory."; + } + LabDevicePolicy policy{"lab-B", {"port-A", "port-B"}, {"port-A"}, {}}; + LabDeviceIdentity first{LabDeviceKind::SERIAL, "port-A"}; + if (test.input.scenario == LeaseScenario::PROFILE){ + first.kind = LabDeviceKind::PROFILE; + } + const std::string root = directory.path().toStdString(); + const std::string path = lab_device_lock_path(root, first); + QProcess child; + try{ + auto owner = prepare_owner(test.input.scenario, path, child); + LabDeviceIdentity second = first; + if (test.input.scenario == LeaseScenario::DIFFERENT_DEVICE){ + second.id = "port-B"; + } + if (test.input.scenario == LeaseScenario::DIFFERENT_KIND){ + second.kind = LabDeviceKind::VIDEO; + } + bool opened = false; + try{ + auto lease = second.kind == LabDeviceKind::PROFILE + ? std::make_unique(lab_device_lock_path(root, second)) + : acquire_lab_device(policy, root, second); + opened = true; + }catch (const std::runtime_error&){} + child.kill(); + child.waitForFinished(5000); + if (opened != test.expectation.second_opens){ + return "Expected second opens " + std::to_string(test.expectation.second_opens) + "; read " + std::to_string(opened); + } + }catch (const std::runtime_error& error){ + child.kill(); + child.waitForFinished(5000); + return error.what(); + } + return {}; +} + +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +class SerialTest : public UnitTest{ +public: + explicit SerialTest(const SerialCase& test) : UnitTest("LabDevices/Serial/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_serial(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const SerialCase& m_test; +}; + +class PolicyTest : public UnitTest{ +public: + explicit PolicyTest(const PolicyCase& test) : UnitTest("LabDevices/Policy/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_policy(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const PolicyCase& m_test; +}; + +class LeaseTest : public UnitTest{ +public: + explicit LeaseTest(const LeaseCase& test) : UnitTest("LabDevices/Lease/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_lease(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const LeaseCase& m_test; +}; +#endif + +} // namespace + +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +void add_tests_LabDeviceAccess(UnitTestDatabase& database){ + for (const SerialCase& test : SERIAL_CASES){ + database.add(test); + } + for (const PolicyCase& test : POLICY_CASES){ + database.add(test); + } + for (const LeaseCase& test : LEASE_CASES){ + database.add(test); + } +} +#endif + +} // namespace PokemonAutomation + +#ifdef PA_LAB_DEVICE_STANDALONE_TESTS +// Small Qt-only runner/child process. It never links the GUI or device backends. +int main(int argc, char* argv[]){ + std::unique_ptr early_lease; + if (argc == 3 && std::string_view(argv[1]) == "--lease-before-qt"){ + early_lease = std::make_unique(argv[2]); + } + QCoreApplication application(argc, argv); + if (early_lease){ + std::cout << "LOCKED" << std::endl; + return application.exec(); + }else if (argc == 3 && std::string_view(argv[1]) == "--lease-child"){ + PokemonAutomation::LabDeviceLease lease(argv[2]); + std::cout << "LOCKED" << std::endl; + return application.exec(); + } + if (argc != 1){ + return 2; + } + int failed = 0; + for (const auto& test : PokemonAutomation::SERIAL_CASES){ + const std::string error = PokemonAutomation::check_serial(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + for (const auto& test : PokemonAutomation::POLICY_CASES){ + const std::string error = PokemonAutomation::check_policy(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + for (const auto& test : PokemonAutomation::LEASE_CASES){ + const std::string error = PokemonAutomation::check_lease(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + return failed; +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h new file mode 100644 index 0000000000..73a5d516a8 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h @@ -0,0 +1,16 @@ +/* Lab Device Access Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceAccessQt_Tests_H +#define PokemonAutomation_LabDeviceAccessQt_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register offline policy, lifecycle and cross-process lease regression cases. +void add_tests_LabDeviceAccess(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp new file mode 100644 index 0000000000..d6a7d8af2c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp @@ -0,0 +1,98 @@ +/* Lab Device Runtime + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "LabDeviceRuntime.h" + +namespace PokemonAutomation{ +namespace{ + +#if defined(__APPLE__) +// Shared by all guarded apps for this user, independent of their profile/bundle. +std::string shared_lease_directory(){ + return (QDir::homePath() + "/Library/Application Support/PokemonAutomation/LabDeviceLeases").toStdString(); +} + +std::string current_serial_identity(std::string_view port){ + const QSerialPortInfo selected(QString::fromUtf8(port.data(), (qsizetype)port.size())); + std::vector devices; + for (const QSerialPortInfo& info : QSerialPortInfo::availablePorts()){ + if (info.hasVendorIdentifier() && info.hasProductIdentifier()){ + devices.emplace_back(LabSerialDevice{ + info.systemLocation().toStdString(), info.vendorIdentifier(), + info.productIdentifier(), info.serialNumber().toStdString() + }); + } + } + return resolve_lab_serial_identity(devices, selected.systemLocation().toStdString()); +} + +const LabDevicePolicy& runtime_policy(){ + static const LabDevicePolicy policy = []{ + const auto path = Filesystem::application_scratch_path() / "lab-devices.json"; + QFile file(QString::fromStdString(path.string())); + if (!file.open(QIODevice::ReadOnly)){ + throw std::runtime_error("Cannot read lab-devices.json; hardware access is disabled."); + } + return parse_lab_device_policy(file.readAll().toStdString(), Filesystem::STARTUP_PROFILE()); + }(); + return policy; +} +#endif + +} // namespace + +std::unique_ptr reserve_lab_profile(){ +#if defined(__APPLE__) + const std::string directory = shared_lease_directory(); + if (!QDir().mkpath(QString::fromStdString(directory))){ + throw std::runtime_error("Cannot create lab profile lease directory."); + } + return std::make_unique(lab_device_lock_path( + directory, {LabDeviceKind::PROFILE, Filesystem::STARTUP_PROFILE()} + )); +#else + return nullptr; +#endif +} + +std::unique_ptr acquire_lab_serial(Logger& logger, std::string_view port){ +#if defined(__APPLE__) + try{ + // Do not trust the UI's cached port list after unplug/replug. + return acquire_lab_device(runtime_policy(), shared_lease_directory(), { + LabDeviceKind::SERIAL, current_serial_identity(port) + }); + }catch (const std::runtime_error& error){ + throw UserSetupError(logger, error.what()); + } +#else + (void)logger; + (void)port; + return nullptr; +#endif +} + +std::unique_ptr acquire_lab_capture(Logger& logger, LabDeviceKind kind, std::string_view id){ +#if defined(__APPLE__) + try{ + return acquire_lab_device(runtime_policy(), shared_lease_directory(), {kind, std::string(id)}); + }catch (const std::runtime_error& error){ + throw UserSetupError(logger, error.what()); + } +#else + (void)logger; + (void)kind; + (void)id; + return nullptr; +#endif +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h new file mode 100644 index 0000000000..5dd5fb88ca --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h @@ -0,0 +1,26 @@ +/* Lab Device Runtime + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceRuntime_H +#define PokemonAutomation_LabDeviceRuntime_H + +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "LabDeviceAccessQt.h" + +namespace PokemonAutomation{ + +// Reserve a macOS GUI profile before loading settings; throw std::runtime_error on failure. +// The returned lease must outlive the GUI. Other platforms return no reservation. +std::unique_ptr reserve_lab_profile(); + +// Resolve a fresh USB identity and reserve it before opening the requested port. +// Missing policy/identity or contention throws UserSetupError. Non-macOS is unchanged. +std::unique_ptr acquire_lab_serial(Logger& logger, std::string_view port); + +// Reserve a camera/audio input by the lowercase hex encoding of its Qt device ID. +// Caller holds the lease until after closing the input. Non-macOS is unchanged. +std::unique_ptr acquire_lab_capture(Logger& logger, LabDeviceKind kind, std::string_view id); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h index f9f6f5d1ff..dfc5ead1b4 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h @@ -7,12 +7,14 @@ #ifndef PokemonAutomation_HardwareValidation_H #define PokemonAutomation_HardwareValidation_H +#include "CommonFramework/Startup/SetupSettings.h" + namespace PokemonAutomation{ // Check user hardware. -// If the hardware is too old, send a QMessageBox to tell user and return false. -// If the hardware may not be powerful enough, send a QMessageBox to inform user. -bool check_hardware(); +// If the hardware is too old, send a QMessageBox (when `dialogs == SHOW`) and return false. +// If the hardware may not be powerful enough, send a QMessageBox (when `dialogs == SHOW`). +bool check_hardware(StartupDialogs dialogs = StartupDialogs::SHOW); } diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp index 7f716e8136..1d184838d6 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp @@ -14,7 +14,8 @@ namespace PokemonAutomation{ -bool check_hardware(){ +bool check_hardware(StartupDialogs dialogs){ + (void)dialogs; // TODO: write computer power check for Apple M-series computers return true; diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp index 361ea8765c..d34f5a3361 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp @@ -11,36 +11,47 @@ #include "HardwareValidation.h" namespace PokemonAutomation{ +namespace{ + +void show_hardware_warning(const QString& message, StartupDialogs dialogs){ + if (dialogs == StartupDialogs::LOG_ONLY){ + return; + } + QMessageBox box; + box.warning(nullptr, "Warning", message); +} + +} // namespace -bool check_hardware(){ +bool check_hardware(StartupDialogs dialogs){ if (!CPU_CAPABILITY_NATIVE.OK_08_Nehalem){ - QMessageBox box; - box.critical( - nullptr, - "Error", - "This computer is too old to run this program.

" - "Please try a much newer computer.

" - "(Reason: SSE4.2 is required to run this program.)" - ); + if (dialogs == StartupDialogs::SHOW){ + QMessageBox box; + box.critical( + nullptr, + "Error", + "This computer is too old to run this program.

" + "Please try a much newer computer.

" + "(Reason: SSE4.2 is required to run this program.)" + ); + } return false; } if (!CPU_CAPABILITY_NATIVE.OK_13_Haswell){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; str += "(i.e. Increased error rate. Fail to reliably detect shinies, etc...)

"; str += "(Reason: Computers that lack AVX2 are likely to be too old and too slow to handle this program.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } ProcessorSpecs specs = get_processor_specs(); if (specs.threads < 4){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -49,11 +60,10 @@ bool check_hardware(){ str += QString::number(specs.threads); str += " vcores is a weak CPU.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } if (specs.base_frequency < 2'000'000'000){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -62,7 +72,7 @@ bool check_hardware(){ str += QString::fromStdString(tostr_fixed(specs.base_frequency / 1000000000., 3)); str += " GHz which is very slow.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } #if (defined _WIN32) || (defined __linux) @@ -71,7 +81,6 @@ bool check_hardware(){ : 0; double efreq = (specs.cores + 0.2 * vcores) * specs.base_frequency; if (efreq < 8'000'000'000){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -87,7 +96,7 @@ bool check_hardware(){ str += QString::fromStdString(tostr_fixed(efreq / 1000000000., 3)); str += " GHz

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } #endif diff --git a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp index df38989010..37f37828cf 100644 --- a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp +++ b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp @@ -119,6 +119,12 @@ GlobalSettings::GlobalSettings() LockMode::UNLOCK_WHILE_RUNNING, false ) + , CONSOLE_SETTINGS_COLLAPSED( + "Start with console settings collapsed:
" + "Each program's \"Console N Settings\" box starts folded up. Click its title to open it.", + LockMode::UNLOCK_WHILE_RUNNING, + false + ) , KEYBOARD_CONTROLS_LAYOUT( CONSTRUCT_TOKEN, "System Keyboard Layout:
" @@ -205,6 +211,7 @@ GlobalSettings::GlobalSettings() PA_ADD_OPTION(WINDOW_SIZE); PA_ADD_OPTION(LOG_WINDOW_SIZE); PA_ADD_OPTION(LOG_WINDOW_STARTUP); + PA_ADD_OPTION(CONSOLE_SETTINGS_COLLAPSED); PA_ADD_OPTION(KEYBOARD_CONTROLS_LAYOUT); PA_ADD_OPTION(DEVICE_ALIASES); diff --git a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h index 29ef1fff03..2afea6a082 100644 --- a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h +++ b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h @@ -92,6 +92,7 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener, priva Pimpl WINDOW_SIZE; Pimpl LOG_WINDOW_SIZE; BooleanCheckBoxOption LOG_WINDOW_STARTUP; + BooleanCheckBoxOption CONSOLE_SETTINGS_COLLAPSED; Pimpl KEYBOARD_CONTROLS_LAYOUT; diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp new file mode 100644 index 0000000000..7afe27818f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp @@ -0,0 +1,345 @@ +/* Headless Host + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_WIN32) +#include +#include +#endif +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/PersistentSettings.h" +#include "CommonFramework/ProgramSession.h" +#include "GameConsole/Framework/ConsoleProgramSession.h" +#include "GameConsole/Framework/ConsoleSystemSession.h" +#include "GameConsole/Framework/MultiConsoleProgramSession.h" +#include "GameConsole/Framework/MultiConsoleSystemSession.h" +#include "PanelCatalog.h" +#include "HeadlessHost.h" + +namespace PokemonAutomation{ +namespace{ + +constexpr std::chrono::seconds READINESS_TIMEOUT{10}; + +#if !defined(_WIN32) +constexpr std::chrono::seconds STOP_TIMEOUT{5}; +struct SignalState{ + std::atomic write_fd{-1}; + std::atomic last_signal{0}; +}; + +SignalState& signal_state(){ + static SignalState state; + return state; +} + +void posix_signal_handler(int sig){ + SignalState& s = signal_state(); + s.last_signal.store(sig, std::memory_order_relaxed); + const int fd = s.write_fd.load(std::memory_order_relaxed); + if (fd >= 0){ + const char byte = static_cast(sig); + const ssize_t ignored = ::write(fd, &byte, /*nbyte=*/1); + (void)ignored; + } +} + +class SignalPipe final{ +public: + SignalPipe(Logger& logger, std::function on_signal){ + if (::pipe(m_fds) != 0){ + logger.log("Failed to create signal pipe; SIGINT/SIGTERM will terminate immediately.", COLOR_ORANGE); + return; + } + for (int fd : m_fds){ + const int flags = ::fcntl(fd, F_GETFL, /*arg=*/0); + if (flags >= 0){ + ::fcntl(fd, F_SETFL, flags | O_NONBLOCK); + } + } + signal_state().write_fd.store(m_fds[1], std::memory_order_release); + struct sigaction sa{}; + sa.sa_handler = posix_signal_handler; + sigemptyset(&sa.sa_mask); + m_has_int = (::sigaction(SIGINT, &sa, &m_old_int) == 0); + m_has_term = (::sigaction(SIGTERM, &sa, &m_old_term) == 0); + + m_notifier = std::make_unique(m_fds[0], QSocketNotifier::Read); + QObject::connect(m_notifier.get(), &QSocketNotifier::activated, m_notifier.get(), [this, on_signal = std::move(on_signal)]{ + std::array buf; + while (::read(m_fds[0], buf.data(), buf.size()) > 0){ + } + on_signal(signal_state().last_signal.load(std::memory_order_relaxed)); + }); + } + + ~SignalPipe(){ + if (m_has_int){ + ::sigaction(SIGINT, &m_old_int, nullptr); + } + if (m_has_term){ + ::sigaction(SIGTERM, &m_old_term, nullptr); + } + signal_state().write_fd.store(-1, std::memory_order_release); + m_notifier.reset(); + for (int& fd : m_fds){ + if (fd >= 0){ + ::close(fd); + fd = -1; + } + } + } + +private: + int m_fds[2] = {-1, -1}; + struct sigaction m_old_int{}; + struct sigaction m_old_term{}; + bool m_has_int = false; + bool m_has_term = false; + std::unique_ptr m_notifier; +}; +#endif + +bool system_ready(GameConsole::ConsoleSystemSession& system){ + for (size_t i = 0; i < system.controllers(); i++){ + if (!system.controller(i).ready()){ + return false; + } + } + return true; +} + +bool controller_ready(ProgramSession& program){ + if (auto* single = dynamic_cast(&program)){ + return system_ready(single->system()); + } + if (auto* multi = dynamic_cast(&program)){ + for (size_t i = 0; i < multi->system().active_consoles(); i++){ + if (!system_ready(multi->system()[i])){ + return false; + } + } + } + return true; +} + +class HeadlessProgramObserver final : public ProgramSession::Listener{ +public: + HeadlessProgramObserver(ProgramSession& program, QObject& context, std::function on_stopped) + : m_program(program), m_context(context), m_on_stopped(std::move(on_stopped)) + {} + + bool saw_error() const{ return m_saw_error.load(std::memory_order_acquire); } + + virtual void state_change(ProgramState state) override{ + if (state == ProgramState::STOPPED){ + QMetaObject::invokeMethod(&m_context, [this]{ + if (m_program.current_state() == ProgramState::STOPPED){ + m_on_stopped(); + } + }, Qt::QueuedConnection); + } + } + virtual void stats_update(const StatsTracker*, const StatsTracker*) override{} + virtual void error(const std::string&) override{ m_saw_error.store(true, std::memory_order_release); } + virtual void download_error(const std::string&) override{} + virtual void download_added(std::shared_ptr) override{} + virtual void all_downloads_done() override{} + +private: + ProgramSession& m_program; + QObject& m_context; + std::function m_on_stopped; + std::atomic m_saw_error{false}; +}; + +std::expected merge_options(const JsonValue& base, const JsonValue& over){ + const JsonObject* over_obj = over.to_object(); + if (over_obj == nullptr){ + return std::unexpected("Options file must contain a JSON object."); + } + if (over_obj->get_value("ConsoleSetup") != nullptr){ + return std::unexpected("--options cannot override ConsoleSetup."); + } + JsonObject merged; + if (const JsonObject* base_obj = base.to_object()){ + merged = base_obj->clone(); + } + for (const auto& [key, value] : *over_obj){ + merged[key] = value.clone(); + } + return JsonValue(std::move(merged)); +} + +HeadlessExit load_headless_panel( + const std::vector>& lists, + const HeadlessRunOptions& options, + HeadlessHost& host, + Logger& logger +){ + std::optional match = find_panel(lists, options.program_id); + if (!match){ + logger.log(std::format("Unknown program identifier: \"{}\".", options.program_id), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + std::shared_ptr descriptor(std::move(match->entry.descriptor)); + + JsonValue json; + if (options.base_panel_json){ + json = options.base_panel_json->clone(); + }else if (const JsonValue* saved = PERSISTENT_SETTINGS().panels.get_value(descriptor->identifier())){ + json = saved->clone(); + } + if (options.options_override){ + auto merged = merge_options(json, *options.options_override); + if (!merged){ + logger.log(merged.error(), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + json = std::move(*merged); + } + + try{ + load_panel_session(descriptor, json, host); + }catch (const UserSetupError& e){ + logger.log(std::format("Device setup error: {}", e.to_str()), COLOR_RED); + return HeadlessExit::DEVICE; + }catch (const Exception& e){ + logger.log(std::format("Failed to load program: {}", e.to_str()), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + return HeadlessExit::OK; +} + +bool wait_for_controller_ready(ProgramSession& program){ + if (controller_ready(program)){ + return true; + } + QEventLoop loop; + QTimer poll; + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&poll, &QTimer::timeout, &loop, [&]{ + if (controller_ready(program)){ + loop.quit(); + } + }); + QObject::connect(&timeout, &QTimer::timeout, &loop, [&]{ loop.quit(); }); + poll.start(std::chrono::milliseconds(50)); + timeout.start(READINESS_TIMEOUT); + loop.exec(); + return controller_ready(program); +} + +HeadlessExit run_program_event_loop(ProgramSession& program, bool handle_signals, Logger& logger){ + QEventLoop loop; + std::optional signal_exit; + QTimer stop_timer; + stop_timer.setSingleShot(true); + + HeadlessProgramObserver observer(program, loop, [&]{ loop.quit(); }); + program.add_listener(observer); + +#if !defined(_WIN32) + std::unique_ptr pipe; + if (handle_signals){ + pipe = std::make_unique(logger, [&](int sig){ + const HeadlessExit code = sig == SIGTERM ? HeadlessExit::SIGNAL_TERM : HeadlessExit::SIGNAL_INT; + if (signal_exit){ + logger.log("Second signal received; exiting immediately.", COLOR_RED); + std::_Exit(static_cast(code)); + } + signal_exit = code; + logger.log("Signal received; stopping program...", COLOR_ORANGE); + program.stop_program(ActionSource::CLI, std::nullopt); + QObject::connect(&stop_timer, &QTimer::timeout, &loop, [&logger, code]{ + logger.log("Stop timeout expired; exiting immediately.", COLOR_RED); + std::_Exit(static_cast(code)); + }); + stop_timer.start(STOP_TIMEOUT); + }); + } +#else + (void)handle_signals; +#endif + + const std::string err = program.start_program(ActionSource::CLI, std::nullopt); + if (!err.empty()){ + logger.log(std::format("Failed to start program: {}", err), COLOR_RED); + program.remove_listener(observer); + return HeadlessExit::DEVICE; + } + if (program.current_state() != ProgramState::STOPPED){ + loop.exec(); + } + program.remove_listener(observer); + if (signal_exit){ + return *signal_exit; + } + const std::optional last = program.last_transition(); + const bool program_error = observer.saw_error() || (last && last->source == ActionSource::PROGRAM_ERROR); + return program_error ? HeadlessExit::PROGRAM_ERROR : HeadlessExit::OK; +} + +} // namespace + + +void HeadlessHost::clear_panel(){ + m_current_panel.reset(); + m_current_panel_descriptor.reset(); +} + +void HeadlessHost::load_panel( + std::shared_ptr descriptor, + std::unique_ptr panel +){ + clear_panel(); + m_current_panel_descriptor = std::move(descriptor); + m_current_panel = std::move(panel); +} + +HeadlessExit run_headless_program( + const std::vector>& lists, + const HeadlessRunOptions& options +){ + Logger& logger = global_logger_tagged(); + HeadlessHost host; + const HeadlessExit load_status = load_headless_panel(lists, options, host, logger); + if (load_status != HeadlessExit::OK){ + return load_status; + } + ProgramSession* program = dynamic_cast(host.current_panel()); + if (program == nullptr){ + logger.log(std::format("Panel \"{}\" is not a runnable program.", options.program_id), COLOR_RED); + host.clear_panel(); + return HeadlessExit::BAD_REQUEST; + } + if (!wait_for_controller_ready(*program)){ + logger.log("Controller did not become ready before timeout.", COLOR_RED); + host.clear_panel(); + return HeadlessExit::DEVICE; + } + const HeadlessExit ret = run_program_event_loop(*program, options.handle_signals, logger); + host.clear_panel(); + return ret; +} + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h new file mode 100644 index 0000000000..28753ea339 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h @@ -0,0 +1,72 @@ +/* Headless Host + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_HeadlessHost_H +#define PokemonAutomation_HeadlessHost_H + +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/Panels/PanelList.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/PanelTools.h" + +namespace PokemonAutomation{ + + +// Process exit codes returned by --headless --run. +enum class HeadlessExit : int{ + OK = 0, + PROGRAM_ERROR = 1, + STARTUP = 3, + BAD_REQUEST = 4, + DEVICE = 5, + SIGNAL_INT = 130, + SIGNAL_TERM = 143, +}; + + +// Widgetless PanelHolder for --headless runs. Never calls make_ui_component() +// or writes persistent settings. +class HeadlessHost final : public PanelHolder{ +public: + PanelSession* current_panel() const{ return m_current_panel.get(); } + + void clear_panel(); + + virtual bool report_new_panel_intent(const PanelDescriptor&) override{ return true; } + virtual void load_panel( + std::shared_ptr descriptor, + std::unique_ptr panel + ) override; + virtual void on_busy() override{} + virtual void on_idle() override{} + +private: + std::shared_ptr m_current_panel_descriptor; + std::unique_ptr m_current_panel; +}; + + +struct HeadlessRunOptions{ + std::string program_id; + std::optional options_override = std::nullopt; + std::optional base_panel_json = std::nullopt; + bool handle_signals = true; +}; + + +// Run one program from `lists` to completion under QCoreApplication::exec(). +HeadlessExit run_headless_program( + const std::vector>& lists, + const HeadlessRunOptions& options +); + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp new file mode 100644 index 0000000000..e7ef7d9d99 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp @@ -0,0 +1,138 @@ +/* Headless Host Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Panels/PanelDescriptor.h" +#include "CommonFramework/Panels/PanelList.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "ComputerPrograms/ComputerProgram.h" +#include "HeadlessHost.h" +#include "HeadlessHost_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +enum class FakeBehavior{SUCCEED, REPORT_ERROR, REFUSE_START}; + +class FakeInstance final : public ComputerProgramInstance{ +public: + explicit FakeInstance(FakeBehavior b) : m_b(b) {} + virtual std::string check_validity() const override{ + return m_b == FakeBehavior::REFUSE_START ? "invalid configuration" : ""; + } + virtual void program(ProgramEnvironment&, CancellableScope&) override{ + if (m_b == FakeBehavior::REPORT_ERROR){ + throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "simulated failure"); + } + } +private: + FakeBehavior m_b; +}; + +class FakeProgramDescriptor final : public ComputerProgramDescriptor{ +public: + FakeProgramDescriptor(std::string id, FakeBehavior b) + : ComputerProgramDescriptor(std::move(id), "Fake", "FakeProgram", /*doc_link=*/"", /*description=*/"", PanelQualityStanding::DEVELOPER) + , m_b(b) + {} + virtual std::unique_ptr make_instance() const override{ + return std::make_unique(m_b); + } +private: + FakeBehavior m_b; +}; + +class FakeSettingsDescriptor final : public PanelDescriptor{ +public: + explicit FakeSettingsDescriptor(std::string id) + : PanelDescriptor(std::move(id), "Fake", "FakeSettings", /*doc_link=*/"", /*description=*/"", PanelQualityStanding::DEVELOPER, /*restore_defaults_button=*/false) + {} + virtual std::unique_ptr make_panel() const override{ + return std::make_unique(*this); + } +}; + +class FakeListDescriptor final : public PanelListDescriptor{ +public: + FakeListDescriptor() : PanelListDescriptor("Fake") {} +protected: + virtual std::vector make_panels() const override{ + std::vector ret; + ret.emplace_back(std::make_unique("Fake:Ok", FakeBehavior::SUCCEED)); + ret.emplace_back(std::make_unique("Fake:Err", FakeBehavior::REPORT_ERROR)); + ret.emplace_back(std::make_unique("Fake:Refused", FakeBehavior::REFUSE_START)); + ret.emplace_back(std::make_unique("Fake:Settings")); + return ret; + } +}; + +struct RunInput{ + std::string_view program_id; + std::optional options_json = std::nullopt; +}; + +struct RunExpectation{ + HeadlessExit exit_code; +}; + +struct RunCase{ + std::string_view name; + RunInput input; + RunExpectation expectation; +}; + +const RunCase RUN_CASES[] = { + {.name = "clean program finishes 0", .input = {.program_id = "Fake:Ok"}, .expectation = {.exit_code = HeadlessExit::OK}}, + {.name = "program error finishes 1", .input = {.program_id = "Fake:Err"}, .expectation = {.exit_code = HeadlessExit::PROGRAM_ERROR}}, + {.name = "unknown identifier finishes 4", .input = {.program_id = "Fake:Missing"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "settings panel finishes 4", .input = {.program_id = "Fake:Settings"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "refused start finishes 5", .input = {.program_id = "Fake:Refused"}, .expectation = {.exit_code = HeadlessExit::DEVICE}}, + {.name = "ConsoleSetup override finishes 4", .input = {.program_id = "Fake:Ok", .options_json = "{\"ConsoleSetup\": {}}"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "non-object override finishes 4", .input = {.program_id = "Fake:Ok", .options_json = "42"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, +}; + +UnitTestResult check_run_case(const RunCase& test){ + std::vector> lists; + lists.push_back(std::make_unique()); + HeadlessRunOptions options{ + .program_id = std::string(test.input.program_id), + .base_panel_json = JsonValue(), + .handle_signals = false, + }; + if (test.input.options_json){ + options.options_override = parse_json(std::string(*test.input.options_json)); + } + const HeadlessExit code = run_headless_program(lists, options); + if (code != test.expectation.exit_code){ + return std::format("Expected exit code {}; got {}.", static_cast(test.expectation.exit_code), static_cast(code)); + } + return true; +} + +class RunTest final : public UnitTest{ +public: + explicit RunTest(const RunCase& test) + : UnitTest(std::format("HeadlessHost/Run/{}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_run_case(m_test); + } +private: + const RunCase& m_test; +}; + +} // namespace + +void add_tests_HeadlessHost(UnitTestDatabase& database){ + for (const RunCase& test : RUN_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h new file mode 100644 index 0000000000..6107fb361f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h @@ -0,0 +1,17 @@ +/* Headless Host Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_HeadlessHost_Tests_H +#define PokemonAutomation_HeadlessHost_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +void add_tests_HeadlessHost(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Main.cpp b/SerialPrograms/Source/CommonFramework/Main.cpp index ef6742e2eb..372dae0189 100644 --- a/SerialPrograms/Source/CommonFramework/Main.cpp +++ b/SerialPrograms/Source/CommonFramework/Main.cpp @@ -1,5 +1,9 @@ #include +#include +#include +#include +#include //#include #include #include "Common/Cpp/Logging/LastLogTracker.h" @@ -17,7 +21,9 @@ #include "Common/Cpp/ScopeExit.h" #include "Common/Qt/GlobalThreadPoolsQt.h" #include "StaticRegistration.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" #include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Headless/HeadlessHost.h" #include "CommonFramework/Tools/GlobalThreadPools.h" #include "CommonFramework/Panels/OptionsPanel.h" #include "CommonFramework/VideoPipeline/Backends/CameraImplementations.h" @@ -41,17 +47,21 @@ #include "Logging/Logger.h" #include "Logging/OutputRedirector.h" #include "Startup/SetupSettings.h" +#include "Startup/StartupArguments.h" +#include "Startup/LabStartupPolicy.h" #include "Startup/NewVersionCheck.h" #include "Windows/MainWindow.h" #include "GlobalSettingsPanel.h" #include "GlobalServices.h" #include "PersistentSettings.h" +#include "PanelCatalog.h" #include "NintendoSwitch/NintendoSwitch_SettingsPanel.h" -using namespace PokemonAutomation; - Q_DECLARE_METATYPE(std::string) +namespace PokemonAutomation{ +namespace{ + void set_working_directory(){ const std::string app_dir = Filesystem::application_binary_directory().string_slash_normalized(); const std::string bundle_dir = "/Contents/MacOS"; @@ -66,10 +76,6 @@ void set_working_directory(){ } } -namespace PokemonAutomation{ - - - FileLogger& global_file_logger(){ static FileLogger logger( GlobalThreadPools::unlimited_normal(), @@ -80,19 +86,47 @@ FileLogger& global_file_logger(){ return logger; } -} +bool init_common_runtime(Logger& logger, StartupDialogs dialogs, bool fail_on_settings_error){ + qRegisterMetaType("size_t"); + qRegisterMetaType("uint8_t"); + qRegisterMetaType("std::string"); + qRegisterMetaType("Resolution"); + + Filesystem::create_directories(Filesystem::Path(SETTINGS_PATH())); + Filesystem::create_directories(Filesystem::Path(SCREENSHOTS_PATH())); + + // Several novice developers struggled to build and run the program due to missing Resources folder. + // Add this check to pop a message box when Resources folder is missing. + if (!check_resource_folder(logger, dialogs)){ + return false; + } + + // Read program settings from json file: SerialPrograms-Settings.json. + try{ + PERSISTENT_SETTINGS().read(); + }catch (const FileException& error){ + logger.log(error.message(), COLOR_RED); + if (fail_on_settings_error){ + return false; + } + }catch (const ParseException& error){ + logger.log(error.message(), COLOR_RED); + if (fail_on_settings_error){ + return false; + } + } + + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + return true; +} int run_program(int argc, char *argv[]){ // qputenv("QT_RHI_BACKEND", "opengl"); -#if defined(__APPLE__) - Filesystem::set_startup_profile(argc, argv); - QApplication application(argc, argv); -#else QApplication application(argc, argv); -#endif #ifdef PA_DPP // Configure OpenSSL before DPP or background tasks create a TLS context. @@ -164,26 +198,18 @@ int run_program(int argc, char *argv[]){ logger.log(dpp_cert_message, dpp_cert_error ? COLOR_RED : Color()); #endif - qRegisterMetaType("size_t"); - qRegisterMetaType("uint8_t"); - qRegisterMetaType("std::string"); - qRegisterMetaType("Resolution"); - #if defined(__linux) || defined(__APPLE__) // By default Qt uses native menubar but this only works on Windows. // We use menubar in our ButtonDiagram window to choose which controller's button mapping image to show. // So we fix it by don't using native menubar on non-Windows OS. QCoreApplication::setAttribute(Qt::AA_DontUseNativeMenuBar); #endif - -//#if QT_VERSION_MAJOR == 5 // AA_EnableHighDpiScaling is deprecated in Qt6 -// QApplication::setAttribute(Qt::AA_EnableHighDpiScaling); -//#endif - - Filesystem::create_directories(Filesystem::Path(SETTINGS_PATH())); - Filesystem::create_directories(Filesystem::Path(SCREENSHOTS_PATH())); - - + if (!init_common_runtime(logger, StartupDialogs::SHOW, /*fail_on_settings_error=*/false)){ + return 1; + } + if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){ + return run_command_line_tests(false); + } ScopeExit cleanup([]{ #ifdef PA_SDL @@ -193,63 +219,10 @@ int run_program(int argc, char *argv[]){ GlobalMediaServices::instance().stop(); }); - // Several novice developers struggled to build and run the program due to missing Resources folder. - // Add this check to pop a message box when Resources folder is missing. - if (!check_resource_folder(logger)){ - return 1; - } - - // Read program settings from json file: SerialPrograms-Settings.json. - try{ - PERSISTENT_SETTINGS().read(); - }catch (const FileException& error){ - logger.log(error.message(), COLOR_RED); - }catch (const ParseException& error){ - logger.log(error.message(), COLOR_RED); - } - - // Load the Switch framework settings now, not only when their panel is opened. OptionsPanelWrapper().make_panel()->load_json_from_global(); - - // Preload a bunch of stuff now so they are ready later. SerialPortPoller::instance().ports(); get_all_cameras(); - // Force all the Qt thread pools to be constructed now on the main thread. - GlobalThreadPools::qt_worker_threadpool(); - GlobalThreadPools::qt_event_threadpool(); - - bool augment_test_color_distortions = false; - std::string test_substring; - bool test_substring_missing_value = false; - for (size_t i = 0; i < argc; i++){ - constexpr const char* force_run_tests = "--command-line-test-mode"; - constexpr const char* distort_test_images = "--augment-color-distortions"; - constexpr const char* filter_tests_substring = "--test-substring"; - - if (strcmp(argv[i], force_run_tests) == 0){ - GlobalSettings::instance().COMMAND_LINE_TEST_MODE = true; - } - if (strcmp(argv[i], distort_test_images) == 0){ - augment_test_color_distortions = true; - } - if (strcmp(argv[i], filter_tests_substring) == 0){ - if (i + 1 < argc){ - test_substring = argv[++i]; - }else{ - test_substring_missing_value = true; - } - } - } - if (test_substring_missing_value){ - global_logger_command_line().log("--test-substring requires a value", COLOR_RED); - return 1; - } - - if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){ - return run_command_line_tests(augment_test_color_distortions, test_substring); - } - // Check whether the hardware is powerful enough to run this program. if (!check_hardware()){ return 1; @@ -297,29 +270,8 @@ int run_program(int argc, char *argv[]){ } -int main(int argc, char *argv[]){ -#if 0 - // Retrieve and store program name - set_program_path(argv[0]); -#endif - -#if defined(__linux__) - // Qt multimedia, default to gstreamer to prevent flickering - // Easier than the alternative which is compiling qt6multimedia with QT_DEFAULT_MEDIA_BACKEND - // See: https://doc.qt.io/qt-6.5/qtmultimedia-index.html - if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){ - qputenv("QT_MEDIA_BACKEND", "gstreamer"); - } -#endif - - // So far, this is only needed on Mac where static initialization is fucked up. - PokemonAutomation::register_all_statics(); - - setup_crash_handler(); - - int ret = run_program(argc, argv); - - +// Stop GUI services after run_program() returns, preserving the existing order. +void stop_application_services(){ #ifdef PA_SOCIAL_SDK Integration::DiscordSocialSDK::DiscordSocial::instance().stop(); #endif @@ -342,6 +294,47 @@ int main(int argc, char *argv[]){ global_watchdog().stop(); global_file_logger().stop(); +} + +// Join the worker pools after service shutdown and the Windows Qt workaround. +void stop_thread_pools(){ + // Force stop the thread pools. + // This is where all the threads in the program are joined. + PokemonAutomation::GlobalThreadPools::computation_realtime().stop(); + PokemonAutomation::GlobalThreadPools::computation_normal().stop(); + PokemonAutomation::GlobalThreadPools::unlimited_realtime().stop(); + PokemonAutomation::GlobalThreadPools::unlimited_normal().stop(); + PokemonAutomation::global_dispatcher.stop(); + + GlobalThreadPools::qt_worker_threadpool().stop(); + GlobalThreadPools::qt_event_threadpool().stop(); + +} + +int run_gui_application(int argc, char *argv[]){ +#if 0 + // Retrieve and store program name + set_program_path(argv[0]); +#endif + +#if defined(__linux__) + // Qt multimedia, default to gstreamer to prevent flickering + // Easier than the alternative which is compiling qt6multimedia with QT_DEFAULT_MEDIA_BACKEND + // See: https://doc.qt.io/qt-6.5/qtmultimedia-index.html + if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){ + qputenv("QT_MEDIA_BACKEND", "gstreamer"); + } +#endif + + // So far, this is only needed on Mac where static initialization is fucked up. + PokemonAutomation::register_all_statics(); + + setup_crash_handler(); + + int ret = run_program(argc, argv); + + + stop_application_services(); // // Workaround Qt 6.9 thread-adoption bug on Windows. // https://github.com/PokemonAutomation/Arduino-Source/issues/570 @@ -354,18 +347,162 @@ int main(int argc, char *argv[]){ std::quick_exit(ret); #endif - // Force stop the thread pools. - // This is where all the threads in the program are joined. - PokemonAutomation::GlobalThreadPools::computation_realtime().stop(); - PokemonAutomation::GlobalThreadPools::computation_normal().stop(); - PokemonAutomation::GlobalThreadPools::unlimited_realtime().stop(); - PokemonAutomation::GlobalThreadPools::unlimited_normal().stop(); - PokemonAutomation::global_dispatcher.stop(); - - GlobalThreadPools::qt_worker_threadpool().stop(); - GlobalThreadPools::qt_event_threadpool().stop(); + stop_thread_pools(); std::cout << "Exiting main()..." << std::endl; return ret; } + +bool load_headless_options(const StartupPlan& plan, Logger& logger, std::optional& out){ + if (!plan.options_path){ + return true; + } + try{ + out = load_json_file(*plan.options_path); + return true; + }catch (const Exception& e){ + logger.log("Failed to read --options \"" + *plan.options_path + "\": " + e.to_str(), COLOR_RED); + return false; + } +} + +int run_headless_application(QApplication& application, const StartupPlan& plan){ + ScopeExit service_cleanup([]{ + stop_application_services(); + stop_thread_pools(); + }); + + GlobalOutputRedirector redirect_stdout(std::cout, "stdout", Color()); + GlobalOutputRedirector redirect_stderr(std::cerr, "stderr", COLOR_RED); + { + MultiOutputLogger& logger = global_multi_logger(); + logger.add_listener(global_last_log_history()); + logger.add_listener(global_file_logger()); + } + + Logger& logger = global_logger_tagged(); + logger.log("================================================================================"); + logger.log("Starting Headless Program: " + *plan.run_program); + + std::optional options_override; + if (!load_headless_options(plan, logger, options_override)){ + return static_cast(HeadlessExit::BAD_REQUEST); + } + + if (!init_common_runtime(logger, StartupDialogs::LOG_ONLY, /*fail_on_settings_error=*/true)){ + return static_cast(HeadlessExit::STARTUP); + } + + ScopeExit device_cleanup([]{ + SerialPortPoller::instance().stop(); + GlobalMediaServices::instance().stop(); + }); + OptionsPanelWrapper().make_panel()->load_json_from_global(); + if (!check_hardware(StartupDialogs::LOG_ONLY)){ + return static_cast(HeadlessExit::STARTUP); + } + + set_working_directory(); + set_permissions(application); + return static_cast(run_headless_program( + make_all_panel_lists(), + HeadlessRunOptions{ + .program_id = *plan.run_program, + .options_override = std::move(options_override), + .handle_signals = true, + } + )); +} + +// Explicit tests use QCoreApplication and never enter GUI/settings/device startup. +int run_test_application(int argc, char* argv[], const StartupPlan& plan){ +#if defined(__APPLE__) + Filesystem::set_startup_profile("unit-tests"); +#endif + QCoreApplication application(argc, argv); + register_all_statics(); + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + const int ret = run_command_line_tests(plan.augment_color_distortions); + OCR::clear_ocr_cache(); + global_periodic_runner().stop(); + global_watchdog().stop(); +#ifdef PA_ENABLE_QT_ADOPTION_WORKAROUND + std::quick_exit(ret); +#endif + stop_thread_pools(); + return ret; +} + +int run_with_profile_lease( + const PokemonAutomation::StartupPlan& selected, + int lease_error_exit, + const std::function& launch +){ +#if defined(__APPLE__) + PokemonAutomation::Filesystem::set_startup_profile(*selected.profile); +#else + (void)selected; +#endif + std::unique_ptr profile_lease; + try{ + profile_lease = PokemonAutomation::reserve_lab_profile(); + }catch (const std::runtime_error& error){ + std::cerr << error.what() << std::endl; + return lease_error_exit; + } + return launch(); +} + +} // namespace +} // namespace PokemonAutomation + +// Validate before even constructing Qt: a typo must never become a hardware launch. +int main(int argc, char* argv[]){ + std::vector arguments; + for (int i = 1; i < argc; i++){ + arguments.emplace_back(argv[i]); + } + PokemonAutomation::LabStartupPolicy policy; +#if defined(__APPLE__) + policy.profiles = PokemonAutomation::LabProfileMode::REQUIRED; +#ifdef PA_LAB_PROFILE + policy.bound_profile = PA_LAB_PROFILE; +#endif +#endif + const PokemonAutomation::StartupPlan plan = PokemonAutomation::apply_lab_startup_policy(PokemonAutomation::parse_startup_arguments(arguments), policy); + if (plan.mode == PokemonAutomation::StartupMode::REJECTED){ + std::cerr << plan.error << std::endl; + } + // Keep backing storage alive for Qt, which may mutate argc and argv. + std::vector qt_storage = plan.qt_arguments; + qt_storage.insert(qt_storage.begin(), argv[0]); + std::vector qt_argv; + for (std::string& arg : qt_storage){ + qt_argv.push_back(arg.data()); + } + int qt_argc = (int)qt_argv.size(); + qt_argv.push_back(nullptr); + return PokemonAutomation::dispatch_startup(plan, { + .gui = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_with_profile_lease(selected, 2, [&]{ + return PokemonAutomation::run_gui_application(qt_argc, qt_argv.data()); + }); + }, + .headless = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_with_profile_lease( + selected, static_cast(PokemonAutomation::HeadlessExit::STARTUP), + [&]{ + PokemonAutomation::register_all_statics(); + PokemonAutomation::setup_crash_handler(); + QApplication application(qt_argc, qt_argv.data()); + return PokemonAutomation::run_headless_application(application, selected); + } + ); + }, + .tests = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_test_application(qt_argc, qt_argv.data(), selected); + }, + }); +} diff --git a/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp b/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp index ae16768c86..0ebce8d7e7 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp +++ b/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp @@ -6,6 +6,7 @@ #include "Common/Cpp/Exceptions.h" #include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/Panels/PanelSession.h" #include "PanelList.h" //#include @@ -33,6 +34,19 @@ PanelListDescriptor::PanelListDescriptor( } } +bool load_panel_session( + const std::shared_ptr& descriptor, + const JsonValue& panel_json, + PanelHolder& holder +){ + if (!holder.report_new_panel_intent(*descriptor)){ + return false; + } + std::unique_ptr panel = descriptor->make_panel(); + panel->load_json(panel_json); + holder.load_panel(descriptor, std::move(panel)); + return true; +} diff --git a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h index b6da3ab2d5..0ab5f36dca 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h +++ b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_PanelList_H #define PokemonAutomation_PanelList_H +#include #include #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/Panels/PanelTools.h" @@ -48,6 +49,16 @@ class PanelListDescriptor{ }; +// Instantiate `descriptor` (non-null), load `panel_json` into it, and install +// it in `holder`. Returns false, loading nothing, if +// `holder.report_new_panel_intent()` declines. +bool load_panel_session( + const std::shared_ptr& descriptor, + const JsonValue& panel_json, + PanelHolder& holder +); + + } #endif diff --git a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp index 387ca75331..277bdfebba 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp @@ -93,18 +93,16 @@ void PanelListWidget::handle_panel_clicked(const std::string& text){ PERSISTENT_SETTINGS().panels[JSON_PROGRAM_PANEL] = ""; return; } - std::shared_ptr& descriptor = iter->second; + const std::shared_ptr& descriptor = iter->second; // cout << descriptor->display_name() << endl; - if (!m_panel_holder.report_new_panel_intent(*descriptor)){ - return; - } try{ - std::unique_ptr panel = descriptor->make_panel(); -// try{ - panel->load_json(PERSISTENT_SETTINGS().panels[descriptor->identifier()]); -// }catch (ParseException&){} - m_panel_holder.load_panel(descriptor, std::move(panel)); - + if (!load_panel_session( + descriptor, + PERSISTENT_SETTINGS().panels[descriptor->identifier()], + m_panel_holder + )){ + return; + } PERSISTENT_SETTINGS().panels[JSON_PROGRAM_PANEL] = iter->first; }catch (const Exception& error){ QMessageBox box; diff --git a/SerialPrograms/Source/CommonFramework/ProgramTransition.h b/SerialPrograms/Source/CommonFramework/ProgramTransition.h index 739918fd1c..34ec365338 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramTransition.h +++ b/SerialPrograms/Source/CommonFramework/ProgramTransition.h @@ -32,6 +32,7 @@ enum class ProgramAction{ enum class ActionSource{ UI, // The Start/Stop button, or an integration (e.g. Discord). LAB, // A lab channel request. + CLI, // A --headless --run command-line launch. PROGRAM, // The program finished by itself. PROGRAM_ERROR, // The program ended with an error. }; diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp new file mode 100644 index 0000000000..710ece1b0c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp @@ -0,0 +1,35 @@ +/* Lab Startup Policy + * + * From: https://github.com/PokemonAutomation/ + */ +#include "LabStartupPolicy.h" + +namespace PokemonAutomation{ + +StartupPlan apply_lab_startup_policy(StartupPlan plan, const LabStartupPolicy& policy){ + if (plan.mode != StartupMode::GUI && plan.mode != StartupMode::HEADLESS){ + return plan; + } + if (policy.profiles == LabProfileMode::UNSUPPORTED){ + if (plan.mode == StartupMode::HEADLESS){ + return {.error = "--headless is supported only on macOS."}; + } + if (plan.profile || policy.bound_profile){ + return {.error = "Settings profiles are supported only on macOS."}; + } + return plan; + } + if (policy.bound_profile){ + if (!valid_startup_profile(*policy.bound_profile) || + (plan.profile && plan.profile != policy.bound_profile)){ + return {.error = "The requested profile does not match this lab build."}; + } + plan.profile = policy.bound_profile; + } + if (!plan.profile){ + return {.error = "Lab launch requires --profile NAME or a build configured with PA_LAB_PROFILE."}; + } + return plan; +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h new file mode 100644 index 0000000000..f8566c3998 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h @@ -0,0 +1,25 @@ +/* Lab Startup Policy + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabStartupPolicy_H +#define PokemonAutomation_LabStartupPolicy_H +#include "StartupArguments.h" + +namespace PokemonAutomation{ + +// Profiles are supported by the macOS settings backend, and required for lab GUIs. +enum class LabProfileMode{UNSUPPORTED, REQUIRED}; + +// Lab-only restrictions, independent of the general application argument grammar. +struct LabStartupPolicy{ + LabProfileMode profiles = LabProfileMode::UNSUPPORTED; + std::optional bound_profile = std::nullopt; +}; + +// Apply lab identity restrictions after parsing. Rejected plans stay rejected; +// tests bypass console identity, while GUI launches must satisfy the lab policy. +StartupPlan apply_lab_startup_policy(StartupPlan plan, const LabStartupPolicy& policy); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp new file mode 100644 index 0000000000..411fda9c49 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp @@ -0,0 +1,150 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Regression context: fork PR #226, 01e665911, lab controller/permissions lesson. + * A typo in a test command must never restore a saved controller panel. + */ +#include +#include +#include "LabStartupPolicy.h" +#include "LabStartupPolicy_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +struct LabStartupInput{ + std::string_view name; + std::vector arguments; + LabStartupPolicy policy; +}; + +struct LabStartupExpectation{ + StartupMode mode; + std::optional profile = std::nullopt; + bool augment = false; + int gui_calls = 0; + int headless_calls = 0; + int test_calls = 0; + int exit_code = 2; + std::vector qt_arguments = {}; +}; + +struct LabStartupCase{ + LabStartupInput input; + LabStartupExpectation expectation; +}; + +const LabStartupCase LAB_STARTUP_CASES[] = { + {.input = {.name = "unbound relaunch", .arguments = {}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "bound relaunch", .arguments = {}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "explicit identity", .arguments = {"--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "matching identity", .arguments = {"--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "wrong console", .arguments = {"--profile", "lab-A"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "missing profile", .arguments = {"--profile"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "empty profile", .arguments = {"--profile", ""}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "duplicate profile", .arguments = {"--profile", "lab-B", "--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "path traversal", .arguments = {"--profile", "../lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "absolute profile", .arguments = {"--profile", "/lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "space in profile", .arguments = {"--profile", "lab B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unicode profile", .arguments = {"--profile", "lab-é"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "option as profile", .arguments = {"--profile", "--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "test typo from incident", .arguments = {"--test", "bdsp"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown after test flag", .arguments = {"--command-line-test-mode", "bdsp"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unsupported filter", .arguments = {"--command-line-test-mode", "--test-filter=bdsp"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "test bypasses profile requirement", .arguments = {"--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::TESTS, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "test bypasses bound console", .arguments = {"--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::TESTS, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "augmented tests", .arguments = {"--augment-color-distortions", "--command-line-test-mode"}, .policy = {}}, + .expectation = {.mode = StartupMode::TESTS, .augment = true, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "augmentation without tests", .arguments = {"--augment-color-distortions"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "tests cannot select live profile", .arguments = {"--command-line-test-mode", "--profile", "lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "non mac GUI unchanged", .arguments = {}, .policy = {}}, + .expectation = {.mode = StartupMode::GUI, .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "unsupported profile platform", .arguments = {"--profile", "lab-B"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "invalid build identity", .arguments = {}, .policy = {LabProfileMode::REQUIRED, "../lab-A"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless with profile on mac", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::HEADLESS, .profile = "emu-1", .headless_calls = 1, .exit_code = 19}}, + {.input = {.name = "headless without profile on mac", .arguments = {"--headless", "--run", "UnitTestRunner"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless rejected off mac", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, +}; + +// The GUI callback represents settings load, window restoration and hardware discovery. +// A rejected or test launch must never reach it, even when a saved panel has devices. +UnitTestResult check_lab_startup_case(const LabStartupCase& test){ + const StartupPlan plan = apply_lab_startup_policy(parse_startup_arguments(test.input.arguments), test.input.policy); + const LabStartupExpectation& expected = test.expectation; + if (plan.mode != expected.mode || plan.profile != expected.profile || + plan.augment_color_distortions != expected.augment){ + return std::format("Expected mode {}, profile {}, augment {}; read mode {}, profile {}, augment {}.", + (int)expected.mode, expected.profile.value_or(""), expected.augment, + (int)plan.mode, plan.profile.value_or(""), plan.augment_color_distortions); + } + if (plan.qt_arguments != expected.qt_arguments){ + return "Expected Qt argument sequence differs from the preserved sequence."; + } + if ((plan.mode == StartupMode::REJECTED) != !plan.error.empty()){ + return "Expected rejection reason exactly when startup is rejected."; + } + int gui_calls = 0; + int headless_calls = 0; + int test_calls = 0; + const int code = dispatch_startup(plan, { + .gui = [&](const StartupPlan&){ gui_calls++; return 17; }, + .headless = [&](const StartupPlan&){ headless_calls++; return 19; }, + .tests = [&](const StartupPlan&){ test_calls++; return 23; }, + }); + if (gui_calls != expected.gui_calls || headless_calls != expected.headless_calls || + test_calls != expected.test_calls || code != expected.exit_code){ + return std::format("Expected GUI calls {}, headless calls {}, test calls {}, exit {}; read GUI calls {}, headless calls {}, test calls {}, exit {}.", + expected.gui_calls, expected.headless_calls, expected.test_calls, expected.exit_code, + gui_calls, headless_calls, test_calls, code); + } + return true; +} + +class LabStartupTest : public UnitTest{ +public: + explicit LabStartupTest(const LabStartupCase& test) + : UnitTest("Startup/LabPolicy/" + std::string(test.input.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_lab_startup_case(m_test); + } +private: + const LabStartupCase& m_test; +}; + +} // namespace + +void add_tests_LabStartupPolicy(UnitTestDatabase& database){ + for (const LabStartupCase& test : LAB_STARTUP_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h new file mode 100644 index 0000000000..786ab3fdc9 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h @@ -0,0 +1,15 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabStartupPolicy_Tests_H +#define PokemonAutomation_LabStartupPolicy_Tests_H +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register lab-policy and startup-dispatch regressions; no Qt or hardware is opened. +void add_tests_LabStartupPolicy(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp index 615379189d..e01e073fa3 100644 --- a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp +++ b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp @@ -76,25 +76,27 @@ void set_permissions(QObject& object){ } -bool check_resource_folder(Logger& logger){ +bool check_resource_folder(Logger& logger, StartupDialogs dialogs){ if (Filesystem::exists(Filesystem::Path(RESOURCE_PATH()))){ logger.log("Found Resource folder at " + RESOURCE_PATH()); return true; } logger.log("No Resources/ folder at " + RESOURCE_PATH()); - QMessageBox box; - box.setTextFormat(Qt::RichText); - box.critical( - nullptr, - "No Resources Folder!", - QString::fromStdString( - "No Resources Folder:

" - "Make sure you download Resources folder from latest release at
" - "https://github.com/PokemonAutomation/ComputerControl/releases/

" - "The Resources folder is part of PA-SerialPrograms-[VERSION]-[DATE].zip

" - "Extract the Resources folder from the zip file and place it at following path:

" + RESOURCE_PATH() - ) - ); + if (dialogs == StartupDialogs::SHOW){ + QMessageBox box; + box.setTextFormat(Qt::RichText); + box.critical( + nullptr, + "No Resources Folder!", + QString::fromStdString( + "No Resources Folder:

" + "Make sure you download Resources folder from latest release at
" + "https://github.com/PokemonAutomation/ComputerControl/releases/

" + "The Resources folder is part of PA-SerialPrograms-[VERSION]-[DATE].zip

" + "Extract the Resources folder from the zip file and place it at following path:

" + RESOURCE_PATH() + ) + ); + } return false; } diff --git a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h index b9e41d2baf..c9cd695b14 100644 --- a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h +++ b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h @@ -15,13 +15,19 @@ namespace PokemonAutomation{ class Logger; +// Controls whether startup checks open modal QMessageBox dialogs or only log. +enum class StartupDialogs{ + SHOW, + LOG_ONLY, +}; + // Use Qt to setup permissions of cameras and microphones on macOS. // See https://www.qt.io/blog/permission-apis-in-qt-6.5 void set_permissions(QObject& object); // Return true if there is the required Resources/ folder. -// When the folder is missing, it will also pop a critical error message box -bool check_resource_folder(Logger& logger); +// When the folder is missing and `dialogs == StartupDialogs::SHOW`, pops a critical error box. +bool check_resource_folder(Logger& logger, StartupDialogs dialogs = StartupDialogs::SHOW); } #endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp new file mode 100644 index 0000000000..061cd37ffc --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp @@ -0,0 +1,132 @@ +/* Application Startup Arguments + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include "StartupArguments.h" + +namespace PokemonAutomation{ +// Profile names are single ASCII path components; never silently rewrite an identity. +bool valid_startup_profile(std::string_view profile){ + return !profile.empty() && profile.front() != '-' && std::ranges::all_of(profile, [](char c){ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '-'; + }); +} + +namespace{ + +// Qt owns option values; recognize only its documented application option spellings. +// References: doc.qt.io/qt-6/{qapplication,qguiapplication}.html constructors. +enum class QtArgument{UNKNOWN, ACCEPTED, INVALID}; +QtArgument consume_qt_argument(std::span arguments, size_t& index, StartupPlan& plan){ + const std::string_view arg = arguments[index]; + if (arg == "-reverse" || arg == "-widgetcount"){ + plan.qt_arguments.emplace_back(arg); + return QtArgument::ACCEPTED; + } + constexpr std::string_view VALUE_OPTIONS[] = { + "-platform", "-platformpluginpath", "-platformtheme", "-plugin", + "-qwindowgeometry", "-qwindowicon", "-qwindowtitle", "-session", + "-display", "-geometry", "-style", "-stylesheet", + }; + if (std::ranges::find(VALUE_OPTIONS, arg) != std::end(VALUE_OPTIONS)){ + if (index + 1 == arguments.size() || arguments[index + 1].empty() || + arguments[index + 1].starts_with('-')){ + return QtArgument::INVALID; + } + plan.qt_arguments.emplace_back(arg); + plan.qt_arguments.emplace_back(arguments[++index]); + return QtArgument::ACCEPTED; + } + for (std::string_view prefix : {"-style=", "-stylesheet=", "-qmljsdebugger="}){ + if (arg.starts_with(prefix)){ + if (arg.size() == prefix.size()){ + return QtArgument::INVALID; + } + plan.qt_arguments.emplace_back(arg); + return QtArgument::ACCEPTED; + } + } + return QtArgument::UNKNOWN; +} + +StartupPlan validate_startup_plan(StartupPlan plan){ + if (plan.run_program && plan.mode != StartupMode::HEADLESS){ + return {.error = "--run requires --headless."}; + } + if (plan.options_path && !plan.run_program){ + return {.error = "--options requires --run."}; + } + if (plan.mode == StartupMode::HEADLESS && !plan.run_program){ + return {.error = "--headless requires --run ."}; + } + if (plan.augment_color_distortions && plan.mode != StartupMode::TESTS){ + return {.error = "--augment-color-distortions requires --command-line-test-mode."}; + } + if (plan.mode == StartupMode::TESTS && plan.profile){ + return {.error = "Test launches cannot select a console profile."}; + } + return plan; +} + +} // namespace + +StartupPlan parse_startup_arguments(std::span arguments){ + StartupPlan plan{.mode = StartupMode::GUI}; + for (size_t i = 0; i < arguments.size(); i++){ + const std::string_view arg = arguments[i]; + if (arg == "--profile"){ + if (plan.profile || i + 1 == arguments.size() || !valid_startup_profile(arguments[i + 1])){ + return {.error = "--profile requires one unique name containing only ASCII letters, digits, '_' or '-' (not starting with '-')."}; + } + plan.profile = arguments[++i]; + }else if (arg == "--headless"){ + if (plan.mode == StartupMode::TESTS){ + return {.error = "--headless and --command-line-test-mode cannot be combined."}; + } + plan.mode = StartupMode::HEADLESS; + }else if (arg == "--run"){ + if (plan.run_program || i + 1 == arguments.size() || arguments[i + 1].empty() || + arguments[i + 1].starts_with('-')){ + return {.error = "--run requires one unique program identifier."}; + } + plan.run_program = arguments[++i]; + }else if (arg == "--options"){ + if (plan.options_path || i + 1 == arguments.size() || arguments[i + 1].empty() || + arguments[i + 1].starts_with('-')){ + return {.error = "--options requires one unique file path."}; + } + plan.options_path = arguments[++i]; + }else if (arg == "--command-line-test-mode"){ + if (plan.mode == StartupMode::HEADLESS){ + return {.error = "--headless and --command-line-test-mode cannot be combined."}; + } + plan.mode = StartupMode::TESTS; + }else if (arg == "--augment-color-distortions"){ + plan.augment_color_distortions = true; + }else{ + const QtArgument result = consume_qt_argument(arguments, i, plan); + if (result != QtArgument::ACCEPTED){ + return {.error = "Unknown or incomplete startup argument: " + std::string(arg)}; + } + } + } + return validate_startup_plan(std::move(plan)); +} + +int dispatch_startup(const StartupPlan& plan, const StartupHandlers& handlers){ + switch (plan.mode){ + case StartupMode::GUI: + return handlers.gui(plan); + case StartupMode::HEADLESS: + return handlers.headless(plan); + case StartupMode::TESTS: + return handlers.tests(plan); + case StartupMode::REJECTED: + return 2; + } + return 2; +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h new file mode 100644 index 0000000000..d3470ba6b6 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h @@ -0,0 +1,52 @@ +/* Application Startup Arguments + * + * From: https://github.com/PokemonAutomation/ + * + * Parse before Qt, settings, logging or device discovery. Example: + * open -n SerialPrograms.app --args --profile lab-B + */ +#ifndef PokemonAutomation_StartupArguments_H +#define PokemonAutomation_StartupArguments_H + +#include +#include +#include +#include +#include +#include + +namespace PokemonAutomation{ + +// Rejected launches must not enter any application entry point. +enum class StartupMode{REJECTED, GUI, HEADLESS, TESTS}; + +// Validated startup intent; error describes a rejected launch. +struct StartupPlan{ + StartupMode mode = StartupMode::REJECTED; + std::optional profile = std::nullopt; + std::optional run_program = std::nullopt; + std::optional options_path = std::nullopt; + bool augment_color_distortions = false; + std::string error = {}; + std::vector qt_arguments = {}; +}; + +// Check a single ASCII profile path component without rewriting its identity. +bool valid_startup_profile(std::string_view profile); + +// Parse argv without argv[0]; preserve documented Qt options for Qt to interpret. +// Unknown or malformed application/Qt arguments fail before settings or devices. +StartupPlan parse_startup_arguments(std::span arguments); + +// Application entry points: the GUI and headless callbacks own all settings and hardware startup. +struct StartupHandlers{ + std::function gui; + std::function headless; + std::function tests; +}; + +// Dispatch a validated plan. Rejection returns 2 without invoking either callback. +int dispatch_startup(const StartupPlan& plan, const StartupHandlers& handlers); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp new file mode 100644 index 0000000000..2f88d67df1 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp @@ -0,0 +1,110 @@ +/* Application Startup Argument Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include "StartupArguments.h" +#include "StartupArguments_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +struct ArgumentInput{ + std::string_view name; + std::vector arguments; +}; + +struct ArgumentExpectation{ + StartupMode mode; + std::optional profile = std::nullopt; + std::vector qt_arguments = {}; +}; + +struct ArgumentCase{ + ArgumentInput input; + ArgumentExpectation expectation; +}; + +const ArgumentCase ARGUMENT_CASES[] = { + {.input = {.name = "ordinary GUI", .arguments = {}}, + .expectation = {.mode = StartupMode::GUI}}, + {.input = {.name = "optional profile", .arguments = {"--profile", "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B"}}, + {.input = {.name = "test dispatch", .arguments = {"--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::TESTS}}, + {.input = {.name = "Qt platform", .arguments = {"-platform", "offscreen"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-platform", "offscreen"}}}, + {.input = {.name = "Qt style assignment", .arguments = {"-style=Fusion"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-style=Fusion"}}}, + {.input = {.name = "Qt style pair", .arguments = {"-style", "Fusion"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-style", "Fusion"}}}, + {.input = {.name = "Qt stylesheet assignment", .arguments = {"-stylesheet=some path.qss"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-stylesheet=some path.qss"}}}, + {.input = {.name = "Qt flags and application profile", .arguments = {"-reverse", "--profile", "lab-B", "-widgetcount"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .qt_arguments = {"-reverse", "-widgetcount"}}}, + {.input = {.name = "repeated Qt plugin", .arguments = {"-plugin", "one", "-plugin", "two"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-plugin", "one", "-plugin", "two"}}}, + {.input = {.name = "Qt missing value", .arguments = {"-platform"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt empty value", .arguments = {"-style", ""}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt empty assignment", .arguments = {"-style="}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt must not swallow application option", .arguments = {"-platform", "--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt must not swallow typo", .arguments = {"-platform", "--test", "bdsp"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown Qt-looking option", .arguments = {"-platfrom", "offscreen"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown after valid Qt option", .arguments = {"-reverse", "--test", "bdsp"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless run", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}}, + .expectation = {.mode = StartupMode::HEADLESS, .profile = "emu-1"}}, + {.input = {.name = "headless run with options", .arguments = {"--headless", "--run", "UnitTestRunner", "--options", "opts.json"}}, + .expectation = {.mode = StartupMode::HEADLESS}}, + {.input = {.name = "headless without run", .arguments = {"--headless", "--profile", "emu-1"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "run without headless", .arguments = {"--run", "UnitTestRunner"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "options without run", .arguments = {"--headless", "--options", "opts.json"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless with tests", .arguments = {"--headless", "--run", "UnitTestRunner", "--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::REJECTED}}, +}; + +UnitTestResult check_argument_case(const ArgumentCase& test){ + const StartupPlan plan = parse_startup_arguments(test.input.arguments); + const ArgumentExpectation& expected = test.expectation; + if (plan.mode != expected.mode || plan.profile != expected.profile){ + return std::format("Expected mode {} and profile {}; read mode {} and profile {}.", + (int)expected.mode, expected.profile.value_or(""), + (int)plan.mode, plan.profile.value_or("")); + } + if (plan.qt_arguments != expected.qt_arguments){ + return "Expected the complete Qt argument sequence to be preserved in order."; + } + return true; +} + +class ArgumentTest : public UnitTest{ +public: + explicit ArgumentTest(const ArgumentCase& test) + : UnitTest("Startup/Arguments/" + std::string(test.input.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_argument_case(m_test); + } +private: + const ArgumentCase& m_test; +}; + +} // namespace + +void add_tests_StartupArguments(UnitTestDatabase& database){ + for (const ArgumentCase& test : ARGUMENT_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h new file mode 100644 index 0000000000..665e0bed7a --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h @@ -0,0 +1,15 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_StartupArguments_Tests_H +#define PokemonAutomation_StartupArguments_Tests_H +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register parser and startup-dispatch regressions; no Qt or hardware is opened. +void add_tests_StartupArguments(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index fc6ceeceaf..401dcd5425 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -141,21 +141,14 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_capture_session.reset(new QMediaCaptureSession()); m_capture_session->setCamera(&m_camera->camera()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() == QCamera::NoError){ - return; - } - m_logger.log("QCamera error: " + m_camera->errorString().toStdString(), COLOR_RED); - }); - - m_camera->start(); -#endif } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp index 13b6dff18d..c801cb6067 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp @@ -137,19 +137,15 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_video_sink.reset(new QVideoSink()); m_capture.reset(new QMediaCaptureSession()); m_capture->setCamera(&m_camera->camera()); m_capture->setVideoSink(m_video_sink.get()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() != QCamera::NoError){ - m_logger.log("QCamera error: " + m_camera->errorString().toStdString()); - } - }); -#endif m_metaobject->connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, &m_camera->camera(), [&](const QVideoFrame& frame){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp index 5264a3aea9..0160ac7aeb 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp @@ -139,19 +139,15 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_video_sink.reset(new QVideoSink()); m_capture.reset(new QMediaCaptureSession()); m_capture->setCamera(&m_camera->camera()); m_capture->setVideoSink(m_video_sink.get()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() != QCamera::NoError){ - m_logger.log("QCamera error: " + m_camera->errorString().toStdString()); - } - }); -#endif m_metaobject->connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, &m_camera->camera(), [&](const QVideoFrame& frame){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h index 9db0137e9d..2c6235f32c 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h @@ -8,8 +8,10 @@ #define PokemonAutomation_VideoPipeline_QCameraThread_H #include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Qt/GlobalThreadPoolsQt.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" //#include //using std::cout; @@ -18,14 +20,25 @@ namespace PokemonAutomation{ +// Own the camera and its device lease across the Qt event-thread lifetime. class QCameraThread : public QObject{ public: + // A denied input must not throw across run_on_main_thread_and_wait's Qt callback. + static std::unique_ptr try_create(Logger& logger, QCameraDevice device, QCameraFormat format){ + try{ + return std::make_unique(logger, std::move(device), std::move(format)); + }catch (UserSetupError&){ + return nullptr; // UserSetupError already logged the refusal. + } + } + QCameraThread( Logger& logger, QCameraDevice device, QCameraFormat format ) - : m_logger(logger) + : m_device_lease(acquire_lab_capture(logger, LabDeviceKind::VIDEO, device.id().toHex().toStdString())) + , m_logger(logger) , m_device(std::move(device)) , m_format(std::move(format)) , m_camera(nullptr) @@ -63,6 +76,7 @@ class QCameraThread : public QObject{ private: + std::unique_ptr m_device_lease; Logger& m_logger; QCameraDevice m_device; QCameraFormat m_format; @@ -71,5 +85,5 @@ class QCameraThread : public QObject{ -} +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/CommonFramework/Windows/LabStateJson.cpp b/SerialPrograms/Source/CommonFramework/Windows/LabStateJson.cpp index 2b0ed03c4d..eb31f9fb1a 100644 --- a/SerialPrograms/Source/CommonFramework/Windows/LabStateJson.cpp +++ b/SerialPrograms/Source/CommonFramework/Windows/LabStateJson.cpp @@ -177,6 +177,8 @@ std::string_view source_name(ActionSource source){ return "ui"; case ActionSource::LAB: return "lab"; + case ActionSource::CLI: + return "cli"; case ActionSource::PROGRAM: return "program"; case ActionSource::PROGRAM_ERROR: diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index b398baa859..f5c6380545 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -35,6 +35,11 @@ #include "CommonFramework/Windows/LabProgramContract_Tests.h" #include "CommonFramework/Windows/LabSnapshot_Tests.h" #include "CommonFramework/Windows/LabStateJson_Tests.h" +#include "CommonFramework/Headless/HeadlessHost_Tests.h" +#include "CommonFramework/Startup/StartupArguments_Tests.h" +#include "PanelCatalog_Tests.h" +#include "CommonFramework/Startup/LabStartupPolicy_Tests.h" +#include "CommonFramework/Devices/LabDeviceAccessQt_Tests.h" #include "CommonTools/VisualDetectors/BlackBorderDetector.h" #include "CommonTools/OCR/OCR_Tests.h" @@ -61,6 +66,23 @@ namespace PokemonAutomation{ namespace ComputerPrograms{ +namespace{ + +bool matches_any(std::string_view test_name, std::span filters){ + if (filters.empty()){ + return true; + } + for (std::string_view filter : filters){ + if (test_name.contains(filter)){ + return true; + } + } + return false; +} + +} // namespace + + @@ -94,6 +116,11 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_LabStateJson(ret); add_tests_ProgramPrecheckRun(ret); add_tests_ProgramTransition(ret); + add_tests_StartupArguments(ret); + add_tests_PanelCatalog(ret); + add_tests_LabStartupPolicy(ret); + add_tests_HeadlessHost(ret); + add_tests_LabDeviceAccess(ret); OCR::add_tests(ret); Kernels::add_tests(ret); Pokemon::add_tests_AutoStorySelect(ret); @@ -122,6 +149,17 @@ const UnitTestDatabase& UNIT_TESTS_ALL(){ static UnitTestDatabase database = make_UNIT_TESTS_ALL(); return database; } + +std::vector unit_test_names(std::span filters){ + std::vector ret; + for (const auto& [test_name, test] : UNIT_TESTS_ALL()){ + if (matches_any(test_name, filters)){ + ret.emplace_back(test_name); + } + } + return ret; +} + StringSelectDatabase make_TEST_DATABASE(){ StringSelectDatabase database; for (auto& test : UNIT_TESTS_ALL()){ @@ -406,7 +444,7 @@ void UnitTestRunner::on_test_finished( -bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::string& test_substring){ +size_t CommandLineUnitTestRunner::run(bool augment_color_distortions, std::span filters){ if (augment_color_distortions){ // Load the distortion data before any test runs. Each `ImageUnitTest` checks the global // manager when it runs, and only runs with distortions if it is loaded. @@ -414,7 +452,7 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s initialize_test_image_distortions(UNIT_TEST_RESOURCE_PATH()); }catch (Exception& e){ m_logger.log("Unable to load the test image distortions: " + e.to_str(), COLOR_RED); - return true; + return 1; } const TestImageDistortionManager& manager = *test_image_distortion_manager(); std::string labels; @@ -432,14 +470,9 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s GlobalThreadPools::unlimited_normal() ); runner.add_listener(*this); - size_t num_matching = add_tests_matching(test_substring, [&](std::shared_ptr test){ - runner.add_test(test); - }); - if (!test_substring.empty()){ - if (num_matching == 0){ - m_logger.log("No test name contains \"" + test_substring + "\". No tests will be run.", COLOR_ORANGE); - }else{ - m_logger.log("Running the " + tostr_u_commas(num_matching) + " tests whose name contains \"" + test_substring + "\""); + for (const auto& [test_name, test] : UNIT_TESTS_ALL()){ + if (matches_any(test_name, filters)){ + runner.add_test(test); } } runner.run(); @@ -450,7 +483,7 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions, const std::s "\n Failed: " + tostr_u_commas(m_failed_tests.load(std::memory_order_acquire)) + "\n Skipped: " + tostr_u_commas(m_skipped_tests.load(std::memory_order_acquire)) ); - return m_failed_tests.load(std::memory_order_acquire) > 0; + return m_failed_tests.load(std::memory_order_acquire); } void CommandLineUnitTestRunner::on_test_finished( std::shared_ptr test, diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h index 3baffcc745..10f69c8ab8 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h @@ -8,6 +8,10 @@ #define PokemonAutomation_ComputerPrograms_UnitTestRunner_H #include +#include +#include +#include +#include #include "Common/Cpp/TestRunners/ParallelUnitTestRunner.h" #include "Common/Cpp/Options/StaticTextOption.h" #include "Common/Cpp/Options/EnumDropdownOption.h" @@ -89,15 +93,13 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List , m_failed_tests(0) {} - // Run the unit tests. Return true if any test failed. + // Run the unit tests whose names contain any of `filters`, or all of them if `filters` is + // empty. Return the number of failed tests. // If `augment_color_distortions` is true, also run each image test under different color // distortion augmentation to widen the test coverage, using the distortion data in the // CaptureCardCalibration/ folder of the test repo. If that data can't be loaded, log the - // error and return true without running any test. - // If `test_substring` is not empty, only run the tests whose name contains that substring - // (case sensitive, the same matching as the GUI's "Run All Tests Containing This Substring" - // mode). If nothing matches, no test runs and a warning is logged. - bool run(bool augment_color_distortions, const std::string& test_substring); + // error and return 1 without running any test. + size_t run(bool augment_color_distortions, std::span filters); private: @@ -115,6 +117,11 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List }; +// The names of the unit tests that contain any of `filters`, or of all the tests if `filters` +// is empty, in name order. +std::vector unit_test_names(std::span filters); + + } diff --git a/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.cpp b/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.cpp index ce8efbbbe0..0b017bd33c 100644 --- a/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.cpp +++ b/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.cpp @@ -109,6 +109,14 @@ void SerialPABotBase2_Connection::naked_wait(WallDuration duration){ std::unique_lock lg(lock); cv.wait_until(lg, deadline, [=]{ return current_time() >= deadline; }); } +void SerialPABotBase2_Connection::open_guarded_serial_port(){ +#ifdef QT_CORE_LIB + m_device_lease = acquire_lab_serial(m_logger, m_device_name); +#endif + m_unreliable_connection = std::make_unique( + GlobalThreadPools::unlimited_realtime(), m_device_name, 115200 + ); +} bool SerialPABotBase2_Connection::open_serial_port(){ { std::string text = "Opening serial port..."; @@ -156,11 +164,7 @@ bool SerialPABotBase2_Connection::open_serial_port(){ return false; } - m_unreliable_connection = std::make_unique( - GlobalThreadPools::unlimited_realtime(), - m_device_name, - 115200 - ); + open_guarded_serial_port(); return true; } @@ -321,6 +325,13 @@ void SerialPABotBase2_Connection::connect_thread_body(){ if (ok){ declare_ready(); }else{ + // Close native resources before releasing the lease on any failed open/handshake. + m_device.reset(); + m_stream_connection.reset(); + m_unreliable_connection.reset(); +#ifdef QT_CORE_LIB + m_device_lease.reset(); +#endif declare_failed(); } }); diff --git a/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.h b/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.h index 61ab4af666..81892aecfc 100644 --- a/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.h +++ b/SerialPrograms/Source/Controllers/PABotBase2/SerialPABotBase2_Connection.h @@ -10,6 +10,9 @@ #include "Common/Cpp/Concurrency/AsyncTask.h" #include "Common/Cpp/SerialConnection/SerialConnection.h" #include "Common/PABotBase2/ReliableConnectionLayer/PABotBase2CC_ReliableStreamConnection.h" +#ifdef QT_CORE_LIB +#include "CommonFramework/Devices/LabDeviceRuntime.h" +#endif #include "Controllers/SerialPort/SerialLogger.h" #include "Controllers/PABotBase2/PABotBase2_Connection.h" @@ -36,6 +39,7 @@ class SerialPABotBase2_Connection final : public PABotBase2::Connection{ private: void naked_wait(WallDuration duration); + void open_guarded_serial_port(); bool open_serial_port(); bool try_connect_to_device(WallDuration timeout); bool connect_to_device(); @@ -49,6 +53,10 @@ class SerialPABotBase2_Connection final : public PABotBase2::Connection{ std::string m_device_name; // Mutex m_lock; +#ifdef QT_CORE_LIB + // Declared before the port: release only after the physical connection closes. + std::unique_ptr m_device_lease; +#endif AsyncTask m_connect_thread; std::unique_ptr m_unreliable_connection; std::unique_ptr m_stream_connection; diff --git a/SerialPrograms/Source/GameConsole/UI/ConsoleSystemWidget.cpp b/SerialPrograms/Source/GameConsole/UI/ConsoleSystemWidget.cpp index ec02dd74e1..f8d434fc65 100644 --- a/SerialPrograms/Source/GameConsole/UI/ConsoleSystemWidget.cpp +++ b/SerialPrograms/Source/GameConsole/UI/ConsoleSystemWidget.cpp @@ -8,6 +8,7 @@ #include #include "CommandRowWidget.h" #include "ConsoleSystemWidget.h" +#include "CommonFramework/GlobalSettingsPanel.h" namespace PokemonAutomation{ @@ -37,7 +38,10 @@ ConsoleSystemWidget::ConsoleSystemWidget( layout->setContentsMargins(0, 0, 0, 0); layout->setAlignment(Qt::AlignTop); - m_group_box = new CollapsibleGroupBox(*this, "Console " + QString::number(m_session.console_number()) + " Settings"); + m_group_box = new CollapsibleGroupBox( + *this, "Console " + QString::number(m_session.console_number()) + " Settings", + !GlobalSettings::instance().CONSOLE_SETTINGS_COLLAPSED + ); layout->addWidget(m_group_box); { diff --git a/SerialPrograms/Source/PanelCatalog.cpp b/SerialPrograms/Source/PanelCatalog.cpp new file mode 100644 index 0000000000..545a036e92 --- /dev/null +++ b/SerialPrograms/Source/PanelCatalog.cpp @@ -0,0 +1,66 @@ +/* Panel Catalog + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/StaticGlobals.h" +#include "ML/ML_Panels.h" +#include "NintendoSwitch/NintendoSwitch_Panels.h" +#include "VideoGames/PokemonSwSh/PokemonSwSh_Panels.h" +#include "VideoGames/PokemonHome/PokemonHome_Panels.h" +#include "PokemonBDSP/PokemonBDSP_Panels.h" +#include "VideoGames/PokemonFRLG/PokemonFRLG_Panels.h" +#include "VideoGames/PokemonLA/PokemonLA_Panels.h" +#include "VideoGames/PokemonLGPE/PokemonLGPE_Panels.h" +#include "VideoGames/PokemonRSE/PokemonRSE_Panels.h" +#include "VideoGames/PokemonSV/PokemonSV_Panels.h" +#include "VideoGames/PokemonLZA/PokemonLZA_Panels.h" +#include "VideoGames/PokemonPokopia/PokemonPokopia_Panels.h" +#include "VideoGames/ZeldaTotK/ZeldaTotK_Panels.h" +#include "PanelCatalog.h" + +namespace PokemonAutomation{ + + +std::vector> make_all_panel_lists(){ + std::vector> ret; + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + ret.emplace_back(std::make_unique()); + if (STATIC_GLOBALS.DEVELOPER_MODE){ + ret.emplace_back(std::make_unique()); + } + + ret.emplace_back(std::make_unique()); + + if (STATIC_GLOBALS.DEVELOPER_MODE){ + ret.emplace_back(std::make_unique()); + } + return ret; +} + +std::optional find_panel( + const std::vector>& lists, + std::string_view identifier +){ + for (size_t c = 0; c < lists.size(); c++){ + for (PanelEntry& entry : lists[c]->panels()){ + if (entry.descriptor == nullptr || entry.descriptor->identifier() != identifier){ + continue; + } + return PanelMatch{.category_index = c, .entry = std::move(entry)}; + } + } + return std::nullopt; +} + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PanelCatalog.h b/SerialPrograms/Source/PanelCatalog.h new file mode 100644 index 0000000000..6d2eb49ad2 --- /dev/null +++ b/SerialPrograms/Source/PanelCatalog.h @@ -0,0 +1,38 @@ +/* Panel Catalog + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PanelCatalog_H +#define PokemonAutomation_PanelCatalog_H + +#include +#include +#include +#include +#include +#include "CommonFramework/Panels/PanelList.h" + +namespace PokemonAutomation{ + + +// Every program category, in dropdown order. +std::vector> make_all_panel_lists(); + + +// A panel found by find_panel(), with the index of its list in `lists`. +struct PanelMatch{ + size_t category_index; + PanelEntry entry; +}; + +// Scans each list's panels() for the panel with `identifier`. +std::optional find_panel( + const std::vector>& lists, + std::string_view identifier +); + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PanelCatalog_Tests.cpp b/SerialPrograms/Source/PanelCatalog_Tests.cpp new file mode 100644 index 0000000000..dccf8ba173 --- /dev/null +++ b/SerialPrograms/Source/PanelCatalog_Tests.cpp @@ -0,0 +1,125 @@ +/* Panel Catalog Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/Panels/PanelDescriptor.h" +#include "CommonFramework/Panels/PanelList.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "PanelCatalog.h" +#include "PanelCatalog_Tests.h" + +namespace PokemonAutomation{ +namespace{ + + +class FakePanelDescriptor final : public PanelDescriptor{ +public: + explicit FakePanelDescriptor(std::string identifier) + : PanelDescriptor( + identifier, + /*category=*/"FakeCategory", + /*display_name=*/identifier, + /*doc_link=*/"", + /*description=*/"", + PanelQualityStanding::DEVELOPER, + /*restore_defaults_button=*/false + ) + {} + + virtual std::unique_ptr make_panel() const override{ + return std::make_unique(*this); + } +}; + +// One category: a divider, then the panel "Fake:Alpha". +class FakePanelList final : public PanelListDescriptor{ +public: + FakePanelList() + : PanelListDescriptor("FakeList") + {} + +protected: + virtual std::vector make_panels() const override{ + std::vector ret; + ret.emplace_back("Divider"); + ret.emplace_back(std::make_unique("Fake:Alpha")); + return ret; + } +}; + +class DecliningPanelHolder final : public PanelHolder{ +public: + virtual bool report_new_panel_intent(const PanelDescriptor&) override{ + return false; + } + virtual void load_panel( + std::shared_ptr, + std::unique_ptr + ) override{ + loaded = true; + } + virtual void on_busy() override{} + virtual void on_idle() override{} + + bool loaded = false; +}; + + +class FindPanelPastDividerTest final : public UnitTest{ +public: + FindPanelPastDividerTest() + : UnitTest("PanelCatalog/FindPanel/past a divider") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + std::vector> lists; + lists.emplace_back(std::make_unique()); + + const std::optional match = find_panel(lists, "Fake:Alpha"); + if (!match){ + return "find_panel() did not find Fake:Alpha."; + } + if (match->category_index != 0){ + return "find_panel() returned the wrong category index."; + } + if (match->entry.descriptor == nullptr || match->entry.descriptor->identifier() != "Fake:Alpha"){ + return "find_panel() returned the wrong panel."; + } + return true; + } +}; + +class LoadPanelSessionDeclinedTest final : public UnitTest{ +public: + LoadPanelSessionDeclinedTest() + : UnitTest("PanelCatalog/LoadPanelSession/declined intent skips load") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + DecliningPanelHolder holder; + const std::shared_ptr descriptor = + std::make_shared("Fake:Alpha"); + + if (load_panel_session(descriptor, JsonValue(), holder)){ + return "load_panel_session() returned true after the holder declined."; + } + if (holder.loaded){ + return "load_panel_session() loaded a panel after the holder declined."; + } + return true; + } +}; + + +} // namespace + +void add_tests_PanelCatalog(UnitTestDatabase& database){ + database.add(); + database.add(); +} + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PanelCatalog_Tests.h b/SerialPrograms/Source/PanelCatalog_Tests.h new file mode 100644 index 0000000000..c2883043c9 --- /dev/null +++ b/SerialPrograms/Source/PanelCatalog_Tests.h @@ -0,0 +1,18 @@ +/* Panel Catalog Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PanelCatalog_Tests_H +#define PokemonAutomation_PanelCatalog_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register PanelCatalog and load_panel_session() unit tests. +void add_tests_PanelCatalog(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PanelLists.cpp b/SerialPrograms/Source/PanelLists.cpp index cf857b7c9c..054dd61006 100644 --- a/SerialPrograms/Source/PanelLists.cpp +++ b/SerialPrograms/Source/PanelLists.cpp @@ -7,24 +7,11 @@ #include #include #include "Common/Qt/NoWheelComboBox.h" -#include "CommonFramework/StaticGlobals.h" #include "CommonFramework/PersistentSettings.h" #include "CommonFramework/ImageTypes/ImageRGB32_Qt.h" #include "CommonFramework/Windows/DpiScaler.h" #include "CommonFramework/Panels/UI/PanelListWidget.h" -#include "ML/ML_Panels.h" -#include "NintendoSwitch/NintendoSwitch_Panels.h" -#include "VideoGames/PokemonSwSh/PokemonSwSh_Panels.h" -#include "VideoGames/PokemonHome/PokemonHome_Panels.h" -#include "PokemonBDSP/PokemonBDSP_Panels.h" -#include "VideoGames/PokemonFRLG/PokemonFRLG_Panels.h" -#include "VideoGames/PokemonLA/PokemonLA_Panels.h" -#include "VideoGames/PokemonLGPE/PokemonLGPE_Panels.h" -#include "VideoGames/PokemonRSE/PokemonRSE_Panels.h" -#include "VideoGames/PokemonSV/PokemonSV_Panels.h" -#include "VideoGames/PokemonLZA/PokemonLZA_Panels.h" -#include "VideoGames/PokemonPokopia/PokemonPokopia_Panels.h" -#include "VideoGames/ZeldaTotK/ZeldaTotK_Panels.h" +#include "PanelCatalog.h" #include "PanelLists.h" //#include @@ -59,24 +46,8 @@ ProgramSelect::ProgramSelect(QWidget& parent, PanelHolder& holder) layout->addWidget(m_dropdown); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - add(std::make_unique()); - if (STATIC_GLOBALS.DEVELOPER_MODE){ - add(std::make_unique()); - } - - add(std::make_unique()); - - if (STATIC_GLOBALS.DEVELOPER_MODE){ - add(std::make_unique()); + for (std::unique_ptr& list : make_all_panel_lists()){ + add(std::move(list)); } @@ -118,8 +89,9 @@ void ProgramSelect::add(std::unique_ptr list){ m_lists.emplace_back(std::move(list)); const PanelListDescriptor& back = *m_lists.back(); if (!m_tab_map.emplace(back.name(), index).second){ + const std::string name = back.name(); m_lists.pop_back(); - throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Duplicate Category Name: " + back.name()); + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Duplicate Category Name: " + name); } if (!back.enabled()){ auto* model = qobject_cast(m_dropdown->model()); @@ -152,20 +124,16 @@ void ProgramSelect::load_persistent_panel(){ } std::string ProgramSelect::select_program(const std::string& identifier){ - for (size_t c = 0; c < m_lists.size(); c++){ - for (const PanelEntry& entry : m_lists[c]->panels()){ - if (entry.descriptor == nullptr || entry.descriptor->identifier() != identifier){ - continue; - } - if (m_locked){ - return "A program is running."; - } - change_list((int)c); - m_active_list->set_panel(entry.display_name); - return ""; - } + const std::optional match = find_panel(m_lists, identifier); + if (!match){ + return "No program with identifier \"" + identifier + "\"."; + } + if (m_locked){ + return "A program is running."; } - return "No program with identifier \"" + identifier + "\"."; + change_list((int)match->category_index); + m_active_list->set_panel(match->entry.display_name); + return ""; } void ProgramSelect::change_list(int index){ diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index 90a9830510..4c73b74376 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -18,7 +18,11 @@ using std::endl; namespace PokemonAutomation{ -int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring){ +int run_command_line_tests(bool augment_color_distortions){ + return run_unit_tests(augment_color_distortions, {}) == 0 ? 0 : 1; +} + +size_t run_unit_tests(bool augment_color_distortions, std::span filters){ install_crash_backtrace_handlers(); // Keep stdout line-buffered: when piped, stdio defaults to full // buffering, and the last few KB of test output - including the lines @@ -26,14 +30,11 @@ int run_command_line_tests(bool augment_color_distortions, const std::string& te // explicit buffer size is required: with 0, MSVC's ucrt can end up with // a zero-size buffer, which corrupts the next stdout write. setvbuf(stdout, nullptr, _IOLBF, 8192); - if (test_substring.empty()){ - cout << "Running parallel unit tests..." << endl; - }else{ - cout << "Running parallel unit tests matching substring \"" << test_substring << "\"..." << endl; - } + cout << "Running parallel unit tests..." << endl; ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions, test_substring)){ - return 1; + size_t failed = runner.run(augment_color_distortions, filters); + if (failed > 0){ + return failed; } cout << "Running parallel unit tests... Done!" << endl; return 0; diff --git a/SerialPrograms/Source/Tests/CommandLineTests.h b/SerialPrograms/Source/Tests/CommandLineTests.h index bea6858b51..b42c0b0c6b 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.h +++ b/SerialPrograms/Source/Tests/CommandLineTests.h @@ -5,7 +5,9 @@ * Run all the unit tests from the command line, without launching the GUI. * * How to run: - * Launch the program with the argument "--command-line-test-mode", or set + * Run the SerialProgramsTests executable (Tests/TestRunner_Main.cpp): it runs only the tests, + * with no GUI, no settings and no device scans, and takes "--filter " and "--list". + * Or launch the program with the argument "--command-line-test-mode", or set * "20-GlobalSettings": "COMMAND_LINE_TESTS": "RUN" to true in SerialPrograms-Settings.json. * * How the tests work: @@ -32,11 +34,6 @@ * loaded, no test runs and the program returns 1. See SerialPrograms/Source/Tests/ImageUnitTest.h * and Common/Cpp/TestRunners/TestImageDistortionManager.h. * - * Substring filter: - * Pass "--test-substring " to only run the tests whose name contains (case - * sensitive, the same matching as the GUI's "Run All Tests Containing This Substring" mode). - * Without it, all the tests are run. An empty value runs all the tests too. - * * The same tests can also be run from the GUI with the "Unit Test Runner" program, which can * run a single test, the tests matching a substring, or all tests. */ @@ -45,9 +42,9 @@ #ifndef PokemonAutomation_Tests_CommandLineTests_H #define PokemonAutomation_Tests_CommandLineTests_H - -#include - +#include +#include +#include namespace PokemonAutomation{ @@ -56,10 +53,13 @@ namespace PokemonAutomation{ // This function is only called when GlobalSettings::COMMAND_LINE_TEST_MODE is true. // `augment_color_distortions` is set by the "--augment-color-distortions" argument: also run image tests // with color distortion augmentation, using the distortion data from the test repo. -// `test_substring` is set by the "--test-substring " argument: only run the tests whose name -// contains that substring. An empty string runs all the tests. // Return 0 if all tests are passed, and 1 if any test failed or the distortion data can't be loaded. -int run_command_line_tests(bool augment_color_distortions, const std::string& test_substring); +int run_command_line_tests(bool augment_color_distortions); + +// Run the unit tests whose names contain any of `filters`, or all of them if `filters` is empty, +// and print the results to stdout. Return the number of failed tests, or 1 if +// `augment_color_distortions` is set and the distortion data can't be loaded. +size_t run_unit_tests(bool augment_color_distortions, std::span filters); diff --git a/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake b/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake new file mode 100644 index 0000000000..f57621abf3 --- /dev/null +++ b/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake @@ -0,0 +1,19 @@ +# This cmake file is to be included by CMakeLists.txt to build SerialProgramsTests, an executable +# that runs the unit tests without the GUI (see Source/Tests/TestRunner_Main.cpp). + +add_executable(SerialProgramsTests Source/Tests/TestRunner_Main.cpp) + +# The tests are registered by explicit calls, not by static constructors, so the linker may +# leave out the parts of the library that the tests don't use. +target_link_libraries(SerialProgramsTests PRIVATE SerialProgramsLib) +apply_common_target_properties(SerialProgramsTests) + +# The tests read Resources/ from in or above the executable's folder, like SerialPrograms. +add_dependencies(SerialProgramsTests clone_packages) +if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/Resources") + add_custom_command( + TARGET SerialProgramsTests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "[Deploy] Copying resources ${RESOURCES_DIR} to $/Resources" + COMMAND ${CMAKE_COMMAND} -E copy_directory "${RESOURCES_DIR}" "$/Resources" + ) +endif() diff --git a/SerialPrograms/Source/Tests/TestRunner_Main.cpp b/SerialPrograms/Source/Tests/TestRunner_Main.cpp new file mode 100644 index 0000000000..2a5f12511d --- /dev/null +++ b/SerialPrograms/Source/Tests/TestRunner_Main.cpp @@ -0,0 +1,182 @@ +/* Unit Test Runner Main + * + * From: https://github.com/PokemonAutomation/ + * + * SerialProgramsTests: an executable that runs the unit tests and nothing else. It runs the + * same tests as "SerialPrograms --command-line-test-mode" (see Tests/CommandLineTests.h), but + * opens no window, reads and writes no settings, and does not look for serial ports or cameras. + * So it can run while SerialPrograms is open on the same computer. + * + * Usage: + * SerialProgramsTests [--filter ]... [--list] [--augment-color-distortions] + * + * --filter Select the tests whose names contain . Repeat it to + * select the tests that match any of them. Default: all tests. + * --list Print the names of the selected tests instead of running them. + * --augment-color-distortions Also run the image tests with color distortions. + * + * Exit code: the number of failed tests (at most 254), or 255 for a bad argument or when no + * test matches the filters. + * + * It finds Resources/ and the CommandLineTests repo the same way SerialPrograms does: in or + * above the folder that holds the executable. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/Qt6.9ThreadBugWorkaround.h" +#include "Common/Cpp/Concurrency/FireForgetDispatcher.h" +#include "Common/Cpp/Concurrency/PeriodicRunner.h" +#include "Common/Cpp/Concurrency/Watchdog.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/FileLogger.h" +#include "Common/Cpp/Logging/GlobalLogger.h" +#include "Common/Cpp/Logging/LastLogTracker.h" +#include "Common/Cpp/Logging/MultiOutputLogger.h" +#include "Common/Qt/GlobalThreadPoolsQt.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/GlobalServices.h" +#include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/Tools/GlobalThreadPools.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" +#include "Tests/CommandLineTests.h" + +namespace PokemonAutomation{ +namespace{ + +constexpr int EXIT_BAD_ARGUMENTS = 255; +constexpr size_t MAX_FAILURE_EXIT_CODE = 254; + +struct TestRunnerArguments{ + // Views of argv, which outlives every use. + std::vector filters; + bool list = false; + bool augment_color_distortions = false; +}; + +// Return nothing if an argument is unknown or incomplete. +std::optional parse_arguments(int argc, char* argv[]){ + TestRunnerArguments arguments; + for (int i = 1; i < argc; i++){ + const std::string_view arg = argv[i]; + if (arg == "--filter" && i + 1 < argc){ + arguments.filters.emplace_back(argv[++i]); + }else if (arg == "--list"){ + arguments.list = true; + }else if (arg == "--augment-color-distortions"){ + arguments.augment_color_distortions = true; + }else{ + std::cerr << "Unknown or incomplete argument: " << arg << "\n" + << "Usage: SerialProgramsTests [--filter ]... [--list] [--augment-color-distortions]" + << std::endl; + return std::nullopt; + } + } + return arguments; +} + +// The full test logs go to "SerialProgramsTests.log" in this executable's own user folder, +// not in the folder of SerialPrograms. +const std::string& test_log_path(){ + static const std::string path = + USER_FILE_PATH() + Filesystem::application_binary_name().stem().string() + ".log"; + return path; +} +FileLogger& test_file_logger(){ + static FileLogger logger( + GlobalThreadPools::unlimited_normal(), + FileLoggerConfig{ + .file_path = test_log_path(), + } + ); + return logger; +} + +int run_tests(const TestRunnerArguments& arguments){ + std::vector names = ComputerPrograms::unit_test_names(arguments.filters); + if (arguments.list){ + for (const std::string& name : names){ + std::cout << name << "\n"; + } + std::cout << names.size() << " tests" << std::endl; + return names.empty() ? EXIT_BAD_ARGUMENTS : 0; + } + if (names.empty()){ + std::cerr << "No test matches the filters." << std::endl; + return EXIT_BAD_ARGUMENTS; + } + + Logger& logger = global_logger_command_line(); + logger.log("Resources folder: " + RESOURCE_PATH()); + logger.log("Unit test resources folder: " + UNIT_TEST_RESOURCE_PATH()); + logger.log("Log file: " + test_log_path()); + + size_t failed = run_unit_tests(arguments.augment_color_distortions, arguments.filters); + return (int)std::min(failed, MAX_FAILURE_EXIT_CODE); +} + +// Same order as the end of main() in CommonFramework/Main.cpp, for the services the tests use. +void stop_global_services(){ + OCR::clear_ocr_cache(); + global_periodic_runner().stop(); + global_watchdog().stop(); + test_file_logger().stop(); +} +void stop_thread_pools(){ + GlobalThreadPools::computation_realtime().stop(); + GlobalThreadPools::computation_normal().stop(); + GlobalThreadPools::unlimited_realtime().stop(); + GlobalThreadPools::unlimited_normal().stop(); + global_dispatcher.stop(); + + GlobalThreadPools::qt_worker_threadpool().stop(); + GlobalThreadPools::qt_event_threadpool().stop(); +} + +// The body of main(). `argc` is a reference because QCoreApplication keeps one. +int test_runner_main(int& argc, char* argv[]){ + std::optional arguments = parse_arguments(argc, argv); + if (!arguments){ + return EXIT_BAD_ARGUMENTS; + } + + // The tests use Qt (images, thread pools, events) but no window. + QCoreApplication application(argc, argv); + + { + MultiOutputLogger& logger = global_multi_logger(); + logger.add_listener(global_last_log_history()); + logger.add_listener(test_file_logger()); + } + + // Construct the Qt thread pools on the main thread, as SerialPrograms does. + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + + int ret = run_tests(*arguments); + + stop_global_services(); +#ifdef PA_ENABLE_QT_ADOPTION_WORKAROUND + // See the Qt 6.9 workaround at the end of main() in CommonFramework/Main.cpp. + std::cout.flush(); + std::quick_exit(ret); +#endif + stop_thread_pools(); + return ret; +} + +} // namespace + + +} // namespace PokemonAutomation + + +int main(int argc, char* argv[]){ + return PokemonAutomation::test_runner_main(argc, argv); +} diff --git a/SerialPrograms/cmake/LabBuildReceipt.cmake b/SerialPrograms/cmake/LabBuildReceipt.cmake new file mode 100644 index 0000000000..c2817753cf --- /dev/null +++ b/SerialPrograms/cmake/LabBuildReceipt.cmake @@ -0,0 +1,21 @@ +# Called after linking a lab executable; never invokes that executable. +set(receipt "${BINARY}.lab-build.json") +if(PROFILE STREQUAL "") + file(REMOVE "${receipt}") + return() +endif() +if(NOT PROFILE MATCHES "^[A-Za-z0-9_][A-Za-z0-9_-]*$") + message(FATAL_ERROR "Invalid lab profile") +endif() +find_package(Git REQUIRED) +execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${SOURCE}" OUTPUT_VARIABLE revision + OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) +execute_process(COMMAND "${GIT_EXECUTABLE}" status --porcelain --untracked-files=normal + WORKING_DIRECTORY "${SOURCE}" OUTPUT_VARIABLE changes COMMAND_ERROR_IS_FATAL ANY) +set(clean false) +if(changes STREQUAL "") + set(clean true) +endif() +file(SHA256 "${BINARY}" digest) +file(WRITE "${receipt}" "{\"schema\":1,\"profile\":\"${PROFILE}\",\"source_revision\":\"${revision}\",\"source_clean\":${clean},\"binary_sha256\":\"${digest}\"}\n") diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index c894056e3d..2174f04b2c 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -756,6 +756,20 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/Startup/NewVersionCheck.h Source/CommonFramework/Startup/SetupSettings.cpp Source/CommonFramework/Startup/SetupSettings.h + Source/CommonFramework/Startup/StartupArguments.cpp + Source/CommonFramework/Startup/StartupArguments.h + Source/CommonFramework/Devices/LabDeviceAccessQt.cpp + Source/CommonFramework/Devices/LabDeviceAccessQt.h + Source/CommonFramework/Devices/LabDeviceRuntime.cpp + Source/CommonFramework/Devices/LabDeviceRuntime.h + Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp + Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h + Source/CommonFramework/Startup/LabStartupPolicy.cpp + Source/CommonFramework/Startup/LabStartupPolicy.h + Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp + Source/CommonFramework/Startup/LabStartupPolicy_Tests.h + Source/CommonFramework/Startup/StartupArguments_Tests.cpp + Source/CommonFramework/Startup/StartupArguments_Tests.h Source/CommonFramework/Tools/DebugDumper.cpp Source/CommonFramework/Tools/DebugDumper.h Source/CommonFramework/Tools/ErrorDumper.cpp @@ -836,6 +850,10 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/Windows/ButtonDiagram.h Source/CommonFramework/Windows/DpiScaler.cpp Source/CommonFramework/Windows/DpiScaler.h + Source/CommonFramework/Headless/HeadlessHost.cpp + Source/CommonFramework/Headless/HeadlessHost.h + Source/CommonFramework/Headless/HeadlessHost_Tests.cpp + Source/CommonFramework/Headless/HeadlessHost_Tests.h Source/CommonFramework/Windows/LabChannel.cpp Source/CommonFramework/Windows/LabChannel.h Source/CommonFramework/Windows/LabChannel_Tests.cpp @@ -1515,6 +1533,10 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_TypewriterTextMapping.cpp Source/NintendoSwitch/Programs/Typewriter/NintendoSwitch_TypewriterTextMapping.h + Source/PanelCatalog.cpp + Source/PanelCatalog.h + Source/PanelCatalog_Tests.cpp + Source/PanelCatalog_Tests.h Source/PanelLists.cpp Source/PanelLists.h Source/Pokemon/Inference/Pokemon_AbilityReader.cpp diff --git a/devtools/lab/DEVICE_OWNERSHIP.md b/devtools/lab/DEVICE_OWNERSHIP.md new file mode 100644 index 0000000000..821a7e06fc --- /dev/null +++ b/devtools/lab/DEVICE_OWNERSHIP.md @@ -0,0 +1,127 @@ +# Lab device ownership + +This is the second preparation step after #236. macOS GUI lab builds reserve +the selected profile before Qt/settings startup and check an explicit allowlist +before constructing a native serial connection, QCamera or audio input. +Without a matching policy, hardware opens are refused. An empty policy permits +the GUI for setup, but permits no devices. Native tests do not reserve live +profiles or enumerate hardware. + +## Configure the allowlist + +Create `lab-devices.json` at the profile root, alongside `UserSettings/`, using +reviewed identities for that console. Example with deliberately fictitious IDs: + +```json +{ + "schema": 1, + "profile": "lab-B", + "serial": ["10c4:ea60:EXAMPLE-USB-SERIAL-B"], + "video": ["4558414d504c452d564944454f2d42"], + "audio": ["4558414d504c452d415544494f2d42"] +} +``` + +All five fields are required; unknown fields, duplicates within a device list, +invalid JSON, wrong types, empty identities, unsupported schema and mismatched +profile names are refused. Empty lists are valid. A successfully loaded policy +is immutable for that process: restart the app after changing it. No default +profile, display alias, first-device fallback or wildcard is used. + +Serial keys are lowercase four-digit USB vendor/product IDs followed by the exact +USB serial number: `VID:PID:SERIAL`. The runtime resolves a fresh USB enumeration +immediately before opening, rather than trusting the UI's cached port record. +`/dev/tty.*` and `/dev/cu.*` aliases share a key. Two distinct current ports +reporting the same USB identity are rejected, as are adapters lacking a serial. +If a port path is reused by a different adapter, its new identity must independently +be allowlisted. VID/PID/serial cannot distinguish hardware that impersonates an +identity; this protects against accidental assignment errors, not hostile USB devices. + +Video/audio entries are the lowercase hexadecimal encoding of the exact byte IDs +reported by `QCameraDevice::id()` / `QAudioDevice::id()`. The settings UI stores +the corresponding original device ID; it does not use the hex string as a device +selector. Do not use capture descriptions, friendly names or list positions. +Audio output to speakers is outside this input-device policy. + +A refusal names the kind and ID of the device the panel selected, e.g. +`Audio input AppleUSBAudioEngine:...:1112100:3 (allowlist ID 4170...) is not +allowlisted for profile lab-B`. That usually means the panel's settings pick the +other console's input, not that the allowlist is wrong: both capture cards are +listed as "USB Digital Audio", and only the ID tells them apart. Select the +other entry in that panel's audio dropdown. + +This PR does not generate a policy from the currently selected panel: that could +approve the very wrong-console binding we are preventing. Identity collection +and confirming A/B assignments remain a deliberate setup step. Existing profiles, +local deployment scripts, certificates and permissions are not modified. + +## Ownership and lifetime + +All guarded apps for a user use the same local directory: + +``` +~/Library/Application Support/PokemonAutomation/LabDeviceLeases/ +``` + +Lock keys hash device kind plus stable identity, independently of profile, +executable name and bundle ID. Different consoles can run concurrently; +two profiles pointing at the same input contend for one lease. A separate +profile lease prevents duplicate GUI instances from loading/writing the same +profile. Video and audio are separate input endpoints so a capture card's +camera and microphone can be used together. + +The implementation uses Qt's existing `QLockFile` backend, with nonblocking +acquisition and age-based expiration disabled. A live owner is never evicted +because a run has lasted a long time. Qt detects a dead process and recovers its +stale lock; the application never forcibly deletes another owner's lock. +Do not put the lease directory on a network filesystem or remove locks manually. +See [Qt's locking contract](https://doc.qt.io/qt-6/qlockfile.html) for platform +and cooperative-locking limits. + +The serial lease survives connection setup, command traffic and teardown until +the physical port closes. Failed setup closes resources before releasing it. +The camera lease survives Qt's synchronous destruction of the camera on its event +thread. The audio lease survives stopping and destroying its native input. +Reconfiguration closes the previous input before opening its replacement. +Capture refusals are logged and return no active input without propagating an +exception through a Qt callback. Serial refusals appear as a failed connection. + +## Scope + +Enforcement is for macOS GUI lab builds in this stack. GUI-free CoreLib and +Windows/Linux application device behavior remain outside this enforcement; +the lease engine itself has portable native tests. The shared audio replacement +ordering also applies on other GUI platforms. + +Every app touching the test devices must use the guarded build. An old/default +binary that does not participate in this locking protocol cannot be excluded by +an advisory lease. Keep it disconnected from lab devices. This is not a system +sandbox, and neither file hashes nor the allowlist replace physical-console +assignment review. #236's package metadata still says +`hardware_ownership_enforced: false`; that older preparation marker is not a +runtime capability check or evidence that a deployed binary contains this PR. +Use the deployed source revision and this stack's CI evidence. + +## Native tests + +The focused target compiles production policy/lease code, without the GUI, +controller drivers, real settings or capture hardware: + +```sh +cmake -S SerialPrograms/Source/CommonFramework/Devices -B /path/to/test-build +cmake --build /path/to/test-build -j4 +ctest --test-dir /path/to/test-build --output-on-failure +``` + +The same cases are registered in the application's native test database. The +small runner is also the subprocess used by cross-process cases and is built +beside the app executable. Tests cover policy rejection before scripted opens, +USB identity/alias resolution, duplicate USB identities, same/different devices, +duplicate profiles, release/reconnect, exception unwinding, live-process exclusion +and recovery after killing an owned test child. The aged-live-lock timestamp +fixture runs on POSIX; Windows disallows editing that live lock file. + +These tests validate real process-lock behavior and scripted opens. They do not +validate drivers' physical close completion, USB replug behavior, macOS capture +identity persistence or real permission dialogs. Live validation still needs a +separately authorized, carefully assigned two-console run. diff --git a/devtools/lab/INSTANCE_PREPARATION.md b/devtools/lab/INSTANCE_PREPARATION.md new file mode 100644 index 0000000000..88a7d38161 --- /dev/null +++ b/devtools/lab/INSTANCE_PREPARATION.md @@ -0,0 +1,113 @@ +# Preparing two lab instances + +Use a dedicated profile and app identity for each console. Default remains the +normal-use profile. This tooling prepares files offline; it never launches an +app, sends a command, changes permissions, or opens a controller or capture device. + +## Seed profiles without copying live state + +Work from an offline snapshot containing `UserSettings/SerialPrograms-Settings.json`, taken while the +source app is closed. Keep that snapshot as the backup; do not seed from a file +being rewritten by a running app. For example, using a scratch root first: + +```sh +python3 devtools/lab/lab_instance.py seed \ + --source /offline/default-snapshot --root /offline/application-support --profile lab-A +python3 devtools/lab/lab_instance.py seed \ + --source /offline/default-snapshot --root /offline/application-support --profile lab-B +``` + +Each destination is `ROOT/Profiles/lab-NAME`. Existing destinations, including +empty directories, are refused. The source is never written. The new directory +is private to its owner. Only theme, window geometry and console-settings collapse +preferences are retained; panels are empty. Saved controller/camera/audio bindings, +device aliases, program options, selected panels, developer credentials, Discord +settings, test settings, statistics/temp paths, logs and channel queues are omitted. +Output-window startup and Rich Presence are disabled. Unknown future settings are +omitted. This is preference seeding, not a full profile clone. Application defaults +still apply to omitted settings; this does not promise a network-free GUI launch. + +After reviewing the scratch output, the same command can seed under +`~/Library/Application Support/SerialPrograms`. Do not rename the executable: +its basename participates in the application's settings path. Do not configure +real devices until the assigned console and device identities have been checked. +`lab-profile.json` deliberately records `hardware_configured: false`; it is a +preparation note, not a runtime device lease. + +## Bind and stage each build + +The lab integration branch merges `lab/main` and the feature under test (MaxLair, +for example). The upstream feature branch never merges lab tooling. Build each +slot with its matching profile, using the existing build/signing setup: + +```sh +cmake -S SerialPrograms -B /build/slot-A -DPA_LAB_PROFILE=lab-A +# Build via the existing lab procedure; use a clean committed source tree. +``` + +macOS linking writes `SerialPrograms.lab-build.json` beside the executable. It +records the compiled profile, executable SHA-256, source revision and whether the +source checkout was clean at link time. Unbound linking removes an old receipt. +This is accidental-mixup detection, not a signed source/build attestation; it +records the checkout observed at link time, and cannot prove when every object +was compiled. Use a clean committed checkout and the normal dependency-tracked +build. Do not edit or regenerate receipts manually to accept a changed binary. + +```sh +python3 devtools/lab/lab_instance.py preflight \ + --app /build/slot-A/SerialPrograms.app --profile lab-A +python3 devtools/lab/lab_instance.py stage \ + --app /build/slot-A/SerialPrograms.app --out /staging/slot-A \ + --profile lab-A --console A --bundle-id com.dayfine.pokemonautomation.labA \ + --signing-identity 'PokemonAutomation Local Development' +``` + +Stage requires a matching linked-build receipt and unused output directory. It +copies the app, sets a distinct lab identifier/display name, signs and verifies +it, then writes `lab-deployment.json` outside the bundle with final executable +and plist hashes. The source app is unchanged. Failure removes only the new +staging directory. No active deployment is replaced and no permissions are reset. +Keep the manifest next to the staged app. The build receipt inside the signed app +describes the pre-packaging artifact; use `verify` for the final signed package. + +Prepare B with `lab-B`, console `B`, a different bundle ID and +`--peer /staging/slot-A/lab-deployment.json`. Peer manifests reject reuse of a +profile, console or bundle ID. Before deployment, cross-check both packages: + +```sh +python3 devtools/lab/lab_instance.py verify \ + --manifest /staging/slot-A/lab-deployment.json --profile lab-A \ + --peer /staging/slot-B/lab-deployment.json +python3 devtools/lab/lab_instance.py verify \ + --manifest /staging/slot-B/lab-deployment.json --profile lab-B \ + --peer /staging/slot-A/lab-deployment.json +``` + +Use this staging entry point instead of editing/signing bundles with the older +local deploy scripts. Those external scripts have not been modified automatically +and can bypass this check. No production/default profile is accepted by this +tool: preparation requires a `lab-` prefix. The existing startup policy additionally +rejects a profile conflicting with the compiled binding, before loading settings. + +## Remaining device boundary + +Two profiles do not reserve two physical devices. A misconfigured profile can +still open another console's serial port, which can disturb that console before +commands are sent. Shell locks and channel ownership do not prevent that. +Before concurrent hardware validation, add an explicit per-slot device allowlist +and exclusive lease checked before serial/capture open, with native tests for +wrong-device access and competing owners. These packages explicitly report +`hardware_ownership_enforced: false`. They are not authorization to start a run. + +## Offline verification + +```sh +python3 -m unittest discover -s devtools/lab -p test_lab_instance.py -v +``` + +Tests use temporary settings, fake executable files and an injected signer. They +exercise the real receipt CMake script without configuring the full application. +They cover sanitization, existing destinations, default/path names, symlink escape, +missing/mismatched/stale/dirty receipts, slot collisions, post-signing hashes, +modified packages and signing failure. They do not validate real certificates, +macOS permission persistence or physical-device exclusion. CI runs on macOS/Linux. diff --git a/devtools/lab/README.md b/devtools/lab/README.md index 3cf6280c0c..4751d3b32c 100644 --- a/devtools/lab/README.md +++ b/devtools/lab/README.md @@ -38,9 +38,10 @@ rules are cheap; use both. `PROFILE` is the app's user-files folder: the one with `SerialPrograms.log`, `Screenshots/` and the settings. On macOS it is -`~/Library/Application Support/SerialPrograms`, or -`.../SerialPrograms/Profiles/NAME` for an app launched with -`--args --profile NAME`. On Windows and Linux it is the folder the app was +`~/Library/Application Support/SerialPrograms/Profiles/NAME` for a lab app +launched with `--args --profile NAME` or built with `PA_LAB_PROFILE=NAME`. +Lab builds refuse an unprofiled macOS GUI launch; they never fall back to the +live app's default settings directory. On Windows and Linux it is the folder the app was started in. `labctl.sh supports BUILD` says whether a build has the channel at all. @@ -71,8 +72,73 @@ touched. On macOS: and capture device (the `ConsoleSetup` of each program in the settings file), so two lab apps drive two consoles at once. -On Windows and Linux the separate copy is just a second folder, and the -profile flag is the same. +On Windows and Linux the separate copy is a second folder; `--profile` is +unsupported and rejected. + +## Startup isolation and tests + +For safe preference seeding and checked packaging of two lab apps, see +[INSTANCE_PREPARATION.md](INSTANCE_PREPARATION.md). Its staging tool validates the +linked build's profile receipt before signing and records the final package hashes. +Physical-device ownership remains a separate required boundary. + +Bind each deployed macOS lab binary to its console when configuring its existing +build directory, for example: + +```sh +cmake -S SerialPrograms -B /path/to/build -DPA_LAB_PROFILE=lab-B +``` + +This compiles the identity into the executable. Finder and macOS permission-dialog +relaunches need no arguments and still select `Profiles/lab-B`. An explicit +`--profile lab-A` against that binary exits with status 2 before Qt, settings, +logging or device discovery. Changing the bundle's name or identifier does not +change its compiled profile; build each console's binary with the correct value. +The existing signing/bundle-identity procedure still applies. + +With `PA_LAB_PROFILE` empty (the build default), macOS GUI startup requires +`--profile NAME`; a no-argument relaunch safely exits. Names are ASCII letters, +digits, `_` and `-`, cannot start with `-`, and must be a single path component. +Empty, missing, duplicate and malformed names are rejected rather than rewritten. + +On this baseline, run the registered native tests with: + +```sh +./SerialPrograms.app/Contents/MacOS/SerialPrograms --command-line-test-mode +# Optional image augmentation: +./SerialPrograms.app/Contents/MacOS/SerialPrograms --command-line-test-mode --augment-color-distortions +``` + +Explicit tests use `QCoreApplication`, bypass settings loading and window +restoration, and never enter startup's serial/camera discovery. macOS test file +output uses `Profiles/unit-tests`; no console profile can be supplied with tests. +The legacy persisted test-mode setting remains supported, but only after a valid +GUI profile is selected. It also runs before device discovery. Prefer the explicit +flag so no saved settings are loaded. + +Unknown or incomplete arguments fail closed with status 2. `--test bdsp` and `--test-filter=...` +are not supported by this baseline. Documented Qt application options (including `-platform offscreen`, `-style Fusion`, +`-style=Fusion`, stylesheet, window and plugin options) are preserved in order and +passed to Qt. Missing values and unknown option spellings are rejected; values in +option/value pairs must not begin with `-`. Qt environment variables such as +`QT_QPA_PLATFORM` remain supported. +The separate `SerialProgramsTests` runner is an independent pending feature; +this lab change does not import it. + +The colocated `StartupArguments_Tests.cpp` table covers the general grammar, +ordinary unprofiled GUI startup, and Qt argument preservation. The separate +`LabStartupPolicy_Tests.cpp` table covers rejection, profile binding, no-argument +lab relaunch and exclusive dispatch to the test entry point. Its injected +GUI callback represents the boundary that owns settings/window/hardware startup. +These tests verify that boundary is not entered; they do not exercise real devices +or claim that all unit-test bodies are incapable of opening hardware. + +Profiles isolate settings, not physical serial devices. A mistakenly configured +profile can still refer to another console's port. Enforcing ownership before a +physical port opens requires a separate device-lease change; the channel owner +and shell lock do not enforce that boundary. + +See [STARTUP_ISOLATION.md](STARTUP_ISOLATION.md) for the review split and upstream path. ## Worked example: run a program with nothing clicked diff --git a/devtools/lab/STARTUP_ISOLATION.md b/devtools/lab/STARTUP_ISOLATION.md new file mode 100644 index 0000000000..a8e986d3d9 --- /dev/null +++ b/devtools/lab/STARTUP_ISOLATION.md @@ -0,0 +1,87 @@ +# Startup isolation: review and upstream path + +A launch intended only to run tests must not restore a saved controller panel. +A lab relaunch must not silently select the live app's default settings. + +The PR #226 lesson at revision `01e6659116f520b51c35ee956aa6a222dfa864f8` +records a `--test bdsp` invocation and profile-free relaunch concerns. That command +is not supported: previously, unrecognized arguments could fall through to GUI +startup. This is a startup problem, independent of AutoStory game logic. + +## Review separately + +1. [PR #234](https://github.com/dayfine/Arduino-Source/pull/234), based on fork + `main`, only extracts service shutdown and worker shutdown from `main()`. + The statements and their order are unchanged, including the Windows Qt + quick-exit branch. It has no lab or gameplay parents and is independently + suitable for an upstream handoff after validation. +2. [PR #233](https://github.com/dayfine/Arduino-Source/pull/233), targeting + `lab/main`, contains the startup behavior change and merges #234's exact + commit without rewriting the previously published history. Its base-relative + diff will still include the mechanical helper extraction until #234 reaches + `lab/main`; review that extraction in #234 rather than as new behavior here. + +To see only behavior changes to Main.cpp after the mechanical prerequisite: + +```sh +git diff origin/startup/shutdown-refactor..origin/lab/startup-isolation -- SerialPrograms/Source/CommonFramework/Main.cpp +``` + +## General mechanism and lab policy + +| Component | Contract | Upstream treatment | +|---|---|---| +| `StartupArguments.h/.cpp` | Pure parser and exclusive GUI/test dispatch; an ordinary no-argument launch remains GUI; documented Qt options are preserved. | Reusable candidate, with its own argument tests. | +| `LabStartupPolicy.h/.cpp` | Require an explicit macOS lab profile or select the compiled identity; reject a conflict. | Keep on lab branches. Never impose this default on normal releases. | +| `Main.cpp` wiring | Parse, apply lab policy, then enter exactly one startup path. Explicit tests skip saved settings, GUI restoration and startup device discovery. | Upstream wiring must omit the lab adapter. | +| `Filesystem::set_startup_profile` | Store an already-validated identity before paths are cached. | Review separately with existing upstream profile behavior and platform support. | +| `PA_LAB_PROFILE` CMake option | Bind the lab binary to one profile even when the OS relaunches it without arguments. | Keep lab-only. | + +The general parser and lab policy have separate native test tables. The lab +regressions also inject entry-point callbacks to verify rejected launches enter +neither path and explicit tests never enter the GUI boundary. macOS CI checks +actual invalid executable invocations. These tests do not exercise hardware or +prove every registered test body is device-free. + +Examples: + +- `SerialPrograms --test bdsp`: reject with status 2 before Qt/settings/devices. +- Ordinary upstream-style parsing with no arguments: select GUI. +- Unbound macOS lab binary with no arguments: lab policy rejects it. +- Binary built with `PA_LAB_PROFILE=lab-B`, relaunched without arguments: GUI + selects `Profiles/lab-B`. +- The same binary with `--profile lab-A`: reject before saved settings load. +- `--command-line-test-mode`: enter the isolated test path, even for a bound lab + build; no console profile may be supplied with the tests. + +## Reuse the pending headless runner upstream + +[PokemonAutomation/Arduino-Source#1845](https://github.com/PokemonAutomation/Arduino-Source/pull/1845) +already provides `SerialProgramsTests` on fork branch `upstream/headless-tests` +(inspected at `006d547f9f5baa2d1e064755b6e45817a6b20aa4`). Its executable owns +QCoreApplication, test selection, logging and shutdown. Do not submit #233's +interim `run_test_application()` as another upstream implementation of that +lifecycle. + +The proposed upstream sequence is: + +1. Review/land the existing headless-runner work and independently review #234's + mechanical extraction. Neither depends on lab policy. +2. Prepare a clean fork draft for general argument validation on those upstream + prerequisites. Preserve no-argument GUI behavior and documented Qt arguments; + run parser regressions on all platforms. Resolve compatibility for the legacy + GUI test-mode flag using the existing runner rather than copying its lifecycle. +3. Keep the lab adapter, compiled profile and lab deployment instructions in + `lab/main`. Consume the accepted general mechanism by merging main as usual. + +No pending headless-runner changes are imported into #233, and this document is +not authorization to publish or modify an upstream PR. The clean upstream +argument-validation follow-up has not been opened yet. + +## Remaining boundary + +Settings isolation cannot stop two different profiles from naming the same +physical serial port. A future device lease must be enforced before a port is +opened, including the first connection/reset. Shell and channel ownership alone +do not protect that boundary. No deployment, permission reset or console run is +part of these PRs. diff --git a/devtools/lab/lab_instance.py b/devtools/lab/lab_instance.py new file mode 100644 index 0000000000..99e232d049 --- /dev/null +++ b/devtools/lab/lab_instance.py @@ -0,0 +1,54 @@ +"""Prepare isolated macOS lab instances; never launch apps or open devices. + +Examples (run from repository root): + python3 devtools/lab/lab_instance.py seed --source /offline/default-copy --root /offline/root --profile lab-A + python3 devtools/lab/lab_instance.py preflight --app /build/SerialPrograms.app --profile lab-A + python3 devtools/lab/lab_instance.py stage --app /build/SerialPrograms.app --out /staging/A --profile lab-A --console A --bundle-id com.dayfine.pokemonautomation.labA --signing-identity 'PokemonAutomation Local Development' + python3 devtools/lab/lab_instance.py verify --manifest /staging/A/lab-deployment.json --profile lab-A --peer /staging/B/lab-deployment.json +""" + +import argparse +import subprocess + +from lab_package import preflight, stage, verify +from lab_profiles import seed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + seeder = commands.add_parser("seed", help="Create an unused sanitized profile") + seeder.add_argument("--source", required=True) + seeder.add_argument("--root", required=True, help="Application support root containing Profiles/") + for command in (seeder, commands.add_parser("preflight"), commands.add_parser("stage"), commands.add_parser("verify")): + command.add_argument("--profile", required=True) + if command.prog.endswith(("preflight", "stage")): + command.add_argument("--app", required=True) + if command.prog.endswith(("stage", "verify")): + command.add_argument("--peer", action="append", default=[]) + if command.prog.endswith("stage"): + command.add_argument("--out", required=True) + command.add_argument("--console", choices=("A", "B"), required=True) + command.add_argument("--bundle-id", required=True) + command.add_argument("--signing-identity", required=True) + if command.prog.endswith("verify"): + command.add_argument("--manifest", required=True) + args = parser.parse_args() + try: + if args.command == "seed": + print(seed(args.source, args.root, args.profile)) + elif args.command == "preflight": + preflight(args.app, args.profile) + print("Linked build matches " + args.profile) + elif args.command == "stage": + print(stage(args.app, args.out, args.profile, args.console, args.bundle_id, args.signing_identity, args.peer)) + else: + verify(args.manifest, args.profile, args.peer) + print("Staged package matches " + args.profile + "; hardware ownership is not enforced yet.") + except (OSError, ValueError, KeyError, TypeError, subprocess.CalledProcessError) as error: + detail = str(error) if isinstance(error, ValueError) else type(error).__name__ + parser.exit(2, "Lab preparation refused: " + detail + "\n") + + +if __name__ == "__main__": + main() diff --git a/devtools/lab/lab_package.py b/devtools/lab/lab_package.py new file mode 100644 index 0000000000..8ef6cc04a0 --- /dev/null +++ b/devtools/lab/lab_package.py @@ -0,0 +1,126 @@ +"""Stage and validate macOS lab apps without launching them or opening devices.""" + +import hashlib +from pathlib import Path +import plistlib +import re +import shutil +import subprocess + +from lab_profiles import lab_profile, read_json, write_json + + +def sha256(path): + with Path(path).open("rb") as stream: + digest = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def app_files(app): + """Require the executable basename used by the existing macOS profiles.""" + app = Path(app) + plist = app / "Contents" / "Info.plist" + with plist.open("rb") as stream: + info = plistlib.load(stream) + if info.get("CFBundleExecutable") != "SerialPrograms": + raise ValueError("Expected executable SerialPrograms; renaming it changes settings paths.") + binary = app / "Contents" / "MacOS" / "SerialPrograms" + if binary.is_symlink() or plist.is_symlink() or not binary.is_file(): + raise ValueError("Executable and plist must be regular files inside the app.") + if binary.resolve().parent != app.resolve() / "Contents" / "MacOS": + raise ValueError("Executable directory must not redirect outside the app.") + return plist, binary, info + + +def preflight(app, profile): + """Check the original linked artifact before any packaging/signature modifications.""" + profile = lab_profile(profile) + _, binary, _ = app_files(app) + receipt = read_json(str(binary) + ".lab-build.json") + if receipt.get("schema") != 1 or receipt.get("profile") != profile: + raise ValueError("Missing or mismatched compiled lab profile receipt.") + if receipt.get("source_clean") is not True: + raise ValueError("Build receipt requires a clean committed source tree.") + if not re.fullmatch(r"[0-9a-f]{40}", receipt.get("source_revision", "")): + raise ValueError("Build receipt requires a full source revision.") + if sha256(binary) != receipt.get("binary_sha256"): + raise ValueError("Binary differs from the build receipt; rebuild before staging.") + return receipt + + +def run_codesign(arguments): + subprocess.run(["/usr/bin/codesign", *arguments], check=True, capture_output=True) + + +def distinct_slots(candidate, peers): + """Reject collisions with the deployment manifests supplied by the caller.""" + for peer in peers: + for field in ("profile", "console", "bundle_id"): + if candidate[field] == read_json(peer)[field]: + raise ValueError("Lab deployment conflicts with peer: " + field) + + +def stage(app, output, profile, console, bundle_id, signing_identity, peers=(), signer=run_codesign): + """Create an unused staging directory and signed app; never replace an active app.""" + receipt = preflight(app, profile) + if not re.fullmatch(r"com\.dayfine\.pokemonautomation\.lab[A-Za-z0-9_-]*", bundle_id): + raise ValueError("Use a dedicated com.dayfine.pokemonautomation.lab... bundle ID.") + if console not in ("A", "B") or not signing_identity.strip() or signing_identity == "-": + raise ValueError("Select console A/B and a persistent signing identity, not ad-hoc '-'.") + manifest = { + "schema": 1, "profile": profile, "console": console, "bundle_id": bundle_id, + "source_revision": receipt["source_revision"], "app": "SerialPrograms.app", + "hardware_ownership_enforced": False, + } + distinct_slots(manifest, peers) + output = Path(output) + if output.resolve() == Path(app).resolve() or Path(app).resolve() in output.resolve().parents: + raise ValueError("Staging directory must be separate from the source app.") + output.mkdir(mode=0o700) # Exclusive; an existing deployment is never overwritten. + try: + staged = output / manifest["app"] + shutil.copytree(app, staged, symlinks=True) + plist, binary, info = app_files(staged) + info.update({ + "CFBundleIdentifier": bundle_id, + "CFBundleName": "PokemonAutomation Lab " + console, + "CFBundleDisplayName": "PokemonAutomation Lab " + console + " (" + profile + ")", + }) + with plist.open("wb") as stream: + plistlib.dump(info, stream) + signer(["--force", "--deep", "--sign", signing_identity, str(staged)]) + signer(["--verify", "--deep", "--strict", str(staged)]) + # Signing can change Mach-O bytes: bind the final artifact separately. + manifest["binary_sha256"] = sha256(binary) + manifest["plist_sha256"] = sha256(plist) + write_json(output / "lab-deployment.json", manifest) + except BaseException: + shutil.rmtree(output) + raise + return output / "lab-deployment.json" + + +def verify(manifest_path, profile, peers=(), signer=run_codesign): + """Validate a staged package's identity, hashes and signature without launching it.""" + manifest_path = Path(manifest_path) + manifest = read_json(manifest_path) + if manifest.get("schema") != 1 or manifest.get("profile") != lab_profile(profile): + raise ValueError("Deployment profile does not match the requested lab.") + if manifest.get("app") != "SerialPrograms.app": + raise ValueError("Unexpected app path in deployment manifest.") + app = manifest_path.parent / manifest["app"] + plist, binary, info = app_files(app) + receipt = read_json(str(binary) + ".lab-build.json") + if (receipt.get("schema") != 1 or receipt.get("profile") != manifest["profile"] + or receipt.get("source_revision") != manifest.get("source_revision") + or receipt.get("source_clean") is not True): + raise ValueError("Deployment identity differs from the packaged build receipt.") + if info.get("CFBundleIdentifier") != manifest["bundle_id"]: + raise ValueError("Bundle identifier differs from the deployment manifest.") + if sha256(binary) != manifest["binary_sha256"] or sha256(plist) != manifest["plist_sha256"]: + raise ValueError("Staged app changed after packaging.") + distinct_slots(manifest, peers) + signer(["--verify", "--deep", "--strict", str(app)]) + return manifest diff --git a/devtools/lab/lab_profiles.py b/devtools/lab/lab_profiles.py new file mode 100644 index 0000000000..6593d9a91a --- /dev/null +++ b/devtools/lab/lab_profiles.py @@ -0,0 +1,70 @@ +"""Seed a new lab profile from display preferences, never a full settings clone.""" + +import json +import os +from pathlib import Path +import re +import shutil +import tempfile + + +SETTINGS = "UserSettings/SerialPrograms-Settings.json" +PREFERENCES = {"THEME", "WINDOW_SIZE", "LOG_WINDOW_SIZE", "CONSOLE_SETTINGS_COLLAPSED"} + + +def lab_profile(name): + """Require an explicit lab name, distinct from default and unit-test profiles.""" + if not re.fullmatch(r"lab-[A-Za-z0-9_][A-Za-z0-9_-]*", name): + raise ValueError("Profile must be named lab-NAME (for example lab-A).") + return name + + +def read_json(path): + with Path(path).open(encoding="utf-8") as stream: + value = json.load(stream) + if not isinstance(value, dict): + raise ValueError("Expected a JSON object.") + return value + + +def write_json(path, value): + with Path(path).open("x", encoding="utf-8") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + + +def seed(source, root, profile): + """Create an unused Profiles/lab-NAME; leave the source and existing profiles alone.""" + profile = lab_profile(profile) + source, root = Path(source).resolve(), Path(root).resolve() + destination = root / "Profiles" / profile + if destination.exists() or destination.is_symlink(): + raise ValueError("Destination profile already exists; refusing to overwrite it.") + original = read_json(source / SETTINGS) + if not isinstance(original, dict): + raise ValueError("Settings root must be an object.") + settings = original.get("20-GlobalSettings", {}) + if not isinstance(settings, dict): + raise ValueError("Global settings must be an object.") + # Allowlist rather than deleting known device keys: future settings default to omitted. + safe = {key: value for key, value in settings.items() if key in PREFERENCES} + safe.update({"LOG_WINDOW_STARTUP": False, "RICH_PRESENCE": False}) + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.parent.resolve() != root / "Profiles": + raise ValueError("Profiles must not redirect outside the requested root.") + temporary = Path(tempfile.mkdtemp(prefix=".seed-", dir=destination.parent)) + os.chmod(temporary, 0o700) + try: + (temporary / "UserSettings").mkdir() + write_json(temporary / SETTINGS, {"20-GlobalSettings": safe, "99-Panels": {}}) + write_json(temporary / "lab-profile.json", { + "schema": 1, "profile": profile, "hardware_configured": False, + "seed_policy": "display-preferences-only", + }) + # mkdir is exclusive, including against an empty destination created concurrently. + destination.mkdir(mode=0o700) + for child in temporary.iterdir(): + child.rename(destination / child.name) + finally: + shutil.rmtree(temporary) + return destination diff --git a/devtools/lab/test_lab_instance.py b/devtools/lab/test_lab_instance.py new file mode 100644 index 0000000000..177ece5a5a --- /dev/null +++ b/devtools/lab/test_lab_instance.py @@ -0,0 +1,218 @@ +"""Offline tool tests: temporary profiles, fake bundles and injected signing only.""" + +import json +from pathlib import Path +import plistlib +import shutil +import subprocess +import tempfile +import unittest + +from lab_package import preflight, sha256, stage, verify +from lab_profiles import SETTINGS, read_json, seed, write_json + + +class ProfileTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "default" + self.source.mkdir() + (self.source / "UserSettings").mkdir() + self.original = { + "20-GlobalSettings": { + "THEME": "dark", "CONSOLE_SETTINGS_COLLAPSED": False, + "DEVICE_ALIASES": {"serial": "console-A"}, "STATS_FILE": "/live/stats", + "COMMAND_LINE_TESTS": {"RUN": True}, "DEBUG": {"SelectedPanel": "MaxLair"}, + "DISCORD": {"secret": "do-not-copy"}, "FUTURE_SETTING": {"port": "live"}, + }, + "99-Panels": {"MaxLair": {"ConsoleSetup": {"Controller": "live-port"}}}, + "future": {"hardware": "do-not-copy"}, + } + write_json(self.source / SETTINGS, self.original) + + def test_two_profiles_copy_preferences_without_live_state(self): + before = (self.source / SETTINGS).read_bytes() + for name in ("lab-A", "lab-B"): + with self.subTest(profile=name): + destination = seed(self.source, self.root, name) + self.assertEqual(read_json(destination / SETTINGS), { + "20-GlobalSettings": { + "THEME": "dark", "CONSOLE_SETTINGS_COLLAPSED": False, + "LOG_WINDOW_STARTUP": False, "RICH_PRESENCE": False, + }, "99-Panels": {}, + }) + self.assertFalse(read_json(destination / "lab-profile.json")["hardware_configured"]) + self.assertEqual((self.source / SETTINGS).read_bytes(), before) + + def test_refuses_existing_profile_without_changing_it(self): + destination = seed(self.source, self.root, "lab-A") + original = (destination / SETTINGS).read_bytes() + with self.assertRaisesRegex(ValueError, "already exists"): + seed(self.source, self.root, "lab-A") + self.assertEqual((destination / SETTINGS).read_bytes(), original) + + def test_invalid_or_default_names_are_refused(self): + for name in ("", "default", "unit-tests", "../lab-A", "lab-", "lab-A/B", "lab-é"): + with self.subTest(profile=name), self.assertRaises(ValueError): + seed(self.source, self.root, name) + + def test_symlink_profile_directory_does_not_escape_root(self): + outside = self.root / "outside" + outside.mkdir() + (self.root / "Profiles").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "redirect"): + seed(self.source, self.root, "lab-A") + self.assertEqual(list(outside.iterdir()), []) + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.app = self.root / "original.app" + self.binary = self.app / "Contents/MacOS/SerialPrograms" + self.binary.parent.mkdir(parents=True) + self.binary.write_bytes(b"fake executable: must never be run") + self.plist = self.app / "Contents/Info.plist" + with self.plist.open("wb") as stream: + plistlib.dump({"CFBundleExecutable": "SerialPrograms", "CFBundleIdentifier": "original"}, stream) + self.receipt_path = Path(str(self.binary) + ".lab-build.json") + self.receipt = { + "schema": 1, "profile": "lab-A", "source_revision": "a" * 40, + "source_clean": True, "binary_sha256": sha256(self.binary), + } + write_json(self.receipt_path, self.receipt) + self.calls = [] + + def signing(self, arguments): + self.calls.append(arguments) + if "--sign" in arguments: + binary = Path(arguments[-1]) / "Contents/MacOS/SerialPrograms" + binary.write_bytes(binary.read_bytes() + b":signed") + + def package(self, output="slot-A", peers=(), signer=None): + return stage(self.app, self.root / output, "lab-A", "A", + "com.dayfine.pokemonautomation.labA", "test identity", peers, + signer or self.signing) + + def replace_receipt(self, receipt): + self.receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + + def test_round_trip_checks_post_signing_hash_and_preserves_original(self): + original = self.binary.read_bytes(), self.plist.read_bytes() + manifest = self.package() + deployed = verify(manifest, "lab-A", signer=self.signing) + self.assertEqual(deployed["source_revision"], "a" * 40) + self.assertFalse(deployed["hardware_ownership_enforced"]) + self.assertNotEqual(deployed["binary_sha256"], self.receipt["binary_sha256"]) + self.assertEqual((self.binary.read_bytes(), self.plist.read_bytes()), original) + self.assertEqual(len(self.calls), 3) + + def test_missing_receipt_is_refused_without_signing(self): + self.receipt_path.unlink() + with self.assertRaises(FileNotFoundError): + self.package() + self.assertEqual(self.calls, []) + self.assertFalse((self.root / "slot-A").exists()) + + def test_wrong_unbound_dirty_or_stale_receipt_is_refused(self): + cases = [ + {"profile": ""}, {"profile": "lab-B"}, {"source_clean": False}, + {"source_revision": "short"}, {"binary_sha256": "0" * 64}, {"schema": 2}, + ] + for changes in cases: + with self.subTest(changes=changes), self.assertRaises(ValueError): + self.replace_receipt(dict(self.receipt, **changes)) + self.package() + self.assertEqual(self.calls, []) + + def test_binary_changed_after_build_is_refused(self): + self.binary.write_bytes(b"a different build") + with self.assertRaisesRegex(ValueError, "differs"): + preflight(self.app, "lab-A") + + def test_peer_collision_on_each_identity_is_refused(self): + peer = {"profile": "lab-B", "console": "B", "bundle_id": "com.dayfine.pokemonautomation.labB"} + for field, value in (("profile", "lab-A"), ("console", "A"), ("bundle_id", "com.dayfine.pokemonautomation.labA")): + with self.subTest(field=field), self.assertRaisesRegex(ValueError, field): + path = self.root / (field + ".json") + write_json(path, dict(peer, **{field: value})) + self.package(peers=[path]) + self.assertEqual(self.calls, []) + + def test_existing_output_is_not_replaced(self): + path = self.root / "slot-A" + path.mkdir() + (path / "keep").write_text("untouched") + with self.assertRaises(FileExistsError): + self.package() + self.assertEqual((path / "keep").read_text(), "untouched") + + def test_sign_failure_removes_only_new_staging_directory(self): + def failure(arguments): + raise subprocess.CalledProcessError(1, "codesign") + with self.assertRaises(subprocess.CalledProcessError): + self.package(signer=failure) + self.assertFalse((self.root / "slot-A").exists()) + self.assertTrue(self.binary.is_file()) + + def test_package_change_or_wrong_profile_is_refused(self): + manifest = self.package() + with self.assertRaises(ValueError): + verify(manifest, "lab-B", signer=self.signing) + binary = manifest.parent / "SerialPrograms.app/Contents/MacOS/SerialPrograms" + binary.write_bytes(b"changed after deployment") + with self.assertRaisesRegex(ValueError, "changed"): + verify(manifest, "lab-A", signer=self.signing) + + def test_signature_failure_is_not_ignored(self): + manifest = self.package() + def failure(arguments): + raise subprocess.CalledProcessError(1, "codesign") + with self.assertRaises(subprocess.CalledProcessError): + verify(manifest, "lab-A", signer=failure) + + def test_relabeling_manifest_cannot_change_compiled_profile(self): + manifest = self.package() + data = read_json(manifest) + data["profile"] = "lab-B" + manifest.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "build receipt"): + verify(manifest, "lab-B", signer=self.signing) + + def test_cli_rejects_old_or_unbound_build_without_launching_it(self): + import sys + self.receipt_path.unlink() + tool = Path(__file__).with_name("lab_instance.py") + result = subprocess.run([sys.executable, str(tool), "preflight", "--app", str(self.app), + "--profile", "lab-A"], capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + self.assertIn("refused", result.stderr) + + @unittest.skipUnless(shutil.which("cmake") and shutil.which("git"), "CMake and Git required") + def test_actual_cmake_receipt_matches_binary_and_clears_unbound_receipt(self): + source = self.root / "source" + source.mkdir() + def git(*arguments): + return subprocess.run(["git", *arguments], cwd=source, check=True, capture_output=True) + git("init") + git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "--allow-empty", "-m", "fixture") + script = Path(__file__).resolve().parents[2] / "SerialPrograms/cmake/LabBuildReceipt.cmake" + def receipt(profile): + subprocess.run(["cmake", "-DBINARY=" + str(self.binary), "-DPROFILE=" + profile, + "-DSOURCE=" + str(source), "-P", str(script)], check=True, capture_output=True) + receipt("lab-A") + self.assertEqual(preflight(self.app, "lab-A")["source_revision"], git("rev-parse", "HEAD").stdout.decode().strip()) + (source / "untracked").write_text("dirty") + receipt("lab-A") + with self.assertRaisesRegex(ValueError, "clean"): + preflight(self.app, "lab-A") + receipt("") + self.assertFalse(self.receipt_path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/devtools/maps/README.md b/devtools/maps/README.md new file mode 100644 index 0000000000..a3c713c0dc --- /dev/null +++ b/devtools/maps/README.md @@ -0,0 +1,264 @@ +# Map tools for AutoStory routes + +Offline helpers that give a route planner its tile grid (standard library +only, Python 3.8+): a decoder for the Pokemon Platinum and Diamond +decompilations, an overlay that draws a grid and a run's routes over the map +image, and a colour parser for the few maps no decompilation covers. + +## Decomp first + +A map grid has two kinds of facts, from two sources: + +- **Terrain** (walls, tall grass, ledges, doors, warps) comes from the + decompilation: `decomp` below decodes it tile for tile. BDSP keeps + Diamond/Pearl's layout; at every anchor probed so far the decoded grid + matched the console. +- **People and the few BDSP differences** come from the console: trainers who + wander in BDSP but stand still in Platinum, a tile blocked by a prop BDSP + added, a sign a row off. These are hand edits, written down in the data + file's header with the run that found them. + +A route is coded only over tiles a human or a probe has walked. Bulbapedia's +DP map images are what humans read, so routes are still drawn and checked on +them, but they are no longer traced for terrain: their rows are 12 px to the +columns' 16 (3/4 of a tile per row) and cliffs are painted taller than their +tiles, so no single offset fixes a traced grid. + +### `decomp`: the grid and objects of one map + +```sh +python3 -m devtools.maps decomp ROUTE_204_NORTH # C++ rows and header comment +python3 -m devtools.maps decomp ROUTE_204_NORTH --text # bare rows +python3 -m devtools.maps decomp RAVAGED_PATH # a cave: its own matrix +python3 -m devtools.maps decomp ROUTE_204_NORTH --rect 160 660 32 46 # any world rectangle +python3 -m devtools.maps decomp ROUTE_204_NORTH --repo pret/pokediamond +``` + +The map is named by its header constant (`MAP_HEADER_` in pokeplatinum, +`MAP_` in pokediamond). Files are fetched once with +`gh api repos//contents/` into `~/.cache/maps/decomp//` +(`--cache` moves it, `--local DIR` reads a checkout instead). What is read: + +- `include/data/map_headers.h` (Platinum) or `arm9/src/map_header.c` and + `include/constants/maps.h` (Diamond): the map's matrix and events file. +- the matrix, `res/field/matrices/map_matrix_NNN.json` or + `files/fielddata/mapmatrix/map_matrix/narc_NNNN.bin`: 32 x 32 cells, the + overworld's is 30 x 30 (matrix 000); caves have their own (the Ravaged Path + is matrix 051, two cells tall). World x = 32 * column + lx, z = 32 * row + lz. + Without `--rect`, the rectangle is the cell(s) carrying the map's header. +- land data per cell, `res/field/maps/data/map_data_NNN.bin` or + `files/fielddata/land_data/land_data_release/narc_NNNN.bin`: a 16-byte header + of four u32 sizes, then 1024 little-endian u16 attributes, row-major, + index z * 32 + x. Bit 0x8000 blocks the tile; the low byte is the behaviour: + 0x02 tall grass `"`; 0x38..0x3b jump east/west/north/south, where only the + south ledge has a legend character (`v`), the others are `#` with a note; + 0x69 door and 0x62..0x65 cave mouths (blocked, walked into) `D`; 0x6c..0x6f + warp tiles (stood on) and 0x5e/0x5f warp stairs `W`; water `#`; cave floor + 0x08, sand 0x21 and anything else unblocked `.`. +- events, `res/field/events/events_.json` or + `files/fielddata/eventdata/zone_event_release/narc_NNNN.bin`: standing people + `N`, signposts `S`, items on the ground and cut trees, smashable rocks and + boulders `#`; a wanderer (`WANDER_*`, `WALK_*` movement) is left off the grid + and listed with its range, the walker waits for him. Warp events are listed + with their destination (a cave mouth's sits on the arrival tile in front of + it). Trainers come with their facing and sight range. Diamond's files give + these by number; pokeplatinum's `generated/*.txt` tables name them (the + numbering is shared: Route 204 North's twelve objects come out the same). + +The C++ output is a header comment ready to paste, then the rows: + +``` +// Map source: the decomp (map_data_031, 2026-10-31) with the objects from +// its events file. +// Console checks: none yet. +// Status: no checkpoint passes on this grid yet. +// +// Decoded from Pokemon Platinum's collision table for this map (the +// pret/pokeplatinum decompilation, map_data_031: one cell of the Sinnoh +// matrix at column 5, row 21), with the objects from its events file +// (events_route_204_north.json); BDSP keeps Diamond/Pearl's layout. Local +// (x, y) = world (x - 160, z - 672); 32 x 32 tiles. +// Water and the items on the ground are '#'. Objects, from the events file: +// warp tile (20, 26) to Ravaged Path; +// Aroma Lady Taylor (17, 20), looks south and east, sees 4; +// Twin Liv (15, 7), looks south, sees 3; +// ... +// Bug Catcher Brandon walk back and forth from (22, 13), columns 18 to 26, +// row 13, sees 1: off the grid; + // 0 1 2 3 + R"(########.#...###################)", // 0 + R"(########.#...###################)", // 1 + ... +``` + +The first three lines are the annotation `Data/MAPS.md` asks every map file to +keep current; fill in the console checks and status as runs happen. Against +`PokemonBDSP_AutoStory_Route204.cpp` the decoded rows differ only where that +file's header records a hand edit (Brandon placed where the console met him, +the cave's arrival tile kept open, a tile blocked on the console). Platinum +and Diamond decode Route 204 North identically. + +Platinum moved a few things after Diamond/Pearl; where the two decompilations +disagree, `--repo pret/pokediamond` is the one BDSP follows. + +## Overlay: a grid, and a run's routes, over the map image + +Map images stay out of the repo; `fetch` gets one from +`https://archives.bulbagarden.net/media/upload///`, where `h` +is the md5 of the file name (e.g. `Sinnoh_Route_204_DP.png`), into +`~/.cache/maps`. `overlay` draws a navigation data file's grid over the map +image it was read from, as SVG, and optionally the walks a console run planned +and walked: + +```sh +python3 -m devtools.maps fetch Sandgem_Town_DP.png +python3 -m devtools.maps overlay path/to/Data/..._SandgemTown.cpp -o sandgem.svg +python3 -m devtools.maps overlay path/to/Data/..._SandgemTown.cpp \ + --log "$HOME/Library/Application Support/SerialPrograms/Profiles//SerialPrograms.log" \ + --last 3 -o sandgem-run.svg +python3 -m devtools.maps decomp ROUTE_204_NORTH --text > r204n.txt +python3 -m devtools.maps overlay r204n.txt --image ~/.cache/maps/Sinnoh_Route_204_DP.png \ + --origin 0,0 --row-px 12 --level 12:24 -o r204n.svg +python3 -m unittest discover -s devtools/maps/tests -t . +``` + +The source is a C++ data file (its `R"(...)"` rows, `const TilePos NAME{x, y};` +tiles, and from its header the image `(Name.png, W x H px)` and where tile +(0, 0) starts, `starts 8 px in from the left edge`) or a text file of bare +rows such as `decomp --text` prints, with `--image` and `--origin X,Y` given. +Walls are red, tall grass green, ledges purple, signs orange, doors blue, +every fifth row and column numbered. With `--log`, a leg planned "to wall" +that stopped on anything but `blocked` is marked yellow: the grid has a wall +where the game doesn't. + +A decomp grid over a DP image needs the image's geometry: `--row-px 12` for +its 12 px rows, and `--level ROW:PX` for each cliff, meaning rows from `ROW` +down sit `PX` lower than the rows above (about 24 px per level; repeat the +option per cliff, reading the rows off the picture). Town maps drawn at 16 px +rows need neither. + +## Maps without a decomp: `levels.py` + +For a map the decompilations don't cover, `levels.py` reads walkable tiles off +the image by colour, level by level. Towns and caves have terraces and ledges +bordered by rock rims, joined only by stairs; judging each 16 px tile by its +colours alone walks a route off one level onto another wherever a rim stripe +is thin, so it works at pixel resolution first: + +1. Each pixel is walkable (paving, road, grass, curb), stairs, or blocked + (rims, roofs, trees, water). +2. Walkable pixels are flood-filled into regions; a rim of any width keeps + two levels apart. +3. Each blob of stair pixels joins the regions it touches. +4. Each tile takes the region holding at least half its pixels. A step is + allowed between tiles of the same region, or onto and off a stair tile + that touches both. +5. A breadth-first search gives the route; the output lists its corners. + +```sh +swiftc -O png2bmp.swift -o png2bmp # sips mangles palette PNGs +./png2bmp Veilstone_City_DP.png veil.bmp +python3 levels.py veil.bmp out/veil --offset 8,9 \ + --stairs 704,290 --stairs 512,527 --stairs 80,665 ... # one pixel per staircase + --from 605,474 --to 82,470 # pixel points +# writes out/veil_regions.bmp, out/veil_grid.txt, out/veil_route.txt +swiftc -O draw_route.swift -o draw_route +./draw_route Veilstone_City_DP.png route.png "$(head -1 out/veil_route.txt)" +``` + +Options for harder maps: + +- `--profile town|sandtown|oreburgh|cave`: colour rules. `oreburgh` counts the + conveyor belts as ground (we walk under them; the pillars block). + `sandtown` adds dirt roads and sand (Sandgem) to `town`. +- `--height H`: stretch a squashed image back to H pixels tall. Oreburgh + Mine B2F is 512x395 for a 512x512 map: its rocks are 16 px apart across + and 12.5 px down. +- `--objects on`: rocks, boulders, crates and people standing on the floor + block the tile under their feet, even when drawn smaller than a tile or + off the grid. +- `--edits FILE`: tiles opened or blocked from what the console showed, + `open ROW COL0 COL1` / `block ROW COL0 COL1`; edits apply before objects. + E.g. rows the coal block's edge is drawn over are walkable, and people + touching the coal block are not found as objects. + +`examples/` keeps the grids levels.py read for Veilstone City, Oreburgh City +and Oreburgh Mine B2F, with the mine's edits file, as text. They were traced +from DP images before the decomp was in use, so they carry the 3/4-row +drift; take the terrain of those maps from `decomp` and use these only to +see what the colour rules make of a town. + +Check per map: + +- **Tile alignment** (`--offset`): not always (0, 0). For Veilstone it is + (8, 9), read from the edges of the three-tile stairs at the bottom. +- **Colours**: the rules are for Sinnoh towns. Caves need their own. +- **Stairs** are given by hand, one pixel inside each. +- **Not modelled:** one-way ledges and people. Static NPCs are added from + what the console shows. +- **Row height**: a DP image's rows are 3/4 of a tile; a traced grid drifts + by the map's height differences. Prefer the decomp wherever it has the map. + +Sandgem Town, levels.py against the console-checked grid: with `sandtown` +and `--objects on`, rows 0..7 agree but for two lit door fronts read as +objects (we stood on (17, 7)) and the Lab's front, (6..9, 7), read as blocked +(walked on the console). Lawns, roads and the beach come out as separate +regions with no ledge between them, and the image squeezes rows 7..18 into +7..15, so the south half is taken from the game's collision instead. + +## Map, route, walk: getting a route to work on the console + +A route that fails on the console has failed in one of three layers. Find +which one before changing anything, and fix that layer: + +| Layer | What it is | How it is checked | +| --- | --- | --- | +| Map | the tile grid: walls, signs, doors, exits, NPCs | terrain from the decomp; people and BDSP's differences by measuring on the console | +| Route | the legs a checkpoint walks | offline, by the plan tests, before any console run | +| Walk | holding the stick until something stops us | console runs; the route absorbs its error | + +- **Map**: a leg planned "to wall" ran out of time, or went through a + transition, or stopped where the grid has no wall. Measure that place + again; don't tune the walker. +- **Route**: the plan used something it must not, such as an exit as a wall or + a long open leg. Add the plan test that would have caught it. +- **Walk**: map and plan are right, but a hold moved a different number of + tiles, or someone stood in the way. End more legs on walls, or add a + re-anchor. + +### Measuring a map + +Decode the grid first; the console then confirms its anchors and finds the +people. Measure with the Command File Runner's `walk x y`, which holds the +stick until the screen stops changing and prints how long it moved. Don't use +timed `stick` holds for this: on BDSP the same 300 ms hold moved one row on +one try and two on the next. Write down: + +- borders and exits: where each edge stops you or takes you elsewhere; +- anchors: tiles a walk can prove it stands on (two walls, a door, the tile + beside a sign), and how each is proven; +- distances between anchors, as `walk` measures them; +- objects: the row and column of each sign, NPC and door relative to the walls + around it, which sides it can be read or talked from, and whether the tiles + next to it are open. + +Decide rows by stops and sign reads, never by eye: at the overworld camera's +angle an NPC on your row looks a row higher, and a sign a row below the trees +can look right under them. Mark exits (steps, walkable edges) `D`, never `#`; +a `#` there is a wall the planner will walk "to", and off the map. Put what +was measured in the data file's header ("Console checks: ..."). + +### Planning the route + +- Start and end each checkpoint on an anchor, and save there, so a reload + starts exactly there; its first event checks the anchor and re-anchors if + the check fails. +- End every leg against something: a wall, a sign, an NPC who answers. Keep + open legs to a tile or two, and follow each with a leg that shows whether it + took. +- Out and back can take different rows; pick each direction's legs for the + walls they end on. +- A sign is a gate: walk until it stops you and press A. If it doesn't open, + you are not on its row. +- Check every planned walk offline, and draw it with `overlay` before the + first console run. diff --git a/devtools/maps/__init__.py b/devtools/maps/__init__.py new file mode 100644 index 0000000000..fc48289a90 --- /dev/null +++ b/devtools/maps/__init__.py @@ -0,0 +1 @@ +"""Map tools for AutoStory routes: grids from map images, and grids and logged routes drawn over them.""" diff --git a/devtools/maps/__main__.py b/devtools/maps/__main__.py new file mode 100644 index 0000000000..57656e2a62 --- /dev/null +++ b/devtools/maps/__main__.py @@ -0,0 +1,89 @@ +"""maps: decode a map's grid from the decomp, fetch a map image, and draw a grid and logged routes over it. + + python3 -m devtools.maps decomp ROUTE_204_NORTH # C++ rows and header, to paste + python3 -m devtools.maps decomp ROUTE_204_NORTH --text > r204n.txt # bare rows, for overlay or diff + python3 -m devtools.maps fetch Sandgem_Town_DP.png + python3 -m devtools.maps overlay [--log SerialPrograms.log [--walk LABEL]] -o out.svg + python3 -m devtools.maps overlay r204n.txt --image ~/.cache/maps/Sinnoh_Route_204_DP.png --row-px 12 --level 12:24 -o r204n.svg +""" +import argparse +import sys +from pathlib import Path + +from . import decomp, fetch, grid_source, route_log, svg + +CACHE = Path.home() / '.cache' / 'maps' + + +def level(text): + """ROW:PX, rows from ROW down drawn PX further down the image (a cliff level).""" + row, px = text.split(':') + return int(row), int(px) + + +def main(argv=None): + parser = argparse.ArgumentParser(prog="maps") + sub = parser.add_subparsers(dest='command', required=True) + dec = sub.add_parser('decomp', help="decode a map's tile grid and objects from the pret decompilation") + dec.add_argument('map', help='the map header name, e.g. ROUTE_204_NORTH or RAVAGED_PATH') + dec.add_argument('--repo', default=decomp.Platinum.REPO, choices=[decomp.Platinum.REPO, decomp.Diamond.REPO], + help='which decompilation to read (default: pret/pokeplatinum)') + dec.add_argument('--rect', nargs=4, type=int, metavar=('X0', 'Z0', 'W', 'H'), + help="world rectangle to decode; default: the map's own cell(s)") + dec.add_argument('--text', action='store_true', help='print the bare rows instead of the C++ block') + dec.add_argument('--local', type=Path, help='a checkout of the decompilation, instead of fetching with gh') + dec.add_argument('--cache', type=Path, default=CACHE) + get = sub.add_parser('fetch', help='download a Bulbapedia map image into the cache') + get.add_argument('name', help='file name on Bulbapedia, e.g. Sandgem_Town_DP.png') + get.add_argument('--cache', type=Path, default=CACHE) + draw = sub.add_parser('overlay', help='draw a grid, and logged routes, over its map image') + draw.add_argument('source', type=Path, help='the C++ data file with the grid, or a text file of bare rows') + draw.add_argument('--image', type=Path, help='the map image; default: the one the file names, fetched') + draw.add_argument('--origin', type=lambda s: tuple(int(v) for v in s.split(',')), metavar='X,Y', + help='pixel of tile (0, 0) on the image; default: from the file header, or 0,0') + draw.add_argument('--row-px', type=int, default=0, + help='height of a tile row in pixels; default 16, the DP route images need 12') + draw.add_argument('--level', type=level, action='append', default=[], metavar='ROW:PX', + help='rows from ROW down sit PX lower on the image (a cliff level; repeatable)') + draw.add_argument('--log', type=Path, help='a SerialPrograms log to take planned walks from') + draw.add_argument('--walk', action='append', help='only walks with this label (repeatable)') + draw.add_argument('--last', type=int, default=0, help='only the last N walks') + draw.add_argument('--cache', type=Path, default=CACHE) + draw.add_argument('-o', '--out', type=Path, required=True) + args = parser.parse_args(argv) + + if args.command == 'decomp': + game = decomp.game_for(args.repo, args.cache / 'decomp', args.local) + decoded = decomp.decode(game, args.map, tuple(args.rect) if args.rect else None) + print(decomp.render_text(decoded) if args.text else decomp.render_cpp(decoded)) + return 0 + if args.command == 'fetch': + print(fetch.fetch(args.name, args.cache)) + return 0 + + source = grid_source.load(args.source) + if args.origin: + source.origin = args.origin + if args.row_px: + source.row_px = args.row_px + source.levels = sorted(args.level) + image = args.image + if image is None and source.image: + image = fetch.fetch(source.image, args.cache) + walks = [] + if args.log: + lines = args.log.read_text(encoding='utf-8', errors='replace').splitlines() + walks = route_log.parse(lines) + if args.walk: + walks = [w for w in walks if w.label in args.walk] + walks = [w for w in walks if any(p[0] == source.name for p in w.start + [e for l in w.legs for e in l.ends])] + if args.last: + walks = walks[-args.last:] + title = f'{source.name}: {len(source.rows)} rows x {source.width}, {len(walks)} walk(s)' + args.out.write_text(svg.render(source, image.read_bytes() if image else None, walks, title), encoding='utf-8') + print(args.out) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/devtools/maps/decomp.py b/devtools/maps/decomp.py new file mode 100644 index 0000000000..f5e85b6c55 --- /dev/null +++ b/devtools/maps/decomp.py @@ -0,0 +1,530 @@ +"""Decodes a map's tile grid from the Pokemon Platinum or Diamond decompilation. + +The pret decompilations hold each 32 x 32 map cell's collision as a land data +file: a 16-byte header of four u32 sizes (attributes, movement models, map +models, height data), then 1024 little-endian u16 tile attributes, row-major, +index z * 32 + x. Bit 0x8000 blocks the tile; the low byte is its behaviour +(tall grass, water, ledges, doors, warps). Cells sit in a matrix: the +overworld's is 30 x 30 (matrix 000), caves have their own; world +x = 32 * column + lx, z = 32 * row + lz, and the events files place objects +in these world coordinates. + +pokeplatinum keeps matrices and events as JSON and its map headers in +include/data/map_headers.h. pokediamond keeps them as binary NARC entries +(the layout pokeplatinum's tools/jsoncnv packs) with the header table in +arm9/src/map_header.c, and gives sprites, movement types and trainer types +by number; pokeplatinum's generated/*.txt tables name them. BDSP keeps +Diamond/Pearl's layout; Platinum's few changes show up on the console. +""" +import datetime +import json +import re +import struct +import subprocess +import textwrap +from dataclasses import dataclass, field +from pathlib import Path + +CELL = 32 +# Tile attribute bits and behaviours (include/constants/field/map_tile_behaviors.h). +BLOCKED = 0x8000 +TALL_GRASS = {0x02, 0x03} +WATER = {0x10, 0x13, 0x15} # river, waterfall, sea +JUMP = {0x38: 'east', 0x39: 'west', 0x3a: 'north', 0x3b: 'south'} +JUMP_TWICE = {0x5a: 'north', 0x5b: 'south', 0x5c: 'west', 0x5d: 'east'} +WARP_STAIRS = {0x5e, 0x5f} +CAVE_MOUTHS = {0x62, 0x63, 0x64, 0x65} # blocked, walked into +DOOR = 0x69 +WARP_TILES = {0x6c, 0x6d, 0x6e, 0x6f} # walkable, stood on +# Objects that stand on the ground and block it. +OBSTACLES = {'CUT_TREE', 'ROCK_SMASH', 'STRENGTH_BOULDER'} +ITEM = 'POKEBALL' +# bg events: 0 facing, 1 wall sign, 2 hidden item (generated/bg_event_types.txt). +BG_SIGNS = {0, 1} + + +def tile_char(attr): + """Our legend for one tile attribute: (char, note or None).""" + behaviour = attr & 0xff + if behaviour in JUMP or behaviour in JUMP_TWICE: + twice = behaviour in JUMP_TWICE + side = JUMP_TWICE[behaviour] if twice else JUMP[behaviour] + if side == 'south': + return 'v', 'double ledge' if twice else None + return '#', f'ledge jumped {side}{" twice" if twice else ""}, shown as a wall' + if behaviour == DOOR or behaviour in CAVE_MOUTHS: + return 'D', None + if behaviour in WARP_TILES or behaviour in WARP_STAIRS: + return 'W', None + if behaviour in WATER: + return '#', 'water' + if attr & BLOCKED: + return '#', None + if behaviour in TALL_GRASS: + return '"', None + return '.', None + + +def decode_land_data(data): + """The 1024 tile attributes of a land data file (map_data_NNN.bin / narc_NNNN.bin).""" + sizes = struct.unpack('<4I', data[:16]) + if sizes[0] != 2 * CELL * CELL: + raise ValueError(f'land data: attribute block is {sizes[0]} bytes, expected {2 * CELL * CELL}') + return list(struct.unpack(f'<{CELL * CELL}H', data[16:16 + sizes[0]])) + + +def gh_fetch(repo, path): + """One file of a GitHub repository, raw; binary works.""" + result = subprocess.run( + ['gh', 'api', f'repos/{repo}/contents/{path}', '-H', 'Accept: application/vnd.github.raw'], + check=True, stdout=subprocess.PIPE) + return result.stdout + + +class Source: + """Files of one decompilation: a local checkout, or fetched once into the cache.""" + + def __init__(self, repo, cache, local=None, fetch=gh_fetch): + self.repo = repo + self.cache = Path(cache) + self.local = Path(local) if local else None + self.fetch = fetch + + def read(self, path): + if self.local: + return (self.local / path).read_bytes() + cached = self.cache / self.repo / path + if not cached.exists(): + data = self.fetch(self.repo, path) + cached.parent.mkdir(parents=True, exist_ok=True) + cached.write_bytes(data) + return cached.read_bytes() + + def text(self, path): + return self.read(path).decode('utf-8', errors='replace') + + +@dataclass +class MapHeader: + name: str + matrix_id: int + events_id: object # Platinum: events archive name; Diamond: NARC index + + +@dataclass +class Matrix: + matrix_id: int + width: int + height: int + maps: list # land data id per cell, row-major, None for MAP_NONE + headers: list = None # map header name per cell, or None when the matrix has none + + def cells_of(self, header_name): + """(row, col) of every cell carrying this header; all cells when the matrix has no headers.""" + cells = [(i // self.width, i % self.width) for i in range(self.width * self.height) + if self.maps[i] is not None and (self.headers is None or self.headers[i] == header_name)] + return cells + + +@dataclass +class ObjectEvent: + name: str + gfx: str + movement: str + trainer: bool + sight: int + range_x: int + range_z: int + x: int + z: int + hidden: str = '' + + +@dataclass +class Warp: + x: int + z: int + dest: str + + +@dataclass +class Sign: + x: int + z: int + + +@dataclass +class Events: + objects: list = field(default_factory=list) + warps: list = field(default_factory=list) + signs: list = field(default_factory=list) + + +def strip_prefix(name, prefix): + return name[len(prefix):] if name.startswith(prefix) else name + + +class Platinum: + """pret/pokeplatinum: JSON matrices and events, map headers in include/data/map_headers.h.""" + REPO = 'pret/pokeplatinum' + GAME = 'Pokemon Platinum' + HEADER = re.compile(r'\[MAP_HEADER_(\w+)\]\s*=\s*\{(.*?)\}', re.S) + + def __init__(self, source): + self.source = source + self._headers = None + + def header(self, name): + if self._headers is None: + text = self.source.text('include/data/map_headers.h') + self._headers = {m.group(1): m.group(2) for m in self.HEADER.finditer(text)} + body = self._headers.get(name) + if body is None: + raise KeyError(f'{name}: no MAP_HEADER_{name} in include/data/map_headers.h') + matrix = re.search(r'\.mapMatrixID\s*=\s*map_matrix_(\d+)', body) + events = re.search(r'\.eventsArchiveID\s*=\s*(\w+)', body) + return MapHeader(name, int(matrix.group(1)), events.group(1)) + + def matrix(self, matrix_id): + data = json.loads(self.source.text(f'res/field/matrices/map_matrix_{matrix_id:03d}.json')) + maps = [None if m == 'MAP_NONE' else int(m[len('MAP_'):]) for row in data['maps'] for m in row] + headers = [strip_prefix(h, 'MAP_HEADER_') for row in data['headers'] for h in row] or None + return Matrix(matrix_id, len(data['maps'][0]), len(data['maps']), maps, headers) + + def land_name(self, land_id): + return f'map_data_{land_id:03d}' + + def land_path(self, land_id): + return f'res/field/maps/data/{self.land_name(land_id)}.bin' + + def land_data(self, land_id): + return decode_land_data(self.source.read(self.land_path(land_id))) + + def events_name(self, events_id): + return f'{events_id}.json' + + def events(self, events_id): + data = json.loads(self.source.text(f'res/field/events/{events_id}.json')) + events = Events() + for o in data.get('object_events', []): + trainer = o['trainer_type'] != 'TRAINER_TYPE_NONE' + events.objects.append(ObjectEvent( + name=title(strip_prefix(o['id'], 'LOCALID_')), + gfx=strip_prefix(o['graphics_id'], 'OBJ_EVENT_GFX_'), + movement=strip_prefix(o['movement_type'], 'MOVEMENT_TYPE_'), + trainer=trainer, + sight=(o.get('data') or [0])[0] if trainer else 0, + range_x=o.get('movement_range_x', 0), range_z=o.get('movement_range_z', 0), + x=o['x'], z=o['z'], + hidden='' if str(o.get('hidden_flag', '0')) == '0' else str(o['hidden_flag']))) + for w in data.get('warp_events', []): + events.warps.append(Warp(w['x'], w['z'], strip_prefix(w['dest_header_id'], 'MAP_HEADER_'))) + for b in data.get('bg_events', []): + if b['type'] in BG_SIGNS: + events.signs.append(Sign(b['x'], b['z'])) + return events + + +class Diamond: + """pret/pokediamond: binary matrices and events, the header table in arm9/src/map_header.c. + + Sprites, movement types and trainer types are numbers there; `names` is a + Source for pret/pokeplatinum, whose generated/*.txt tables list them in + order (checked on Route 204 North: the same twelve objects come out). + """ + REPO = 'pret/pokediamond' + GAME = 'Pokemon Diamond' + HEADER_LINE = re.compile(r'\{([^{}]*)\}\s*,?\s*//\s*(MAP_\w+)') + DEFINE = re.compile(r'#define\s+(MAP_\w+)\s+(\d+)') + + def __init__(self, source, names): + self.source = source + self.names = names + self._headers = None + self._map_ids = None + self._tables = {} + + def header(self, name): + if self._headers is None: + text = self.source.text('arm9/src/map_header.c') + self._headers = {m.group(2): m.group(1) for m in self.HEADER_LINE.finditer(text)} + body = self._headers.get(f'MAP_{name}') + if body is None: + raise KeyError(f'{name}: no MAP_{name} row in arm9/src/map_header.c') + matrix = re.search(r'NARC_map_matrix_narc_(\d+)_bin', body) + events = re.search(r'NARC_zone_event_release_narc_(\d+)_bin', body) + return MapHeader(name, int(matrix.group(1)), int(events.group(1))) + + def map_name(self, map_id): + """MAP_ constant name (without the prefix) for a numeric map header id.""" + if self._map_ids is None: + text = self.source.text('include/constants/maps.h') + self._map_ids = {int(m.group(2)): m.group(1) for m in self.DEFINE.finditer(text)} + return strip_prefix(self._map_ids.get(map_id, f'MAP_{map_id}'), 'MAP_') + + def table(self, name): + if name not in self._tables: + self._tables[name] = self.names.text(f'generated/{name}.txt').split() + return self._tables[name] + + def named(self, table, prefix, value): + entries = self.table(table) + return strip_prefix(entries[value], prefix) if value < len(entries) else str(value) + + def matrix(self, matrix_id): + data = self.source.read(f'files/fielddata/mapmatrix/map_matrix/narc_{matrix_id:04d}.bin') + width, height, has_headers, has_altitudes, name_len = struct.unpack_from('<5B', data, 0) + offset = 5 + name_len + count = width * height + headers = None + if has_headers: + headers = [self.map_name(h) for h in struct.unpack_from(f'<{count}H', data, offset)] + offset += 2 * count + if has_altitudes: + offset += count + maps = [None if m == 0xffff else m for m in struct.unpack_from(f'<{count}H', data, offset)] + return Matrix(matrix_id, width, height, maps, headers) + + def land_name(self, land_id): + return f'land_data narc_{land_id:04d}' + + def land_path(self, land_id): + return f'files/fielddata/land_data/land_data_release/narc_{land_id:04d}.bin' + + def land_data(self, land_id): + return decode_land_data(self.source.read(self.land_path(land_id))) + + def events_name(self, events_id): + return f'zone_event narc_{events_id:04d}' + + def events(self, events_id): + data = self.source.read(f'files/fielddata/eventdata/zone_event_release/narc_{events_id:04d}.bin') + events = Events() + offset = 0 + (count,) = struct.unpack_from('