diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml new file mode 100644 index 0000000000..cb577266da --- /dev/null +++ b/.github/workflows/code-health.yml @@ -0,0 +1,53 @@ +name: C++ Code Health +# Stub: the linter (devtools/code_health) and its self-tests live on the fork's agents branch. +# This checks that branch out next to the code and runs it against the code checkout. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + inputs: + base-ref: + description: Revision to compare with (e.g. origin/main); empty = whole tree + required: false + +permissions: + contents: read + +jobs: + code-health: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/checkout@v7 + with: + repository: dayfine/Arduino-Source + ref: agents + path: .agents-tools + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: .agents-tools/devtools/code_health/requirements.txt + - run: python -m pip install -r .agents-tools/devtools/code_health/requirements.txt + - name: Linter self-tests + working-directory: .agents-tools + env: + CODE_HEALTH_REPO: ${{ github.workspace }} + run: python -m unittest discover -s devtools/code_health/tests -v + - name: Check changed C++ files + working-directory: .agents-tools + env: + PR_BASE: ${{ github.event.pull_request.base.sha }} + INPUT_BASE: ${{ inputs.base-ref }} + PUSH_BASE: ${{ github.event.before }} + CODE: ${{ github.workspace }} + shell: bash + run: | + base="${PR_BASE:-${INPUT_BASE:-$PUSH_BASE}}" + if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]]; then + base=$(git -C "$CODE" hash-object -w -t tree --stdin /Resources/Tesseract`. +- `SerialProgramsTests` (built by default next to `SerialPrograms`; `-DTEST_RUNNER_EXE=OFF` + turns it off) runs the same tests without the GUI, settings or device scans, so it can run + while SerialPrograms is open: `ninja SerialProgramsTests`, then + `SerialProgramsTests --filter ` (repeatable) or `--list`. Its exit code is the number + of failed tests. Same requirements as above. - Test framework: test classes derive `UnitTest` (`Common/Cpp/TestRunners/UnitTest.h`) and are collected via per-module `add_tests(UnitTestDatabase&)` — see `SerialPrograms/Source/Tests/CommandLineTests.h`. diff --git a/Common/Cpp/Filesystem/Filesystem.h b/Common/Cpp/Filesystem/Filesystem.h index 21d350f014..b6cedfd202 100644 --- a/Common/Cpp/Filesystem/Filesystem.h +++ b/Common/Cpp/Filesystem/Filesystem.h @@ -22,6 +22,7 @@ #ifndef PokemonAutomation_Filesystem_Filesystem_H #define PokemonAutomation_Filesystem_Filesystem_H +#include #include #include #include "FilePath.h" @@ -115,11 +116,11 @@ Path application_install_path(); Path application_scratch_path(); -// Set a profile for program settings (/UserSettings/PROFILE_NAME/) on MacOS. -// Have to run the program with command-line argument "open -n PATH_TO_APP --args --profile PROFILE_NAME" to set the profile and launch a new window. -// This allows multiple instances of the program to run since settings are no longer shared. #if defined(__APPLE__) -void set_startup_profile(int& argc, char* argv[]); +// Set a validated macOS settings profile before any settings paths are cached. +// GUI launch uses --profile NAME or the build-bound PA_LAB_PROFILE identity. +void set_startup_profile(std::string_view profile); +// Return the identity used for the window title and scratch directory. const std::string& STARTUP_PROFILE(); #endif diff --git a/Common/Cpp/Filesystem/Filesystem_Mac.h b/Common/Cpp/Filesystem/Filesystem_Mac.h index a9dbc281ab..472233e0cc 100644 --- a/Common/Cpp/Filesystem/Filesystem_Mac.h +++ b/Common/Cpp/Filesystem/Filesystem_Mac.h @@ -21,22 +21,9 @@ namespace Filesystem{ static std::string g_startup_profile; -void set_startup_profile(int& argc, char* argv[]){ - for (int i = 1; i + 1 < argc; i++){ - if (strcmp(argv[i], "--profile") == 0){ - std::string profile = argv[i + 1]; - for (char c : profile){ - if (!(std::isalpha(c) || std::isdigit(c)) && c != u'_' && c != u'-') c = u'_'; - } - g_startup_profile = std::move(profile); - // Shift everything after --profile down by 2. - for (int j = i; j + 2 < argc; j++){ - argv[j] = argv[j + 2]; - } - argc -= 2; - return; - } - } +// Store the already-validated identity before any scratch/settings path is cached. +void set_startup_profile(std::string_view profile){ + g_startup_profile = profile; } const std::string& STARTUP_PROFILE(){ diff --git a/Common/Cpp/HttpServer/HttpServer.cpp b/Common/Cpp/HttpServer/HttpServer.cpp new file mode 100644 index 0000000000..b4c97cbab1 --- /dev/null +++ b/Common/Cpp/HttpServer/HttpServer.cpp @@ -0,0 +1,821 @@ +/* HTTP Server + * + * From: https://github.com/PokemonAutomation/ + * + * Implementation notes: + * - The socket's receive/disconnect callbacks only append data to the + * per-connection buffer and flip flags. One "connection task" per + * connection (from the thread pool) does all parsing and writing, so + * the handler may run for a long time without stalling the socket's + * receive loop. + * - Every write goes through "AbstractServerSocket::send()" with the + * connection's socket id (captured at accept time), so a response + * that outlives its connection - the peer disconnected and the next + * client was accepted in the meantime - is refused by the socket + * instead of reaching the new client's stream. + * - "Expect: 100-continue" is answered as soon as the headers are + * validated, while the body is still pending (a waiting client is + * blocked on that interim response). + * - Task handles live in "m_tasks" under a dedicated "m_task_lock" + * (the socket library's "m_task_lock" pattern): destroying an + * "AsyncTask" handle blocks until that task finishes, and the task's + * final cleanup takes "m_lock", so a handle must never be destroyed + * while holding "m_lock". "on_connect()" may therefore find the + * previous connection's task still finishing (for example, still in a + * long handler call); its handle is kept and reaped later. + * - "stop()" marks the active connection dead before it waits for the + * in-flight task: the socket suppresses its disconnect callback for a + * close_connection() close, so without the mark a handler polling + * "connection_alive()" would never see the shutdown. "stop()" then + * moves the socket out under "m_lock" and destroys it after releasing + * the lock: "~ServerSocket" joins the socket's worker threads, and + * their callbacks take "m_lock". + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/ThreadPool.h" +#include "Common/Cpp/HttpServer/HttpServer.h" +#include "Common/Cpp/Sockets/ServerSocket.h" + +namespace PokemonAutomation{ + +namespace{ + + +constexpr size_t MAX_REQUEST_LINE_BYTES = 8192; +constexpr size_t MAX_HEADER_SECTION_BYTES = 65536; +constexpr size_t MAX_HEADER_COUNT = 100; +constexpr size_t MAX_BODY_BYTES = 1u << 20; // 1 MiB +// The total per-connection receive buffer; enforced when data is +// appended, not at parse time. +constexpr size_t MAX_BUFFER_BYTES = 2u << 20; + +// A stalled peer is declared dead after this; loopback writes of a few MB +// finish in milliseconds, so this is very generous. +constexpr std::chrono::milliseconds WRITE_TIMEOUT{30000}; + + +bool is_token_char(char c){ + return std::isalnum((unsigned char)c) != 0 + || std::strchr("-!#$%&'*+.^_`|~", c) != nullptr; +} + + +std::string to_lower(std::string value){ + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c){ + return (char)std::tolower(c); + }); + return value; +} + + +// Strips the leading and trailing spaces and tabs (OWS). +std::string trim_ows(std::string value){ + static const char OWS[] = " \t"; + const auto begin = value.find_first_not_of(OWS); + if (begin == std::string::npos){ + return ""; + } + const auto end = value.find_last_not_of(OWS); + return value.substr(begin, end - begin + 1); +} + + +// The concrete HandlerContext: "connection_alive()" is simply the +// connection's dead flag. +struct HandlerContextImpl : AbstractHttpServer::HandlerContext{ + explicit HandlerContextImpl(const std::atomic& dead) + : m_dead(dead) + {} + + virtual bool connection_alive() const override{ + return !m_dead.load(std::memory_order_acquire); + } + +private: + const std::atomic& m_dead; +}; + + + +} // anonymous namespace + + + +const char* HttpServer::status_reason(int status){ + switch (status){ + case 100: + return "Continue"; + case 200: + return "OK"; + case 201: + return "Created"; + case 204: + return "No Content"; + case 301: + return "Moved Permanently"; + case 302: + return "Found"; + case 304: + return "Not Modified"; + case 400: + return "Bad Request"; + case 401: + return "Unauthorized"; + case 403: + return "Forbidden"; + case 404: + return "Not Found"; + case 405: + return "Method Not Allowed"; + case 408: + return "Request Timeout"; + case 411: + return "Length Required"; + case 413: + return "Payload Too Large"; + case 426: + return "Upgrade Required"; + case 500: + return "Internal Server Error"; + case 501: + return "Not Implemented"; + case 503: + return "Service Unavailable"; + case 505: + return "Version Not Supported"; + default: + return "Unknown Status"; + } +} + + +std::string HttpServer::format_response(const Response& response, bool keep_alive){ + std::string out; + out += "HTTP/1.1 "; + out += std::to_string(response.status); + out += " "; + const char* reason = response.reason.empty() ? status_reason(response.status) : response.reason.c_str(); + out += reason; + out += "\r\n"; + + for (const auto& header : response.headers){ + // The server sets both of these below. + if (to_lower(header.first) == "content-length" || + to_lower(header.first) == "connection" + ){ + continue; + } + out += header.first; + out += ": "; + out += header.second; + out += "\r\n"; + } + + out += "Content-Length: "; + out += std::to_string(response.body.size()); + out += "\r\n"; + out += "Connection: "; + out += keep_alive ? "keep-alive" : "close"; + out += "\r\n\r\n"; + out += response.body; + + return out; +} + + + +HttpServer::HttpServer(ThreadPool& thread_pool) + : m_thread_pool(thread_pool) +{} + + +HttpServer::~HttpServer(){ + stop(); +} + + +void HttpServer::start(const std::string& address, uint16_t port, Handler handler){ + AbstractServerSocket* socket = nullptr; + { + std::lock_guard lg(m_lock); + if (m_socket != nullptr){ + return; + } + m_handler = std::move(handler); + m_stop_requested.store(false, std::memory_order_release); + m_socket = std::make_unique(m_thread_pool); + // Register before start(): the accept thread is dispatched by + // start() and may report on_listen_finished() as soon as it runs. + m_socket->add_listener(*this); + socket = m_socket.get(); + } + + // Asynchronous: on_listen_finished() fires on a thread from the pool. + socket->start(address, port); +} + + +void HttpServer::stop(){ + m_stop_requested.store(true, std::memory_order_release); + + AbstractServerSocket* socket = nullptr; + { + std::lock_guard lg(m_lock); + socket = m_socket.get(); + } + if (socket == nullptr){ + return; + } + + // Mark the active connection dead: the socket suppresses its + // disconnect callback for a close_connection() close, so without + // this a handler polling connection_alive() would never see the + // shutdown, and the wait for it below would never end. + { + std::lock_guard lg(m_lock); + if (m_connection != nullptr){ + m_connection->dead.store(true, std::memory_order_release); + m_connection->cv.notify_all(); + } + } + + // Kill the active connection, if any: it cancels in-flight writes and + // makes the socket's connection thread exit. The socket keeps + // listening until it is destroyed below, so it may accept a few more + // clients in the meantime; on_connect() sees m_stop_requested and + // closes them without dispatching a connection task. + socket->close_connection(); + + // Wait for the in-flight connection tasks. The handles are moved out + // under m_task_lock (never destroyed while holding m_lock: a task's + // final cleanup takes m_lock, and destroying a handle blocks until + // the task finishes). Loop, because a task may have been dispatched + // between the previous ones being joined and the socket being + // destroyed. + while (true){ + std::vector tasks; + { + std::lock_guard lg(m_task_lock); + tasks.swap(m_tasks); + } + if (tasks.empty()){ + break; + } + for (auto& task : tasks){ + task.wait_and_ignore_exceptions(); + } + } + + // Take the socket out under m_lock and destroy it after releasing + // the lock: "~ServerSocket" joins the socket's worker threads, and + // their callbacks (on_connect and friends) take m_lock. + std::unique_ptr owned; + { + std::lock_guard lg(m_lock); + owned = std::move(m_socket); + m_connection = nullptr; + m_handler = Handler(); + } +} + + +bool HttpServer::is_running() const{ + std::lock_guard lg(m_lock); + return (m_socket != nullptr); +} + + + +void HttpServer::on_listen_finished(const std::string& error_message){ + m_listeners.run_method(&Listener::on_listen_finished, error_message); +} + + +void HttpServer::on_connect(){ + AbstractServerSocket* socket = nullptr; + std::shared_ptr conn; + { + std::lock_guard lg(m_lock); + socket = m_socket.get(); + if (socket != nullptr && + !m_stop_requested.load(std::memory_order_acquire) + ){ + conn = std::make_shared(); + // The previous socket connection is over (the socket only + // accepts a new client once the previous one is gone), but + // its task may still be finishing (for example, still inside + // a long handler call), and the socket's on_disconnect + // callback for it may not have run yet: it runs after the + // socket has already moved back to listening. Mark it dead + // now, before the new connection takes over the socket's + // active fd, so the old task discards its response instead + // of delivering it to the new client. + if (m_connection != nullptr){ + m_connection->dead.store(true, std::memory_order_release); + m_connection->cv.notify_all(); + } + m_connection = conn; + // The token of the connection this one will serve; send() + // refuses to write through a socket that has moved on. + conn->socket_id = socket->connection_id(); + } + } + if (conn == nullptr){ + // Stopping (or the socket is gone): close the connection without + // a connection task. + if (socket != nullptr){ + socket->close_connection(); + } + }else{ + // Dispatch outside m_lock: the task's final cleanup takes + // m_lock, and so does stop(). The task handles live in m_tasks + // under a dedicated m_task_lock (the socket library's + // m_task_lock pattern): destroying an AsyncTask handle blocks + // until its task finishes, so a handle must never be destroyed + // while holding m_lock. + try{ + AsyncTask task = m_thread_pool.dispatch( + [this, conn, socket]{ + connection_task(conn, socket); + } + ); + std::lock_guard lg(m_task_lock); + // Reap finished handles (destroying one that is already + // finished is a no-op). Keep unfinished ones: the + // previous connection's task may still be finishing up + // (for example, still inside a long handler call) by the + // time this connection is accepted; it holds only its + // own (dead) connection. + m_tasks.erase( + std::remove_if(m_tasks.begin(), m_tasks.end(), [](const AsyncTask& task){ + return task.is_finished(); + }), + m_tasks.end() + ); + m_tasks.push_back(std::move(task)); + }catch (...){ + // The task could not be started: drop the connection and + // keep listening. + { + std::lock_guard lg(m_lock); + if (m_connection.get() == conn.get()){ + m_connection = nullptr; + } + } + socket->close_connection(); + } + } + m_listeners.run_method(&Listener::on_connect); +} + + +void HttpServer::on_connect_rejected(){ + m_listeners.run_method(&Listener::on_connect_rejected); +} + + +void HttpServer::on_receive_data(const void* data, size_t bytes){ + std::shared_ptr conn; + AbstractServerSocket* socket = nullptr; + { + std::lock_guard lg(m_lock); + conn = m_connection; + socket = m_socket.get(); + } + if (conn == nullptr){ + return; + } + bool exceeded = false; + { + std::lock_guard lg(conn->lock); + exceeded = conn->buffer.size() + bytes > MAX_BUFFER_BYTES; + if (!exceeded){ + const char* ptr = (const char*)data; + conn->buffer.append(ptr, bytes); + } + } + conn->cv.notify_all(); + if (exceeded && socket != nullptr){ + // The stream is longer than we are willing to buffer. It is + // likely in the middle of a request, so there is no reliable + // response left to send: just end the connection. ("socket" can + // be null only while stop() is tearing the socket down, in which + // case there is no connection task left to wake up.) + end_connection(conn, socket); + } +} + + +void HttpServer::on_disconnect(const std::string& error_message){ + std::shared_ptr conn; + { + std::lock_guard lg(m_lock); + // The socket may already be serving the next client by the time + // this callback runs: the socket's connection thread sets the + // state back to LISTENING before it reports the disconnect, so a + // fast reconnect can be accepted in between. In that case this + // disconnect is about the previous connection, which on_connect() + // has already marked dead; the current one belongs to the new + // client and must not be killed. + if (m_socket == nullptr || + m_socket->state() != AbstractServerSocket::State::CONNECTED + ){ + conn = m_connection; + } + } + if (conn != nullptr){ + conn->dead.store(true, std::memory_order_release); + conn->cv.notify_all(); + } + m_listeners.run_method(&Listener::on_disconnect, error_message); +} + + + +void HttpServer::connection_task( + const std::shared_ptr& conn, + AbstractServerSocket* socket +){ + // The handler may be replaced or cleared by start()/stop() while this + // task is waiting; a copy taken under m_lock is stable for the task's + // whole lifetime, because stop() waits for the task before clearing it. + Handler handler; + { + std::lock_guard lg(m_lock); + handler = m_handler; + } + + // "Expect: 100-continue" is answered at most once, and only while + // the body is still pending. + bool sent_continue = false; + + while (true){ + if (m_stop_requested.load(std::memory_order_acquire) || + conn->dead.load(std::memory_order_acquire) + ){ + break; + } + + ParsedRequest parsed; + size_t buffer_size; + { + std::lock_guard lg(conn->lock); + parsed = try_parse(conn->buffer); + buffer_size = conn->buffer.size(); + } + + if (parsed.state == ParseState::NEED_BODY){ + // The headers are complete and valid, but the body has not + // fully arrived. A client that sent "Expect: 100-continue" is + // still waiting for this before sending the body: answer it + // now, not after the (still missing) body arrives. (If the + // whole request was already buffered, the state is READY and + // the final response alone moves the client on; RFC 9110). + if (parsed.expect_continue && !sent_continue){ + static const char CONTINUE_RESPONSE[] = "HTTP/1.1 100 Continue\r\n\r\n"; + if (!write_raw(conn, socket, CONTINUE_RESPONSE)){ + break; + } + sent_continue = true; + } + } + + if (parsed.state == ParseState::NEED_MORE || + parsed.state == ParseState::NEED_BODY + ){ + // Wait for the rest of this request, a disconnect, or a + // stop. The predicate requires the buffer to have grown past + // what the parse just saw: with a partial request, + // "!buffer.empty()" alone is permanently true and would spin + // this loop at 100% CPU until the request completes. + std::unique_lock lg(conn->lock); + conn->cv.wait_for(lg, std::chrono::milliseconds(10), [this, &conn, buffer_size]{ + return conn->buffer.size() != buffer_size + || conn->dead.load(std::memory_order_relaxed) + || m_stop_requested.load(std::memory_order_relaxed); + }); + continue; + } + + if (parsed.state == ParseState::ERROR_RESPONSE){ + Response response; + response.status = parsed.error_status; + response.headers = std::move(parsed.error_headers); + response.body = parsed.error_reason + "\n"; + write_response(conn, socket, response, false); + end_connection(conn, socket); + break; + } + + // A complete request is ready to be handled. + Response response; + { + HandlerContextImpl context(conn->dead); + try{ + if (handler){ + handler(parsed.request, response, context); + }else{ + response.status = 500; + response.body = "No handler is registered.\n"; + response.connection_close = true; + } + }catch (...){ + response = Response(); + response.status = 500; + response.body = "The handler threw an exception.\n"; + response.connection_close = true; + } + } + + // The connection may have closed or been replaced by a new + // client while the handler was running; discard the response in + // that case. + if (conn->dead.load(std::memory_order_acquire) || + m_stop_requested.load(std::memory_order_acquire) + ){ + break; + } + + const bool keep_alive = parsed.http_1_1 + && !parsed.connection_close_requested + && !response.connection_close; + if (!write_response(conn, socket, response, keep_alive)){ + break; + } + if (!keep_alive){ + end_connection(conn, socket); + break; + } + // Keep-alive: the next request (possibly already buffered) is + // picked up at the top of the loop. + } + + conn->dead.store(true, std::memory_order_release); + { + std::lock_guard lg(m_lock); + if (m_connection.get() == conn.get()){ + m_connection = nullptr; + } + } +} + + +HttpServer::ParsedRequest HttpServer::try_parse(std::string& buffer){ + ParsedRequest result; + + auto error = [&result](int status, const char* reason, std::vector> headers = {}){ + result.state = ParseState::ERROR_RESPONSE; + result.error_status = status; + result.error_reason = reason; + result.error_headers = std::move(headers); + }; + + // The header section ends at the blank line. "header_end" points at + // the "\r\n" that terminates the last header (or the request line + // when there are no headers). + const size_t header_end = buffer.find("\r\n\r\n"); + if (header_end == std::string::npos){ + if (buffer.size() > MAX_HEADER_SECTION_BYTES){ + error(400, "Malformed request."); + } + return result; // NEED_MORE: the headers have not fully arrived yet. + } + // The same limit applies to a complete header section: the + // in-progress check above only fires while the terminator is still + // missing, so a large section that arrives in one piece would + // otherwise sail through. + if (header_end + 4 > MAX_HEADER_SECTION_BYTES){ + error(400, "Malformed request."); + return result; + } + + const size_t line_end = buffer.find("\r\n"); + if (line_end == std::string::npos){ + error(400, "Malformed request."); + return result; + } + if (line_end > MAX_REQUEST_LINE_BYTES){ + error(400, "Request line too long."); + return result; + } + + // "METHOD SP TARGET SP VERSION" + const std::string line = buffer.substr(0, line_end); + const size_t sp1 = line.find(' '); + if (sp1 == std::string::npos || sp1 == 0){ + error(400, "Malformed request."); + return result; + } + std::string method = line.substr(0, sp1); + for (char c : method){ + if (c < 'A' || c > 'Z'){ + error(400, "Malformed request."); + return result; + } + } + const size_t sp2 = line.find(' ', sp1 + 1); + if (sp2 == std::string::npos){ + error(400, "Malformed request."); + return result; + } + std::string target = line.substr(sp1 + 1, sp2 - sp1 - 1); + if (target.empty()){ + error(400, "Malformed request."); + return result; + } + for (char c : target){ + if (c == ' ' || c == '\r' || c == '\n' || c == '\0'){ + error(400, "Malformed request."); + return result; + } + } + const std::string version = line.substr(sp2 + 1); + const bool http_1_1 = (version == "HTTP/1.1"); + const bool http_1_0 = (version == "HTTP/1.0"); + if (!http_1_1 && !http_1_0){ + error(505, "HTTP version not supported."); + return result; + } + + // Headers: [line_end + 2, header_end) + const size_t section_begin = line_end + 2; + std::string section; + if (header_end >= section_begin){ + section = buffer.substr(section_begin, header_end - section_begin); + } + size_t header_count = 0; + size_t pos = 0; + while (pos < section.size()){ + const size_t eol = section.find("\r\n", pos); + const size_t length = (eol == std::string::npos) ? section.size() - pos : eol - pos; + const std::string header_line = section.substr(pos, length); + + if (header_line.empty() || ++header_count > MAX_HEADER_COUNT){ + error(400, "Malformed request."); + return result; + } + + const size_t colon = header_line.find(':'); + if (colon == std::string::npos || colon == 0){ + error(400, "Malformed request."); + return result; + } + std::string name = header_line.substr(0, colon); + for (char c : name){ + if (!is_token_char(c)){ + error(400, "Malformed request."); + return result; + } + } + std::string value = trim_ows(header_line.substr(colon + 1)); + for (char c : value){ + if ((unsigned char)c < 0x20 || (unsigned char)c == 0x7F){ + error(400, "Malformed request."); + return result; + } + } + name = to_lower(std::move(name)); + if (name == "content-length" && result.request.headers.count("content-length") != 0){ + error(400, "Duplicate content-length."); + return result; + } + result.request.headers[std::move(name)] = std::move(value); + + if (eol == std::string::npos){ + break; + } + pos = eol + 2; + } + + // Special headers. + if (result.request.headers.count("transfer-encoding") != 0){ + // "Not Implemented" rather than "Length Required": chunked + // encoding is a (unsupported) representation of the body, not a + // missing length (RFC 9112). + error(501, "Chunked transfer encoding is not supported."); + return result; + } + + uint64_t content_length = 0; + { + const auto content_length_it = result.request.headers.find("content-length"); + if (content_length_it != result.request.headers.end()){ + const std::string& value = content_length_it->second; + if (value.empty() || value.size() > 10){ + error(400, "Malformed content-length."); + return result; + } + for (char c : value){ + if (c < '0' || c > '9'){ + error(400, "Malformed content-length."); + return result; + } + } + content_length = std::stoull(value); + if (content_length > MAX_BODY_BYTES){ + error(413, "Request body too large."); + return result; + } + } + } + + const auto connection_it = result.request.headers.find("connection"); + if (connection_it != result.request.headers.end() && + to_lower(connection_it->second).find("close") != std::string::npos + ){ + result.connection_close_requested = true; + } + + const auto expect_it = result.request.headers.find("expect"); + if (expect_it != result.request.headers.end() && + to_lower(expect_it->second).find("100-continue") != std::string::npos + ){ + result.expect_continue = true; + } + + const size_t consumed = header_end + 4 + (size_t)content_length; + if (buffer.size() < consumed){ + // NEED_BODY: the headers are complete and valid, the body has + // not fully arrived yet. In this state the buffer holds nothing + // but the bytes of this one request ("consumed" is at most a + // header section plus the body cap), so the buffer cannot exceed + // MAX_BUFFER_BYTES here; the overall cap is enforced when data + // is appended instead. + result.state = ParseState::NEED_BODY; + return result; + } + + result.state = ParseState::READY; + result.request.method = std::move(method); + const size_t query = target.find('?'); + result.request.path = target.substr(0, query); + if (query != std::string::npos){ + result.request.query = target.substr(query + 1); + } + result.request.body = buffer.substr(header_end + 4, content_length); + buffer.erase(0, consumed); + result.http_1_1 = http_1_1; + return result; +} + + +void HttpServer::end_connection( + const std::shared_ptr& conn, + AbstractServerSocket* socket +){ + conn->dead.store(true, std::memory_order_release); + conn->cv.notify_all(); + // Close the socket's connection only while it still belongs to this + // connection: by the time a task notices its own connection died, + // the socket may already be serving the next client, and closing + // unconditionally would kill that client's connection instead. + { + std::lock_guard lg(m_lock); + if (m_connection.get() != conn.get()){ + return; + } + } + socket->close_connection(); +} + + +bool HttpServer::write_raw( + const std::shared_ptr& conn, + AbstractServerSocket* socket, + const std::string& bytes +){ + // "conn->socket_id" is the socket's connection token when this + // connection was accepted: if the socket has moved on to the next + // client since then, the send is refused instead of being delivered + // into the new client's socket. + const size_t sent = socket->send(bytes.data(), bytes.size(), WRITE_TIMEOUT, conn->socket_id); + if (sent != bytes.size()){ + // The peer went away, or the write stalled: the connection is + // unusable either way. + end_connection(conn, socket); + return false; + } + return true; +} + + +bool HttpServer::write_response( + const std::shared_ptr& conn, + AbstractServerSocket* socket, + const Response& response, + bool keep_alive +){ + return write_raw(conn, socket, format_response(response, keep_alive)); +} + + + +} diff --git a/Common/Cpp/HttpServer/HttpServer.h b/Common/Cpp/HttpServer/HttpServer.h new file mode 100644 index 0000000000..a770ea88fb --- /dev/null +++ b/Common/Cpp/HttpServer/HttpServer.h @@ -0,0 +1,169 @@ +/* HTTP Server + * + * From: https://github.com/PokemonAutomation/ + * + * The "AbstractServerSocket" implementation of "AbstractHttpServer": a + * minimal HTTP/1.1 server on the platform's default server socket, for a + * single local client that issues one request at a time. The request/ + * response types, the handler contract, and the threading rules are in + * "AbstractHttpServer.h"; this file documents the implementation. + * + */ + +#ifndef PokemonAutomation_HttpServer_H +#define PokemonAutomation_HttpServer_H + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/AsyncTask.h" +#include "Common/Cpp/Concurrency/ConditionVariable.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/HttpServer/AbstractHttpServer.h" +#include "Common/Cpp/Sockets/AbstractServerSocket.h" + +namespace PokemonAutomation{ + +class ThreadPool; + + + +class HttpServer : public AbstractHttpServer, private AbstractServerSocket::Listener{ +public: + // The interface's types, re-exported under this class's name. + using AbstractHttpServer::Request; + using AbstractHttpServer::Response; + using AbstractHttpServer::HandlerContext; + using AbstractHttpServer::Handler; + using AbstractHttpServer::Listener; + + +public: + explicit HttpServer(ThreadPool& thread_pool); + ~HttpServer(); + + virtual void start(const std::string& address, uint16_t port, Handler handler) override; + virtual void stop() override; + virtual bool is_running() const override; + + +private: + // AbstractServerSocket::Listener. + virtual void on_listen_finished(const std::string& error_message) override; + virtual void on_connect() override; + virtual void on_connect_rejected() override; + virtual void on_receive_data(const void* data, size_t bytes) override; + virtual void on_disconnect(const std::string& error_message) override; + + +private: + // One request that is waiting to be parsed out of the buffer. + enum class ParseState{ + NEED_MORE, + NEED_BODY, + READY, + // Named "ERROR_RESPONSE" rather than "ERROR" because "ERROR" is + // a macro on Windows (winerror.h). + ERROR_RESPONSE, + }; + + struct ParsedRequest{ + ParseState state = ParseState::NEED_MORE; + + // ERROR_RESPONSE + int error_status = 400; + std::string error_reason; + std::vector> error_headers; + + // READY / NEED_BODY + Request request; + bool http_1_1 = true; + bool connection_close_requested = false; + bool expect_continue = false; + }; + + // Per-connection state. The connection task is the only thread that + // parses or writes; "on_receive_data"/"on_disconnect" only append + // data and flip flags. + struct Connection{ + Mutex lock; + ConditionVariable cv; + std::string buffer; + std::atomic dead{false}; + + // The server socket's connection_id() when this connection was + // accepted. Passed to every send(), so a write that outlives the + // connection is refused instead of reaching the next client. + uint64_t socket_id = 0; + }; + + // Runs on a thread from the thread pool, one instance per + // connection. + void connection_task( + const std::shared_ptr& conn, + AbstractServerSocket* socket + ); + + // Parses one complete request out of "buffer", removing it on + // success. Must be called with "conn->lock" held. + static ParsedRequest try_parse(std::string& buffer); + + // Marks the connection dead and closes it at the socket level. No-op + // if the socket is no longer connected to this client. + void end_connection( + const std::shared_ptr& conn, + AbstractServerSocket* socket + ); + + // Writes "bytes" to the connection completely; returns false (and + // ends the connection) if the peer went away or the write stalled. + bool write_raw( + const std::shared_ptr& conn, + AbstractServerSocket* socket, + const std::string& bytes + ); + + // Formats and writes the final response. + bool write_response( + const std::shared_ptr& conn, + AbstractServerSocket* socket, + const Response& response, + bool keep_alive + ); + + // The standard reason phrase for "status", or "Unknown Status". + static const char* status_reason(int status); + + // Builds the wire format of a response. + static std::string format_response(const Response& response, bool keep_alive); + + +private: + ThreadPool& m_thread_pool; + + // Guards m_socket, m_connection, and m_handler. Never held across a + // blocking wait, a task destruction, or a socket destruction. + mutable Mutex m_lock; + std::unique_ptr m_socket; + std::shared_ptr m_connection; + Handler m_handler; + + // Guards m_tasks. Never held while an AsyncTask destructor runs: + // the destructor blocks until the task finishes, and the task's + // final cleanup takes m_lock. + Mutex m_task_lock; + std::vector m_tasks; + + // Set by "stop()"; checked by the connection tasks between requests + // and by "HandlerContext". + std::atomic m_stop_requested{false}; +}; + + + +} +#endif diff --git a/Common/Cpp/HttpServer/HttpServer_Tests.cpp b/Common/Cpp/HttpServer/HttpServer_Tests.cpp new file mode 100644 index 0000000000..65b4ca31f9 --- /dev/null +++ b/Common/Cpp/HttpServer/HttpServer_Tests.cpp @@ -0,0 +1,1372 @@ +/* HTTP Server Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Loopback tests that run a real "HttpServer" (the platform's default + * server socket backend) on 127.0.0.1 and drive it with raw + * "ClientSocket" clients. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/AsyncTask.h" +#include "Common/Cpp/Concurrency/ConditionVariable.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Concurrency/ThreadPool.h" +#include "Common/Cpp/HttpServer/HttpServer.h" +#include "Common/Cpp/Sockets/ClientSocket.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "HttpServer_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + + +struct Waiter{ + Mutex lock; + ConditionVariable cv; + + bool wait_for(std::chrono::milliseconds timeout, const std::function& pred){ + std::unique_lock lg(lock); + return cv.wait_for(lg, timeout, pred); + } + + void notify(){ + std::lock_guard lg(lock); + cv.notify_all(); + } +}; + + +// Long enough that a moderately loaded machine can make it, short enough +// that a real regression is still reported within the CI timeouts. +constexpr std::chrono::seconds HTTP_TEST_WAIT{60}; + + +// A handler gate: handlers notify "started" when they begin, and block on +// "wait_for_release()" until the test releases them. +struct HandlerGate{ + Waiter started; + + Mutex lock; + ConditionVariable cv; + std::atomic released{false}; + + void wait_for_release(std::chrono::milliseconds timeout){ + std::unique_lock lg(lock); + cv.wait_for(lg, timeout, [this]{ + return released.load(std::memory_order_relaxed); + }); + } + + void release(){ + released.store(true, std::memory_order_release); + std::lock_guard lg(lock); + cv.notify_all(); + } +}; + + +// A raw TCP client that accumulates everything it receives. +// "received" and the error strings are protected by "waiter.lock": the +// socket's receive thread writes them, the test thread reads them. +struct TestClient{ + Waiter waiter; + std::string received; + std::atomic connected{0}; + std::atomic disconnected{0}; + std::string connect_error; + std::string disconnect_error; + + struct Listener : AbstractClientSocket::Listener{ + TestClient* client; + + virtual void on_connect_finished(const std::string& error_message) override{ + { + std::lock_guard lg(client->waiter.lock); + client->connect_error = error_message; + } + client->connected++; + client->waiter.notify(); + } + + virtual void on_receive_data(const void* data, size_t bytes) override{ + { + std::lock_guard lg(client->waiter.lock); + client->received.append((const char*)data, bytes); + } + client->waiter.notify(); + } + + virtual void on_disconnect(const std::string& error_message) override{ + { + std::lock_guard lg(client->waiter.lock); + client->disconnect_error = error_message; + } + client->disconnected++; + client->waiter.notify(); + } + } listener_impl; + + ClientSocket socket; + + explicit TestClient(ThreadPool& thread_pool) + : socket(thread_pool) + { + listener_impl.client = this; + socket.add_listener(listener_impl); + } + + bool connect(uint16_t port){ + socket.connect("127.0.0.1", port); + return waiter.wait_for( + HTTP_TEST_WAIT, + [this]{ return connected.load() >= 1; } + ); + } + + bool send_all(const std::string& bytes){ + size_t total = 0; + while (total < bytes.size()){ + const size_t sent = socket.send(bytes.data() + total, bytes.size() - total); + if (sent == 0){ + return false; + } + total += sent; + } + return true; + } + + bool wait_for_bytes(size_t n){ + // The predicate runs with "waiter.lock" held, so it may read + // "received" directly. + return waiter.wait_for( + HTTP_TEST_WAIT, + [this, n]{ return received.size() >= n; } + ); + } + + // A consistent copy of everything received so far, for assertions. + std::string received_snapshot(){ + std::lock_guard lg(waiter.lock); + return received; + } +}; + + +struct TestServerListener : HttpServer::Listener{ + Waiter waiter; + std::atomic listen_finished{0}; + // Protected by "waiter.lock" (written by the socket's thread). + std::string last_listen_error; + std::atomic connected{0}; + std::atomic rejected{0}; + std::atomic disconnected{0}; + + virtual void on_listen_finished(const std::string& error_message) override{ + { + std::lock_guard lg(waiter.lock); + last_listen_error = error_message; + } + listen_finished++; + waiter.notify(); + } + + std::string last_listen_error_snapshot(){ + std::lock_guard lg(waiter.lock); + return last_listen_error; + } + + virtual void on_connect() override{ + connected++; + waiter.notify(); + } + + virtual void on_connect_rejected() override{ + rejected++; + waiter.notify(); + } + + virtual void on_disconnect(const std::string& error_message) override{ + disconnected++; + waiter.notify(); + } +}; + + +class TestHttpServer : public UnitTest{ +public: + TestHttpServer(ThreadPool& thread_pool, std::string name) + : UnitTest(std::move(name)) + , m_thread_pool(thread_pool) + { + m_threads = 2; + } + + +protected: + // Starts the server with "handler" on a free loopback port, waiting + // for the listen to finish. "handler" is copied on each attempt. + bool start_server(HttpServer& server, TestServerListener& listener, HttpServer::Handler handler, uint16_t& port) const{ + for (int attempt = 0; attempt < 8; attempt++){ + uint16_t candidate; + { + std::lock_guard lg(m_rng_lock); + candidate = (uint16_t)(40000 + m_rng() % 20000); + } + const int finished_before = listener.listen_finished.load(); + server.start("127.0.0.1", candidate, handler); + if (!listener.waiter.wait_for( + HTTP_TEST_WAIT, + [finished_before, &listener]{ + return listener.listen_finished.load() > finished_before; + } + )){ + server.stop(); + continue; + } + if (listener.last_listen_error_snapshot().empty()){ + port = candidate; + return true; + } + // A failed listen leaves the server "started but not + // listening"; stop() cleans that up before the next attempt. + server.stop(); + } + return false; + } + +protected: + ThreadPool& m_thread_pool; + +private: + mutable Mutex m_rng_lock; + mutable std::mt19937 m_rng{std::random_device{}()}; +}; + + +// A POST request is parsed (method, path, query, headers, body) and the +// response carries the extra headers and the exact body; the connection +// stays keep-alive. +class Test_Http_PostEcho : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request& request, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.headers.push_back({"X-Test", "yes"}); + response.body = request.path + "|" + request.query + "|" + request.header("X-Client") + "|" + request.body; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 1; } + )){ + error = "The server did not call on_connect"; + }else if (!client.send_all( + "POST /mcp?session=abc HTTP/1.1\r\n" + "Host: 127.0.0.1\r\n" + "X-Client: unit-test\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "X-Test: yes\r\n" + "Content-Length: 32\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "/mcp|session=abc|unit-test|hello"; + if (!client.wait_for_bytes(expected.size())){ + error = "The client did not receive the full response"; + }else if (client.received_snapshot() != expected){ + error = "Bad response:\n" + client.received_snapshot(); + } + } + // The client goes out of scope here and closes the socket. + } + + if (error.empty() && + !server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.disconnected.load() >= 1; } + ) + ){ + error = "The server did not call on_disconnect"; + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// Two requests sent back to back on one connection (the second is already +// buffered while the first is being answered): both must be answered, in +// order. +class Test_Http_PipelinedKeepAlive : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request& request, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = "r" + request.body; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST /a HTTP/1.1\r\n" + "Content-Length: 3\r\n" + "\r\n" + "one" + "POST /b HTTP/1.1\r\n" + "Content-Length: 3\r\n" + "\r\n" + "two" + )){ + error = "The client did not send the requests"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "rone" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "rtwo"; + if (!client.wait_for_bytes(expected.size())){ + error = "The client did not receive both responses (got " + std::to_string(client.received_snapshot().size()) + " of " + std::to_string(expected.size()) + " bytes)"; + }else if (client.received_snapshot() != expected){ + error = "Bad response:\n" + client.received_snapshot(); + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// "Connection: close" in the request: the response says so, the server +// closes the connection, and it keeps serving new clients. +class Test_Http_ConnectionClose : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = "done"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "Connection: close\r\n" + "\r\n" + "x" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: close\r\n" + "\r\n" + "done"; + if (!client.wait_for_bytes(expected.size())){ + error = "The client did not receive the full response"; + }else if (client.received_snapshot() != expected){ + error = "Bad response:\n" + client.received_snapshot(); + }else if (!client.waiter.wait_for( + HTTP_TEST_WAIT, + [&client]{ return client.disconnected.load() >= 1; } + )){ + error = "The server did not close the connection"; + } + } + } + + if (error.empty()){ + // The server must keep listening for a new client. + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not reconnect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "y" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 4\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "done"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "The reconnected client did not get a good response:\n" + client.received_snapshot(); + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// A request line that is not "METHOD SP TARGET SP VERSION" gets a 400 and +// a closed connection; the server recovers and serves a new client. +class Test_Http_BadRequest : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all("GARBAGE\r\n\r\n")){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 400 Bad Request\r\n" + "Content-Length: 19\r\n" + "Connection: close\r\n" + "\r\n" + "Malformed request.\n"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "Bad error response:\n" + client.received_snapshot(); + } + } + } + + if (error.empty()){ + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not reconnect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 0\r\n" + "\r\n" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "The server did not recover:\n" + client.received_snapshot(); + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// Chunked request bodies are not supported: 501 and a closed connection, +// and the handler is never called. +class Test_Http_ChunkedRejected : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + std::atomic handler_called{false}; + auto handler = [&handler_called](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + handler_called = true; + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 501 Not Implemented\r\n" + "Content-Length: 44\r\n" + "Connection: close\r\n" + "\r\n" + "Chunked transfer encoding is not supported.\n"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "Bad error response:\n" + client.received_snapshot(); + } + } + } + + if (error.empty() && handler_called.load()){ + error = "The handler was called for a rejected request"; + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// A content-length above the body limit is rejected with 413 before the +// (never sent) body arrives, and the handler is never called. +class Test_Http_BodyTooLarge : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + std::atomic handler_called{false}; + auto handler = [&handler_called](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + handler_called = true; + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 2097152\r\n" + "\r\n" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 413 Payload Too Large\r\n" + "Content-Length: 24\r\n" + "Connection: close\r\n" + "\r\n" + "Request body too large.\n"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "Bad error response:\n" + client.received_snapshot(); + } + } + } + + if (error.empty() && handler_called.load()){ + error = "The handler was called for a rejected request"; + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// A complete header section over the size limit is rejected with 400, +// even though it arrives in one piece (the in-progress limit check only +// fires while the terminator is still missing). +class Test_Http_HeadersTooLarge : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + std::atomic handler_called{false}; + auto handler = [&handler_called](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + handler_called = true; + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + // 50 headers of about 1.5 KiB each: about 75 KiB of headers - + // under the 100-header count limit, over the 64 KiB size limit. + std::string request = "POST / HTTP/1.1\r\n"; + for (int i = 0; i < 50; i++){ + request += "X-Filler-"; + request += std::to_string(i); + request += ": "; + request += std::string(1490, 'a'); + request += "\r\n"; + } + request += "\r\n"; + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all(request)){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 400 Bad Request\r\n" + "Content-Length: 19\r\n" + "Connection: close\r\n" + "\r\n" + "Malformed request.\n"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "Bad error response:\n" + client.received_snapshot(); + } + } + } + + if (error.empty() && handler_called.load()){ + error = "The handler was called for a rejected request"; + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// A connection that sends more data than the receive buffer cap (about +// 2 MiB) is closed without a response, and the server keeps listening. +class Test_Http_BufferCap : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + std::atomic handler_calls{0}; + auto handler = [&handler_calls](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + handler_calls++; + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else{ + // A never-completing request far larger than the buffer + // cap: the server closes the connection without a + // response. (The send may fail partway through once the + // server has hung up; that is expected.) + const std::string flood(3u << 20, 'A'); + client.send_all(flood); + if (!client.waiter.wait_for( + HTTP_TEST_WAIT, + [&client]{ return client.disconnected.load() >= 1; } + )){ + error = "The server did not close the over-cap connection"; + } + } + } + + if (error.empty() && handler_calls.load() != 0){ + error = "The handler was called for a capped connection"; + } + + if (error.empty()){ + // The server must still be listening afterwards. + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect after the cap test"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 0\r\n" + "\r\n" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "The server did not answer after the cap test:\n" + client.received_snapshot(); + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// "Expect: 100-continue" is answered with "100 Continue" as soon as the +// headers are validated - before the body arrives, which is what a +// client waiting for the interim response is waiting for - and the +// final response carries the body. +class Test_Http_ExpectContinue : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request& request, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = request.body; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 5\r\n" + "Expect: 100-continue\r\n" + "\r\n" + )){ + error = "The client did not send the headers"; + }else{ + const std::string continue_response = "HTTP/1.1 100 Continue\r\n\r\n"; + if (!client.wait_for_bytes(continue_response.size()) + || client.received_snapshot().compare(0, continue_response.size(), continue_response) != 0 + ){ + error = "The server did not answer the interim response:\n" + client.received_snapshot(); + }else if (!client.send_all("hello")){ + error = "The client did not send the body"; + }else{ + const std::string expected = + "HTTP/1.1 100 Continue\r\n\r\n" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "hello"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "Bad response:\n" + client.received_snapshot(); + } + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// The client disconnects while the handler is still running: the handler +// sees it via "HandlerContext::connection_alive()", its response is +// discarded, and the server keeps serving new clients. +class Test_Http_DisconnectDuringHandler : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + HandlerGate gate; + std::atomic call_count{0}; + std::atomic first_call_alive{false}; + std::atomic first_call_saw_disconnect{false}; + + auto handler = [&gate, &call_count, &first_call_alive, &first_call_saw_disconnect]( + const HttpServer::Request&, + HttpServer::Response& response, + HttpServer::HandlerContext& context + ){ + const int call = call_count.fetch_add(1) + 1; + if (call == 1){ + first_call_alive = context.connection_alive(); + gate.started.notify(); + gate.wait_for_release(HTTP_TEST_WAIT); + first_call_saw_disconnect = !context.connection_alive(); + } + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "x" + )){ + error = "The client did not send the request"; + }else if (!gate.started.wait_for( + HTTP_TEST_WAIT, + [&first_call_alive]{ return first_call_alive.load(); } + )){ + error = "The handler did not start"; + } + // The client goes out of scope here and closes the socket, + // while the handler is still blocked on the gate. + } + + if (error.empty() && + !server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.disconnected.load() >= 1; } + ) + ){ + error = "The server did not call on_disconnect"; + } + + if (error.empty()){ + gate.release(); + } + + if (error.empty()){ + // The server must keep serving new clients. + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not reconnect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "y" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "The server did not recover:\n" + client.received_snapshot(); + } + } + } + + if (error.empty()){ + // Wait for the first handler to have recorded its observation. + if (!gate.started.wait_for( + HTTP_TEST_WAIT, + [&first_call_saw_disconnect, &call_count]{ + return call_count.load() >= 1 && first_call_saw_disconnect.load(); + } + )){ + error = "The handler did not record the disconnect"; + } + } + + if (error.empty()){ + if (!first_call_alive.load()){ + error = "The handler did not see the connection as alive on entry"; + }else if (!first_call_saw_disconnect.load()){ + error = "The handler did not see the disconnect via connection_alive()"; + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// The client disconnects while the handler is still running, and a new +// client connects before the handler returns: the old task must discard +// its response instead of writing it into the new client's socket. +class Test_Http_ReconnectWhileHandlerRunning : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + HandlerGate gate; + std::atomic first_call_alive{false}; + + auto handler = [&gate, &first_call_alive]( + const HttpServer::Request& request, + HttpServer::Response& response, + HttpServer::HandlerContext& context + ){ + if (request.path == "/a"){ + first_call_alive = context.connection_alive(); + gate.started.notify(); + gate.wait_for_release(HTTP_TEST_WAIT); + } + response.status = 200; + response.body = (request.path == "/a") ? "A" : "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + TestClient client_a(m_thread_pool); + TestClient client_b(m_thread_pool); + { + if (!client_a.connect(port)){ + error = "The first client did not connect"; + }else if (!client_a.send_all( + "POST /a HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "x" + )){ + error = "The first client did not send the request"; + }else if (!gate.started.wait_for( + HTTP_TEST_WAIT, + [&first_call_alive]{ return first_call_alive.load(); } + )){ + error = "The handler did not start"; + } + } + + if (error.empty()){ + // Disconnect the first client (closing its socket while it is + // still in scope, so its connection task keeps running) and + // wait for the server to report the disconnect. + client_a.socket.close(); + if (!server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.disconnected.load() >= 1; } + )){ + error = "The server did not report the first client's disconnect"; + } + } + + if (error.empty()){ + // Connect the second client while the first connection's + // handler is still blocked on the gate. Waiting for the + // second on_connect guarantees the second connection has + // replaced the first one in the server by the time the gate + // is released. + if (!client_b.connect(port)){ + error = "The second client did not connect"; + }else if (!server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 2; } + )){ + error = "The server did not accept the second client"; + } + } + + if (error.empty()){ + gate.release(); + } + + if (error.empty()){ + // The second client's first bytes must be its own response: + // the first client's late response was discarded, not + // written into this socket. + if (!client_b.send_all( + "POST /b HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "y" + )){ + error = "The second client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + if (!client_b.wait_for_bytes(expected.size()) || client_b.received_snapshot() != expected){ + error = "The second client received a stray response:\n" + client_b.received_snapshot(); + } + } + } + + if (error.empty() && !first_call_alive.load()){ + error = "The handler did not see the connection as alive on entry"; + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// "stop()" while the handler is still running blocks until the handler +// returns, and "start()" works afterwards. +class Test_Http_StopDuringHandler : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + HandlerGate gate; + std::atomic handler_started{false}; + std::atomic saw_disconnect_on_stop{false}; + + auto handler = [&gate, &handler_started, &saw_disconnect_on_stop]( + const HttpServer::Request&, + HttpServer::Response& response, + HttpServer::HandlerContext& context + ){ + handler_started = true; + gate.started.notify(); + gate.wait_for_release(HTTP_TEST_WAIT); + // Then stay alive while the connection is alive: the pattern + // a long-running tool call uses. stop() marks the connection + // dead before it waits for this handler (the socket's + // close_connection() close suppresses the disconnect + // callback), so the loop must terminate - without that mark + // this would spin forever and stop() would hang. + while (context.connection_alive()){ + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + saw_disconnect_on_stop = !context.connection_alive(); + response.status = 200; + response.body = "late"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 1\r\n" + "\r\n" + "x" + )){ + error = "The client did not send the request"; + }else if (!gate.started.wait_for( + HTTP_TEST_WAIT, + [&handler_started]{ return handler_started.load(); } + )){ + error = "The handler did not start"; + } + } + + if (error.empty()){ + // stop() blocks until the in-flight handler returns. Release + // the handler from a pool thread so this thread can call it. + // The handle is kept (dispatch is [[nodiscard]]); its + // destructor waits for the trivial release task to finish. + AsyncTask releaser_task = m_thread_pool.dispatch([&gate]{ + gate.release(); + }); + server.stop(); + if (server.is_running()){ + error = "The server is still running after stop()"; + } + } + + if (error.empty() && !saw_disconnect_on_stop.load()){ + error = "The handler did not see the connection as gone after stop()"; + } + + if (error.empty()){ + // Start again (on a fresh port) and serve a fresh request. + auto simple_handler = [](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = "ok"; + }; + if (!start_server(server, server_listener, simple_handler, port)){ + error = "Failed to restart the server"; + }else{ + TestClient client(m_thread_pool); + if (!client.connect(port)){ + error = "The client did not connect to the restarted server"; + }else if (!client.send_all( + "POST / HTTP/1.1\r\n" + "Content-Length: 0\r\n" + "\r\n" + )){ + error = "The client did not send the request"; + }else{ + const std::string expected = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + if (!client.wait_for_bytes(expected.size()) || client.received_snapshot() != expected){ + error = "The restarted server did not answer:\n" + client.received_snapshot(); + } + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + +// While one client is connected, a second one is rejected. +class Test_Http_RejectSecondClient : public TestHttpServer{ +public: + using TestHttpServer::TestHttpServer; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + TestServerListener server_listener; + HttpServer server(m_thread_pool); + server.add_listener(server_listener); + + auto handler = [](const HttpServer::Request&, HttpServer::Response& response, HttpServer::HandlerContext&){ + response.status = 200; + response.body = "ok"; + }; + + uint16_t port; + if (!start_server(server, server_listener, handler, port)){ + return "Failed to start the server"; + } + + std::string error; + { + TestClient first(m_thread_pool); + if (!first.connect(port)){ + error = "The first client did not connect"; + }else if (!server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 1; } + )){ + error = "The server did not call on_connect"; + }else{ + TestClient second(m_thread_pool); + second.connect(port); // May complete at the TCP level; the server closes it. + if (!server_listener.waiter.wait_for( + HTTP_TEST_WAIT, + [&server_listener]{ return server_listener.rejected.load() >= 1; } + )){ + error = "The server did not call on_connect_rejected"; + } + } + } + + server.stop(); + if (!error.empty()){ + return error; + } + return true; + } +}; + + + +} // anonymous namespace + + +void add_tests_HttpServer(UnitTestDatabase& database, ThreadPool& thread_pool){ + database.add( + thread_pool, "HttpServer - POST request/response with headers, query and body" + ); + database.add( + thread_pool, "HttpServer - Two pipelined requests on one keep-alive connection" + ); + database.add( + thread_pool, "HttpServer - Connection: close and reconnection" + ); + database.add( + thread_pool, "HttpServer - Malformed request line" + ); + database.add( + thread_pool, "HttpServer - Chunked request bodies are rejected" + ); + database.add( + thread_pool, "HttpServer - Oversized content-length is rejected" + ); + database.add( + thread_pool, "HttpServer - An oversized header section is rejected" + ); + database.add( + thread_pool, "HttpServer - A connection over the buffer cap is closed" + ); + database.add( + thread_pool, "HttpServer - Expect: 100-continue" + ); + database.add( + thread_pool, "HttpServer - Disconnect while the handler is running" + ); + database.add( + thread_pool, "HttpServer - Reconnect while the previous handler is still running" + ); + database.add( + thread_pool, "HttpServer - stop() while the handler is running" + ); + database.add( + thread_pool, "HttpServer - A second client is rejected" + ); +} + + + +} diff --git a/Common/Cpp/HttpServer/HttpServer_Tests.h b/Common/Cpp/HttpServer/HttpServer_Tests.h new file mode 100644 index 0000000000..6fe6c25cb1 --- /dev/null +++ b/Common/Cpp/HttpServer/HttpServer_Tests.h @@ -0,0 +1,28 @@ +/* HTTP Server Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_HttpServer_Tests_H +#define PokemonAutomation_HttpServer_Tests_H + +namespace PokemonAutomation{ + +class ThreadPool; +class UnitTestDatabase; + + +// Loopback tests that run a real "HttpServer" (the platform's default +// server socket backend) on 127.0.0.1 and drive it with raw "ClientSocket" +// clients: request/response parsing, keep-alive, pipelining, +// "Connection: close", "Expect: 100-continue", error responses, and the +// lifecycle (disconnect during a handler, "stop()" during a handler, +// restart, second-client rejection). `thread_pool` runs the server and +// the test workers; it must be a long-lived pool (e.g. +// `GlobalThreadPools::unlimited_normal()`). +void add_tests_HttpServer(UnitTestDatabase& database, ThreadPool& thread_pool); + + +} +#endif diff --git a/Common/Cpp/Options/BatchOption.cpp b/Common/Cpp/Options/BatchOption.cpp index a3747e0ad6..d9c309a9a9 100644 --- a/Common/Cpp/Options/BatchOption.cpp +++ b/Common/Cpp/Options/BatchOption.cpp @@ -95,6 +95,9 @@ FixedLimitVector BatchOption::options() const{ } return ret; } +const std::vector>& keyed_options(const BatchOption& batch){ + return batch.m_data->m_options; +} diff --git a/Common/Cpp/Options/BatchOption.h b/Common/Cpp/Options/BatchOption.h index c8ac257570..ec9b21a131 100644 --- a/Common/Cpp/Options/BatchOption.h +++ b/Common/Cpp/Options/BatchOption.h @@ -7,6 +7,9 @@ #ifndef PokemonAutomation_Options_BatchOption_H #define PokemonAutomation_Options_BatchOption_H +#include +#include +#include #include "Common/Cpp/Containers/Pimpl.h" #include "Common/Cpp/Containers/FixedLimitVector.h" #include "ConfigOption.h" @@ -42,6 +45,8 @@ class BatchOption : public UiState{ bool horizontal() const; FixedLimitVector options() const; + friend const std::vector>& keyed_options(const BatchOption& batch); + private: struct Data; @@ -51,5 +56,11 @@ class BatchOption : public UiState{ +// Each option of `batch` with the key it is saved under in to_json(), in the +// order they were added. Static options have an empty key. +const std::vector>& keyed_options(const BatchOption& batch); + + + } #endif diff --git a/Common/Cpp/Sockets/AbstractServerSocket.h b/Common/Cpp/Sockets/AbstractServerSocket.h index 4139a0f120..1c2da259ef 100644 --- a/Common/Cpp/Sockets/AbstractServerSocket.h +++ b/Common/Cpp/Sockets/AbstractServerSocket.h @@ -13,6 +13,7 @@ #define PokemonAutomation_Sockets_AbstractServerSocket_H #include +#include #include #include "Common/Cpp/ListenerSet.h" @@ -58,6 +59,18 @@ class AbstractServerSocket{ return m_state.load(std::memory_order_relaxed); } + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. Capture it when a write + // starts and pass it to "send()", so that a write that outlives its + // connection (the peer disconnected and a new client was accepted + // in the meantime) refuses to reach the new client's socket. + // Implementations keep the underlying counter monotonic (never + // reset it when a connection closes, or a queued write for the old + // connection could match the next connection's id); 0 is only the + // "no connection is active" report, never a counter value. + virtual uint64_t connection_id() = 0; + // Binds to "address" and listens on "port". // Use "0.0.0.0" to accept connections on all interfaces. virtual void start(const std::string& address, uint16_t port) = 0; @@ -69,7 +82,10 @@ class AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + // The connection_id() captured when this write started. 0 skips + // the check: the call applies to whatever connection is active. + uint64_t connection = 0 ) = 0; diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp index 75b2a14ead..ad38549eb6 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.cpp +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.cpp @@ -83,6 +83,14 @@ void ServerSocket_POSIX::start(const std::string& address, uint16_t port){ } } +uint64_t ServerSocket_POSIX::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_POSIX::close_connection(){ int socket; { @@ -159,16 +167,16 @@ void ServerSocket_POSIX::close_and_wait() noexcept{ } } -size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ int socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_client - : -1; - } - if (socket == -1){ - return 0; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_client; } const char* ptr = (const char*)data; @@ -184,7 +192,12 @@ size_t ServerSocket_POSIX::send(const void* data, size_t bytes, std::chrono::mil while (bytes > 0){ { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::CONNECTED){ + // The connection may have been replaced (or closed) since the + // last iteration: in that case this write must stop rather + // than reach the new client's socket. + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ return sent; } } @@ -359,6 +372,7 @@ void ServerSocket_POSIX::accept_thread_loop(std::string address, uint16_t port, fcntl(client_socket, F_SETFL, O_NONBLOCK); m_client = client_socket; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; } diff --git a/Common/Cpp/Sockets/ServerSocket_POSIX.h b/Common/Cpp/Sockets/ServerSocket_POSIX.h index 6786a631fc..f08b83ca68 100644 --- a/Common/Cpp/Sockets/ServerSocket_POSIX.h +++ b/Common/Cpp/Sockets/ServerSocket_POSIX.h @@ -42,9 +42,15 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + private: @@ -87,6 +93,12 @@ class ServerSocket_POSIX final : public AbstractServerSocket{ int m_client = -1; bool m_close_requested = false; + // A connection token, incremented (under m_lock) on every accept. + // send() compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // One task per start() call. start() must never destroy the previous // task's AsyncTask here: its destructor blocks until the task has // finished, and a task from an earlier attempt may still be running diff --git a/Common/Cpp/Sockets/ServerSocket_Qt.cpp b/Common/Cpp/Sockets/ServerSocket_Qt.cpp index fd48c72001..359a45b1f1 100644 --- a/Common/Cpp/Sockets/ServerSocket_Qt.cpp +++ b/Common/Cpp/Sockets/ServerSocket_Qt.cpp @@ -174,13 +174,16 @@ void ServerSocket_Qt::close_and_wait() noexcept{ m_close_requested = false; } -size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ QTcpSocket* socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_socket - : nullptr; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_socket; } if (socket == nullptr){ return 0; @@ -188,6 +191,7 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis QSharedPointer send_data = QSharedPointer::create(); send_data->packet.assign((const char*)data, bytes); + send_data->connection = connection; // "time_point::max()" (the infinite default) is never reached; using // it also avoids overflowing "now() + timeout". send_data->deadline = @@ -213,7 +217,8 @@ size_t ServerSocket_Qt::send(const void* data, size_t bytes, std::chrono::millis std::lock_guard lg(m_lock); registered = m_state.load(std::memory_order_relaxed) == State::CONNECTED - && m_socket != nullptr; + && m_socket != nullptr + && (connection == 0 || m_connection_id == connection); if (registered){ m_inflight_sends.push_back(send_data); } @@ -310,6 +315,7 @@ void ServerSocket_Qt::on_new_connection(){ }else{ m_socket = sock; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; rejected = false; @@ -434,11 +440,28 @@ void ServerSocket_Qt::on_close_connection(QSharedPointer comple completion->cv.notify_all(); } +uint64_t ServerSocket_Qt::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_Qt::on_send(QSharedPointer send_data){ QTcpSocket* sock; { std::lock_guard lg(m_lock); - sock = m_socket; + if (send_data->connection != 0 + && m_connection_id != send_data->connection + ){ + // The connection this send was queued for was replaced (or + // torn down) while it was waiting in the queue: the write + // loop must not reach into the new connection's socket. + sock = nullptr; + }else{ + sock = m_socket; + } } { diff --git a/Common/Cpp/Sockets/ServerSocket_Qt.h b/Common/Cpp/Sockets/ServerSocket_Qt.h index 0bcf70c0a6..370152e98c 100644 --- a/Common/Cpp/Sockets/ServerSocket_Qt.h +++ b/Common/Cpp/Sockets/ServerSocket_Qt.h @@ -54,6 +54,12 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ // "close_and_wait()"/"close_connection()" cancel the send. bool done = false; + // The connection_id() the send was queued for; 0 skips the + // check. "on_send" refuses to write if the connection was + // replaced while the send was queued, so it cannot reach the + // new client's socket. + uint64_t connection = 0; + Mutex lock; ConditionVariable cv; }; @@ -86,9 +92,15 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + signals: void internal_send(QSharedPointer send_data); @@ -142,6 +154,12 @@ class ServerSocket_Qt final : public QObject, public AbstractServerSocket{ bool m_close_requested = false; bool m_starting = false; + // A connection token, incremented (under m_lock) on every accept. + // "send()" compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // The "send()" calls that are queued on the event thread but not // finished yet, so that "close_and_wait()"/"close_connection()" can // cancel diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp index 55889f4b38..066a76ab24 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.cpp @@ -94,6 +94,14 @@ void ServerSocket_WinSocket::start(const std::string& address, uint16_t port){ } } +uint64_t ServerSocket_WinSocket::connection_id(){ + std::lock_guard lg(m_lock); + return m_state.load(std::memory_order_relaxed) == State::CONNECTED + ? m_connection_id + : 0; +} + + void ServerSocket_WinSocket::close_connection(){ SOCKET socket; { @@ -170,13 +178,16 @@ void ServerSocket_WinSocket::close_and_wait() noexcept{ } } -size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono::milliseconds timeout){ +size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono::milliseconds timeout, uint64_t connection){ SOCKET socket; { std::lock_guard lg(m_lock); - socket = m_state.load(std::memory_order_relaxed) == State::CONNECTED - ? m_client - : INVALID_SOCKET; + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ + return 0; + } + socket = m_client; } if (socket == INVALID_SOCKET){ return 0; @@ -195,7 +206,12 @@ size_t ServerSocket_WinSocket::send(const void* data, size_t bytes, std::chrono: while (bytes > 0){ { std::lock_guard lg(m_lock); - if (m_state.load(std::memory_order_relaxed) != State::CONNECTED){ + // The connection may have been replaced (or closed) since the + // last iteration: in that case this write must stop rather + // than reach the new client's socket. + if (m_state.load(std::memory_order_relaxed) != State::CONNECTED + || (connection != 0 && m_connection_id != connection) + ){ return sent; } } @@ -379,6 +395,7 @@ void ServerSocket_WinSocket::accept_thread_loop(std::string address, uint16_t po ioctlsocket(client_socket, FIONBIO, &client_non_blocking); m_client = client_socket; m_close_requested = false; + m_connection_id++; m_state.store(State::CONNECTED, std::memory_order_release); accepted = true; } diff --git a/Common/Cpp/Sockets/ServerSocket_WinSocket.h b/Common/Cpp/Sockets/ServerSocket_WinSocket.h index 6b68519ab8..f478e69b46 100644 --- a/Common/Cpp/Sockets/ServerSocket_WinSocket.h +++ b/Common/Cpp/Sockets/ServerSocket_WinSocket.h @@ -45,9 +45,15 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ virtual size_t send( const void* data, size_t bytes, - std::chrono::milliseconds timeout = std::chrono::milliseconds::max() + std::chrono::milliseconds timeout = std::chrono::milliseconds::max(), + uint64_t connection = 0 ) override; + // A token identifying the connection the socket is currently + // serving. It changes every time a new client is accepted, and it + // is 0 when no connection is active. + virtual uint64_t connection_id() override; + private: // Winsock has to be started before any of its functions can be used, and @@ -96,6 +102,12 @@ class ServerSocket_WinSocket final : public AbstractServerSocket{ SOCKET m_client = INVALID_SOCKET; bool m_close_requested = false; + // A connection token, incremented (under m_lock) on every accept. + // send() compares it against the token the caller captured, so a + // write that outlived its connection cannot reach the client that + // replaced it. 0 means no connection is active. + uint64_t m_connection_id = 0; + // One task per start() call. start() must never destroy the previous // task's AsyncTask here: its destructor blocks until the task has // finished, and a task from an earlier attempt may still be running diff --git a/Common/Cpp/Sockets/Sockets_Tests.cpp b/Common/Cpp/Sockets/Sockets_Tests.cpp index 7864a7d34a..3fe3bd915d 100644 --- a/Common/Cpp/Sockets/Sockets_Tests.cpp +++ b/Common/Cpp/Sockets/Sockets_Tests.cpp @@ -822,6 +822,117 @@ class Test_Sockets_RejectSecondClient : public TestSockets +class Test_Sockets_StaleSend : public TestSockets{ +public: + using TestSockets::TestSockets; + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + ServerListener server_listener; + ServerSocketType server(this->m_thread_pool); + server.add_listener(server_listener); + + uint16_t port; + if (!this->start_server(server, server_listener, port)){ + return "Failed to start the server"; + } + + std::string error; + if (server.connection_id() != 0){ + error = "connection_id() was not 0 before any connection"; + } + + uint64_t first_id = 0; + { + ClientListener client_listener; + ClientSocketType client(this->m_thread_pool); + client.add_listener(client_listener); + + if (!this->connect_client(client, client_listener, port, 1)){ + error = "The first client did not finish connecting"; + }else if (!server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 1; } + )){ + error = "The server did not call on_connect"; + }else{ + first_id = server.connection_id(); + if (first_id == 0){ + error = "The server did not assign a connection id"; + } + client.close(); + } + } + + if (error.empty() && !server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.disconnected.load() >= 1; } + )){ + error = "The server did not call on_disconnect"; + } + + // The counter is monotonic and not reset, but with no active + // connection the id must report 0. + if (error.empty() && server.connection_id() != 0){ + error = "connection_id() was not 0 while no connection was active"; + } + + uint64_t second_id = 0; + { + ClientListener client_listener; + ClientSocketType client(this->m_thread_pool); + client.add_listener(client_listener); + + if (!this->connect_client(client, client_listener, port, 1)){ + error = "The second client did not finish connecting"; + }else if (!server_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&server_listener]{ return server_listener.connected.load() >= 2; } + )){ + error = "The server did not call on_connect for the second client"; + }else{ + second_id = server.connection_id(); + if (second_id == first_id){ + error = "The connection id did not change after reconnect"; + } + // A send for the replaced connection must be refused, + // not delivered into the new client's socket. + if (error.empty() && + server.send("stale", 5, std::chrono::milliseconds::max(), first_id) != 0 + ){ + error = "A send for a replaced connection was not refused"; + } + // A send for the current connection still goes through. + if (error.empty() && + server.send("hello", 5, std::chrono::milliseconds::max(), second_id) != 5 + ){ + error = "The send for the current connection was not delivered"; + } + if (error.empty() && !client_listener.waiter.wait_for( + SOCKET_TEST_WAIT, + [&client_listener]{ return client_listener.received.size() >= 5; } + )){ + error = "The second client did not receive the data"; + }else if (error.empty() && + std::memcmp(client_listener.received.data(), "hello", 5) != 0 + ){ + error = "The second client received the wrong data"; + } + client.close(); + } + } + + if (!error.empty()){ + return error; + } + return true; + } +}; + + // Starting a second server on an already-bound port must fail through // on_listen_finished and leave the server not running. template @@ -2676,6 +2787,9 @@ void add_socket_tests(UnitTestDatabase& database, ThreadPool& thread_pool, const database.add>( thread_pool, prefix + "Reject a second client" ); + database.add>( + thread_pool, prefix + "A send for a replaced connection is refused" + ); database.add>( thread_pool, prefix + "Port in use" ); diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index d7fc0c3e76..539f6ec13a 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -66,6 +66,8 @@ find_package(Threads REQUIRED) # Define a toggle option (ON by default) option(COMMAND_LINE_EXE "Build the optional executable" ON) +# Build SerialProgramsTests, which runs the unit tests without the GUI. +option(TEST_RUNNER_EXE "Build the unit test executable, SerialProgramsTests" ON) # Build the _pa_core Python (pybind11) module; see Source/PythonBindings/README.md. # OFF by default because it needs a Python interpreter with development headers # and pybind11 (a system package or `pip install pybind11`). @@ -236,6 +238,27 @@ if (APPLE) # https://stackoverflow.com/questions/53560288/how-to-create-a-macos-app-bundle-with-cmake # https://cmake.org/cmake/help/latest/command/add_executable.html add_executable(SerialPrograms MACOSX_BUNDLE ${SerialPrograms_ICON} ${EXECUTABLE_SOURCES}) + # Compiled identity survives Finder/TCC relaunch without command-line arguments. + set(PA_LAB_PROFILE "" CACHE STRING "Bind this lab binary to one macOS settings profile") + if(NOT PA_LAB_PROFILE STREQUAL "") + if(NOT PA_LAB_PROFILE MATCHES "^[A-Za-z0-9_][A-Za-z0-9_-]*$") + message(FATAL_ERROR "PA_LAB_PROFILE must contain only ASCII letters, digits, '_' or '-'.") + endif() + target_compile_definitions(SerialPrograms PRIVATE PA_LAB_PROFILE="${PA_LAB_PROFILE}") + endif() + + # Receipt binds packaging to the linked binary, not just a CMake cache value. + set_property(TARGET SerialPrograms APPEND PROPERTY LINK_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LabBuildReceipt.cmake") + add_custom_command(TARGET SerialPrograms POST_BUILD + COMMAND "${CMAKE_COMMAND}" + "-DBINARY=$" + "-DPROFILE=${PA_LAB_PROFILE}" + "-DSOURCE=${CMAKE_CURRENT_SOURCE_DIR}/.." + -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LabBuildReceipt.cmake" + VERBATIM + ) + set_target_properties(SerialPrograms PROPERTIES BUNDLE True MACOSX_BUNDLE_GUI_IDENTIFIER PokemonAutomation.SerialPrograms @@ -368,6 +391,10 @@ if(NOT EXISTS "${DEST_DIR}/Scripts") ) endif() +if(TEST_RUNNER_EXE) + include(Source/Tests/TestRunnerExecutable.cmake) +endif() + if (WIN32) if (PA_SDL) add_library(SDL3_lib IMPORTED UNKNOWN) @@ -944,6 +971,15 @@ if(COMMAND_LINE_EXE) include(Source/CommandLine/CommandLineExecutable.cmake) endif() +# The registered cross-process tests use this GUI-free native runner as their child. +add_subdirectory(Source/CommonFramework/Devices) +if(WIN32) + set_target_properties(LabDeviceAccessTests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${WIN_DEPLOY_DIR}") +else() + set_target_properties(LabDeviceAccessTests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$") +endif() +add_dependencies(SerialPrograms LabDeviceAccessTests) + if(PA_PYTHON_BINDINGS) # Python bindings (GUI-free) from subdirectory include(Source/PythonBindings/PythonBindings.cmake) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp index 8327b7e48a..b2a1cfad58 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.cpp @@ -11,6 +11,8 @@ #include "Sources/AudioSource_File.h" #include "Sources/AudioSource_Device.h" #include "AudioOption.h" +#include "AudioPipelineOptions.h" +#include "CommonFramework/GlobalSettingsPanel.h" //#include //using std::cout; @@ -55,6 +57,7 @@ std::string AudioOption::audioDisplayTypeToString(AudioOption::AudioDisplayType AudioOption::AudioOption() : m_input_source(new AudioSourceDescriptor_Null()) + , m_display_type(GlobalSettings::instance().AUDIO_PIPELINE->DEFAULT_DISPLAY) {} void AudioOption::set_input(std::shared_ptr descriptor){ diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h index 73e002f2a8..c63b0a83ee 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioOption.h @@ -72,7 +72,7 @@ class AudioOption{ AudioInputChoice m_input_choice; AudioDeviceInfo m_output_device; - AudioDisplayType m_display_type = AudioDisplayType::FREQ_BARS; + AudioDisplayType m_display_type; // the global default until load_json() sets one double m_volume = 1.0; // Volume Range: [0, 1.0] }; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h index 5f4a889b8c..d0f7e3bd73 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/AudioPipelineOptions.h @@ -11,6 +11,8 @@ #include "Common/Cpp/Options/BooleanCheckBoxOption.h" #include "Common/Cpp/Options/SimpleIntegerOption.h" #include "Common/Cpp/Options/FloatingPointOption.h" +#include "Common/Cpp/Options/EnumDropdownOption.h" +#include "CommonFramework/AudioPipeline/AudioOption.h" #include "CommonFramework/StaticGlobals.h" namespace PokemonAutomation{ @@ -26,6 +28,17 @@ class AudioPipelineOptions : public GroupOption{ true, false ) + , DEFAULT_DISPLAY( + "Default Audio Display:
" + "What a program's console shows for its audio until you pick a display in that console's settings.", + { + {AudioOption::AudioDisplayType::NO_DISPLAY, "none", "No Display"}, + {AudioOption::AudioDisplayType::FREQ_BARS, "freq-bars", "Frequency Bars"}, + {AudioOption::AudioDisplayType::SPECTROGRAM, "spectrogram", "Spectrogram"}, + }, + LockMode::UNLOCK_WHILE_RUNNING, + AudioOption::AudioDisplayType::FREQ_BARS + ) , FILE_VOLUME_SCALE( "Audio File Input Volume Scale:
" "Multiply audio file playback by this factor. (This is linear scale. So each factor of 10 is 20dB.)", @@ -58,6 +71,7 @@ class AudioPipelineOptions : public GroupOption{ 5 ) { + PA_ADD_OPTION(DEFAULT_DISPLAY); PA_ADD_OPTION(FILE_VOLUME_SCALE); PA_ADD_OPTION(DEVICE_VOLUME_SCALE); PA_ADD_OPTION(USE_KAISER_WINDOW); @@ -68,6 +82,7 @@ class AudioPipelineOptions : public GroupOption{ } public: + EnumDropdownOption DEFAULT_DISPLAY; FloatingPointOption FILE_VOLUME_SCALE; FloatingPointOption DEVICE_VOLUME_SCALE; BooleanCheckBoxOption USE_KAISER_WINDOW; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp index 4bb1f84370..51bfcf9936 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.cpp @@ -406,16 +406,12 @@ void AudioPassthroughPairQt::reset( auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this, format, descriptor, output, output_volume]{ auto scope_check = m_sanitizer.check_scope(); + close_audio_source(); std::unique_ptr source; if (descriptor){ source = descriptor->make_AudioSource(m_logger, format); } WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_reader){ - teardown_tree(); - m_sink_device.reset(); - m_reader.reset(); - } m_input_choice = format; m_output_device = output; m_output_volume = output_volume; @@ -427,6 +423,15 @@ void AudioPassthroughPairQt::reset( } }); } +// Called on this object's Qt thread before constructing a replacement input. +void AudioPassthroughPairQt::close_audio_source(){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + if (m_reader){ + teardown_tree(); + m_sink_device.reset(); + m_reader.reset(); + } +} void AudioPassthroughPairQt::clear_audio_source(){ auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this]{ @@ -447,16 +452,12 @@ void AudioPassthroughPairQt::set_audio_source( auto scope_check = m_sanitizer.check_scope(); QMetaObject::invokeMethod(this, [this, descriptor, format]{ auto scope_check = m_sanitizer.check_scope(); + close_audio_source(); std::unique_ptr source; if (descriptor){ source = descriptor->make_AudioSource(m_logger, format); } WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_reader){ - teardown_tree(); - m_sink_device.reset(); - m_reader.reset(); - } m_input_choice = format; if (source){ m_reader = std::move(source); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h index 014b9c2da7..872904e33f 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Backends/AudioPassthroughPairQt.h @@ -97,6 +97,7 @@ class AudioPassthroughPairQt final : public QObject, public AudioPassthroughPair // thread only ever sees them through the node listener sets, where // ListenerSet::remove blocks on in-flight pushes so rebuilds never // race a live push. + void close_audio_source(); void teardown_tree(); void teardown_sink_branch(); void build_tree(float input_volume); diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp index 758ccc70b8..0795682ce4 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.cpp @@ -53,6 +53,7 @@ AudioInputDevice::AudioInputDevice( RAW_BUFFER_CAPACITY ) , ObjectStreamListener(raw_output_frame_bytes(format, reinterpret_96k_mono_as_stereo)) + , m_device_lease(acquire_lab_capture(logger, LabDeviceKind::AUDIO, device.id().toHex().toStdString())) , m_sample_type(sample_format_of(format)) , m_sample_rate(format.sampleRate()) , m_channels(format.channelCount()) diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h index 2ef2c7365b..fba2d91614 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/IO/AudioInputDevice.h @@ -17,6 +17,7 @@ using NativeAudioSource = QAudioSource; #include "Common/Cpp/StreamConverters.h" #include "Kernels/AudioPipeline/Kernels_AudioPipeline.h" #include "CommonFramework/AudioPipeline/AudioFormat.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" class QAudioFormat; @@ -87,6 +88,7 @@ class AudioInputDevice final private: + std::unique_ptr m_device_lease; std::unique_ptr m_source; Kernels::AudioPipeline::SampleFormat m_sample_type; size_t m_sample_rate; diff --git a/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp b/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp index 18706767e7..afca55d3ba 100644 --- a/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp +++ b/SerialPrograms/Source/CommonFramework/AudioPipeline/Sources/AudioSource_Device.cpp @@ -68,7 +68,11 @@ std::unique_ptr AudioSourceDescriptor_Device::make_AudioSource( return nullptr; } // cout << "format = " << (int)format << endl; - return std::make_unique(logger, m_info, format); + try{ + return std::make_unique(logger, m_info, format); + }catch (UserSetupError&){ + return nullptr; // Refusal is logged; never throw across the Qt audio callback. + } } diff --git a/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt b/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt new file mode 100644 index 0000000000..dba4f55b10 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/CMakeLists.txt @@ -0,0 +1,20 @@ +# Focused native tests, without configuring/linking the GUI application: +# cmake -S SerialPrograms/Source/CommonFramework/Devices -B /path/to/test-build +# cmake --build /path/to/test-build -j4 +# ctest --test-dir /path/to/test-build --output-on-failure +cmake_minimum_required(VERSION 3.24) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(LabDeviceAccessTests LANGUAGES CXX) + enable_testing() + find_package(Qt6 REQUIRED COMPONENTS Core) +endif() +add_executable(LabDeviceAccessTests LabDeviceAccessQt.cpp LabDeviceAccessQt_Tests.cpp) +target_compile_features(LabDeviceAccessTests PRIVATE cxx_std_23) +target_compile_definitions(LabDeviceAccessTests PRIVATE PA_LAB_DEVICE_STANDALONE_TESTS) +target_link_libraries(LabDeviceAccessTests PRIVATE Qt6::Core) +if(MSVC) + target_compile_options(LabDeviceAccessTests PRIVATE /W4 /WX) +else() + target_compile_options(LabDeviceAccessTests PRIVATE -Wall -Wextra -Werror) +endif() +add_test(NAME LabDeviceAccess COMMAND LabDeviceAccessTests) diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp new file mode 100644 index 0000000000..d420befadf --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.cpp @@ -0,0 +1,153 @@ +/* Lab Device Access (Qt process-lock seam) + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "LabDeviceAccessQt.h" + +namespace PokemonAutomation{ +namespace{ + +QString text(std::string_view value){ + return QString::fromUtf8(value.data(), (qsizetype)value.size()); +} + +std::set read_ids(const QJsonObject& object, const char* key){ + if (!object.value(key).isArray()){ + throw std::runtime_error(std::string("Expected device array: ") + key); + } + std::set ids; + for (const QJsonValue& value : object.value(key).toArray()){ + const std::string id = value.toString().toStdString(); + if (!value.isString() || id.empty() || id.find('\0') != std::string::npos || !ids.emplace(id).second){ + throw std::runtime_error(std::string("Empty, invalid or duplicate device ID in ") + key); + } + } + return ids; +} + +bool permitted(const LabDevicePolicy& policy, const LabDeviceIdentity& device){ + switch (device.kind){ + case LabDeviceKind::SERIAL: return policy.serial.contains(device.id); + case LabDeviceKind::VIDEO: return policy.video.contains(device.id); + case LabDeviceKind::AUDIO: return policy.audio.contains(device.id); + case LabDeviceKind::PROFILE: return false; + } + return false; +} + +// Capture IDs are allowlisted as hex; show the readable ID beside it so a +// refusal names the device the settings selected. +std::string readable_capture_id(const std::string& hex){ + const bool is_hex = hex.size() % 2 == 0 && std::ranges::all_of(hex, [](char c){ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + }); + if (!is_hex){ + return hex; + } + const QByteArray bytes = QByteArray::fromHex(QByteArray(hex.data(), (qsizetype)hex.size())); + return bytes.toStdString() + " (allowlist ID " + hex + ")"; +} + +std::string describe(const LabDeviceIdentity& device){ + switch (device.kind){ + case LabDeviceKind::SERIAL: return "Serial device " + device.id; + case LabDeviceKind::VIDEO: return "Video input " + readable_capture_id(device.id); + case LabDeviceKind::AUDIO: return "Audio input " + readable_capture_id(device.id); + case LabDeviceKind::PROFILE: return "Profile " + device.id; + } + return device.id; +} + +std::string normalize_serial_path(std::string_view path){ + if (path.starts_with("/dev/tty.")){ + return "/dev/cu." + std::string(path.substr(9)); + } + return std::string(path); +} + +} // namespace + +LabDevicePolicy parse_lab_device_policy(std::string_view json, std::string_view profile){ + QJsonParseError error; + const QJsonDocument document = QJsonDocument::fromJson(QByteArray(json.data(), (qsizetype)json.size()), &error); + const QJsonObject object = document.object(); + const std::set keys{"schema", "profile", "serial", "video", "audio"}; + for (auto it = object.begin(); it != object.end(); ++it){ + if (!keys.contains(it.key())){ + throw std::runtime_error("Unknown field in lab device policy."); + } + } + if (error.error != QJsonParseError::NoError || !document.isObject() || object.size() != 5 || + !object.value("schema").isDouble() || object.value("schema").toDouble() != 1 || + profile.empty() || object.value("profile").toString() != text(profile)){ + throw std::runtime_error("Missing, malformed or mismatched lab device policy."); + } + return {std::string(profile), read_ids(object, "serial"), read_ids(object, "video"), read_ids(object, "audio")}; +} + +std::string lab_serial_identity(unsigned vendor, unsigned product, std::string_view serial){ + if (vendor > 0xffff || product > 0xffff || serial.empty() || serial.find('\0') != std::string_view::npos){ + throw std::runtime_error("A unique USB serial, vendor and product identity are required."); + } + return std::format("{:04x}:{:04x}:{}", vendor, product, serial); +} + +std::string resolve_lab_serial_identity(std::span devices, std::string_view port){ + const std::string path = normalize_serial_path(port); + const auto selected = std::ranges::find_if(devices, [&](const LabSerialDevice& device){ + return normalize_serial_path(device.port) == path; + }); + if (selected == devices.end()){ + throw std::runtime_error("Controller has no current USB identity; hardware access is disabled."); + } + const std::string identity = lab_serial_identity(selected->vendor, selected->product, selected->serial); + for (const LabSerialDevice& other : devices){ + if (normalize_serial_path(other.port) != path && !other.serial.empty() && + lab_serial_identity(other.vendor, other.product, other.serial) == identity){ + throw std::runtime_error("Two adapters report the same USB identity; refusing ambiguous ownership."); + } + } + return identity; +} + +std::string lab_device_lock_path(std::string_view directory, const LabDeviceIdentity& device){ + if (directory.empty() || device.id.empty()){ + throw std::runtime_error("Device lease directory and identity must be nonempty."); + } + const QByteArray key = QByteArray::number((int)device.kind) + ':' + QByteArray(device.id.data(), (qsizetype)device.id.size()); + const QByteArray digest = QCryptographicHash::hash(key, QCryptographicHash::Sha256).toHex(); + return QDir(text(directory)).filePath(QString::fromLatin1(digest) + ".lock").toStdString(); +} + +LabDeviceLease::LabDeviceLease(std::string_view path) + : m_lock(text(path)) +{ + // Age alone must never evict an active multi-hour lab run. + m_lock.setStaleLockTime(std::chrono::milliseconds::zero()); + if (!m_lock.tryLock(std::chrono::milliseconds::zero())){ + throw std::runtime_error("Device/profile is already leased, or its lock cannot be created. Nothing was opened."); + } +} + +std::unique_ptr acquire_lab_device( + const LabDevicePolicy& policy, std::string_view directory, const LabDeviceIdentity& device +){ + if (!permitted(policy, device)){ + throw std::runtime_error(describe(device) + " is not allowlisted for profile " + policy.profile + ". Nothing was opened."); + } + if (!QDir().mkpath(text(directory))){ + throw std::runtime_error("Cannot create shared device lease directory. Nothing was opened."); + } + return std::make_unique(lab_device_lock_path(directory, device)); +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h new file mode 100644 index 0000000000..669ce69f9d --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt.h @@ -0,0 +1,72 @@ +/* Lab Device Access (Qt process-lock seam) + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceAccessQt_H +#define PokemonAutomation_LabDeviceAccessQt_H + +#include +#include +#include +#include +#include +#include + +namespace PokemonAutomation{ + +// Separate namespaces for serial, video, audio and whole-profile reservations. +enum class LabDeviceKind{ SERIAL, VIDEO, AUDIO, PROFILE }; + +// Stable hardware identity, never a display alias, ordinal or USB port pathname. +struct LabDeviceIdentity{ + LabDeviceKind kind; + std::string id; +}; + +// Immutable allowlist loaded once for the selected lab profile. +struct LabDevicePolicy{ + std::string profile; + std::set serial; + std::set video; + std::set audio; +}; + +// One currently enumerated USB port; tty/cu aliases may describe the same adapter. +struct LabSerialDevice{ + std::string port; + unsigned vendor; + unsigned product; + std::string serial; +}; + +// Resolve a selected current port, rejecting missing or ambiguous physical identity. +std::string resolve_lab_serial_identity(std::span devices, std::string_view port); + +// Parse the strict schema; malformed/mismatched input throws std::runtime_error. +LabDevicePolicy parse_lab_device_policy(std::string_view json, std::string_view profile); + +// Normalize VID/PID and a nonempty USB serial; empty identifiers are refused. +std::string lab_serial_identity(unsigned vendor, unsigned product, std::string_view serial); + +// Compute a shared lock path independent of profile, executable and display names. +std::string lab_device_lock_path(std::string_view directory, const LabDeviceIdentity& device); + +// A noncopyable process lease. Destruction releases it; dead owners are recovered by Qt. +// Keep this alive until after the native device has closed. Acquisition never waits. +class LabDeviceLease{ +public: + explicit LabDeviceLease(std::string_view path); + LabDeviceLease(const LabDeviceLease&) = delete; + LabDeviceLease& operator=(const LabDeviceLease&) = delete; +private: + QLockFile m_lock; +}; + +// Check the allowlist before acquiring a lease. A denial never touches hardware. +// Throws std::runtime_error on denied access, contention or filesystem failure. +std::unique_ptr acquire_lab_device( + const LabDevicePolicy& policy, std::string_view directory, const LabDeviceIdentity& device +); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp new file mode 100644 index 0000000000..83f5613e96 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.cpp @@ -0,0 +1,328 @@ +/* Lab Device Access Tests + * + * From: https://github.com/PokemonAutomation/ + * + * PR #226 startup incident: a Console B test build opened Console A's controller. + * These tests never enumerate or open physical devices, or load real profiles. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "LabDeviceAccessQt.h" +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +#include "LabDeviceAccessQt_Tests.h" +#endif + +namespace PokemonAutomation{ +namespace{ + +struct PolicyInput{ + std::string_view name; + std::string_view json; + LabDeviceIdentity requested; +}; + +struct PolicyExpectation{ + int opens = 0; + std::string_view refusal = {}; // The refusal message starts with this. +}; + +struct PolicyCase{ + PolicyInput input; + PolicyExpectation expectation; +}; + +struct SerialInput{ + std::string_view name; + std::vector devices; + std::string_view port; +}; + +struct SerialExpectation{ + std::optional identity = std::nullopt; +}; + +struct SerialCase{ + SerialInput input; + SerialExpectation expectation; +}; + +const SerialCase SERIAL_CASES[] = { + {.input = {.name = "USB serial identity", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "A"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:A"}}, + {.input = {.name = "tty and cu share a key", .devices = {{"/dev/tty.A", 0x10c4, 0xea60, "A"}, {"/dev/cu.A", 0x10c4, 0xea60, "A"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:A"}}, + {.input = {.name = "two adapters with duplicate serial", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "same"}, {"/dev/cu.B", 0x10c4, 0xea60, "same"}}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "missing USB serial", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, ""}}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "unplugged adapter", .devices = {}, .port = "/dev/cu.A"}, .expectation = {}}, + {.input = {.name = "path reused by another adapter", .devices = {{"/dev/cu.A", 0x10c4, 0xea60, "B"}}, .port = "/dev/cu.A"}, .expectation = {.identity = "10c4:ea60:B"}}, +}; + +std::string check_serial(const SerialCase& test){ + std::optional identity; + try{ + identity = resolve_lab_serial_identity(test.input.devices, test.input.port); + }catch (const std::runtime_error&){} + if (identity != test.expectation.identity){ + return "Expected identity " + test.expectation.identity.value_or("") + "; read " + identity.value_or(""); + } + return {}; +} + +const PolicyCase POLICY_CASES[] = { + {.input = {.name = "matching serial", .json = R"({"schema":1,"profile":"lab-B","serial":["10c4:ea60:B"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "10c4:ea60:B"}}, .expectation = {.opens = 1}}, + {.input = {.name = "wrong console serial", .json = R"({"schema":1,"profile":"lab-B","serial":["10c4:ea60:B"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "10c4:ea60:A"}}, .expectation = {.refusal = "Serial device 10c4:ea60:A is not allowlisted for profile lab-B."}}, + {.input = {.name = "camera allowed", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":["766964656f42"],"audio":[]})", .requested = {LabDeviceKind::VIDEO, "766964656f42"}}, .expectation = {.opens = 1}}, + {.input = {.name = "audio allowed", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":[],"audio":["617564696f42"]})", .requested = {LabDeviceKind::AUDIO, "617564696f42"}}, .expectation = {.opens = 1}}, + // lab-B, 2026-11-08: a panel selected console A's audio input, and the refusal didn't say which device it was. + {.input = {.name = "wrong console audio", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":[],"audio":["617564696f42"]})", .requested = {LabDeviceKind::AUDIO, "617564696f41"}}, .expectation = {.refusal = "Audio input audioA (allowlist ID 617564696f41) is not allowlisted for profile lab-B."}}, + {.input = {.name = "camera not audio", .json = R"({"schema":1,"profile":"lab-B","serial":[],"video":["same-id"],"audio":[]})", .requested = {LabDeviceKind::AUDIO, "same-id"}}, .expectation = {}}, + {.input = {.name = "mismatched profile", .json = R"({"schema":1,"profile":"lab-A","serial":["port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "missing policy", .json = "", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "malformed policy", .json = "{", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "unsupported schema", .json = R"({"schema":2,"profile":"lab-B","serial":["port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "duplicate identity", .json = R"({"schema":1,"profile":"lab-B","serial":["port","port"],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "missing audio list", .json = R"({"schema":1,"profile":"lab-B","serial":["port"],"video":[]})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, + {.input = {.name = "empty identity", .json = R"({"schema":1,"profile":"lab-B","serial":[""],"video":[],"audio":[]})", .requested = {LabDeviceKind::SERIAL, ""}}, .expectation = {}}, + {.input = {.name = "unknown policy field", .json = R"({"schema":1,"profile":"lab-B","serial":["port"],"video":[],"audio":[],"allow_all":true})", .requested = {LabDeviceKind::SERIAL, "port"}}, .expectation = {}}, +}; + +std::string check_policy(const PolicyCase& test){ + QTemporaryDir directory; + if (!directory.isValid()){ + return "Cannot create temporary lease directory."; + } + int opens = 0; + std::string refusal; + try{ + const auto policy = parse_lab_device_policy(test.input.json, "lab-B"); + auto lease = acquire_lab_device(policy, directory.path().toStdString(), test.input.requested); + opens++; // Scripted native open: no real port, camera or microphone exists. + }catch (const std::runtime_error& error){ + refusal = error.what(); + } + if (opens != test.expectation.opens){ + return "Expected opens " + std::to_string(test.expectation.opens) + "; read " + std::to_string(opens); + } + if (!refusal.starts_with(test.expectation.refusal)){ + return "Expected a refusal starting \"" + std::string(test.expectation.refusal) + "\"; read \"" + refusal + "\""; + } + return {}; +} + +enum class LeaseScenario{ SAME_PROCESS, DIFFERENT_DEVICE, DIFFERENT_KIND, RELEASE, FAILED_OPEN, LONG_RUNNING, LIVE_CHILD, DEAD_CHILD, PROFILE, EARLY_CHILD }; + +struct LeaseInput{ + std::string_view name; + LeaseScenario scenario; +}; + +struct LeaseExpectation{ + bool second_opens; +}; + +struct LeaseCase{ + LeaseInput input; + LeaseExpectation expectation; +}; + +const LeaseCase LEASE_CASES[] = { + {.input = {.name = "same device two profiles", .scenario = LeaseScenario::SAME_PROCESS}, .expectation = {.second_opens = false}}, + {.input = {.name = "different consoles", .scenario = LeaseScenario::DIFFERENT_DEVICE}, .expectation = {.second_opens = true}}, + {.input = {.name = "video and audio inputs", .scenario = LeaseScenario::DIFFERENT_KIND}, .expectation = {.second_opens = true}}, + {.input = {.name = "disconnect then reconnect", .scenario = LeaseScenario::RELEASE}, .expectation = {.second_opens = true}}, + {.input = {.name = "failed native open releases lease", .scenario = LeaseScenario::FAILED_OPEN}, .expectation = {.second_opens = true}}, +#if !defined(_WIN32) + // Windows' live lock prevents editing this timestamp fixture; exercise it on POSIX. + {.input = {.name = "age never steals live lease", .scenario = LeaseScenario::LONG_RUNNING}, .expectation = {.second_opens = false}}, +#endif + {.input = {.name = "live second process", .scenario = LeaseScenario::LIVE_CHILD}, .expectation = {.second_opens = false}}, + {.input = {.name = "profile reserved before Qt startup", .scenario = LeaseScenario::EARLY_CHILD}, .expectation = {.second_opens = false}}, + {.input = {.name = "killed owner recovery", .scenario = LeaseScenario::DEAD_CHILD}, .expectation = {.second_opens = true}}, + {.input = {.name = "duplicate profile", .scenario = LeaseScenario::PROFILE}, .expectation = {.second_opens = false}}, +}; + +std::string test_helper_path(){ + QString name = "LabDeviceAccessTests"; +#if defined(_WIN32) + name += ".exe"; +#endif + return QDir(QCoreApplication::applicationDirPath()).filePath(name).toStdString(); +} + +void start_lease_child(QProcess& child, const std::string& path, LeaseScenario scenario){ + const QString flag = scenario == LeaseScenario::EARLY_CHILD ? "--lease-before-qt" : "--lease-child"; + child.start(QString::fromStdString(test_helper_path()), {flag, QString::fromStdString(path)}); + if (!child.waitForStarted(5000) || !child.waitForReadyRead(5000) || child.readAllStandardOutput().trimmed() != "LOCKED"){ + throw std::runtime_error("Lease test child did not confirm ownership."); + } +} + +std::unique_ptr prepare_owner(LeaseScenario scenario, const std::string& path, QProcess& child){ + if (scenario == LeaseScenario::LIVE_CHILD || scenario == LeaseScenario::DEAD_CHILD || scenario == LeaseScenario::EARLY_CHILD){ + start_lease_child(child, path, scenario); + if (scenario == LeaseScenario::DEAD_CHILD){ + child.kill(); + if (!child.waitForFinished(5000)){ + throw std::runtime_error("Child did not exit after kill."); + } + } + return nullptr; + } + auto owner = std::make_unique(path); + if (scenario == LeaseScenario::FAILED_OPEN){ + try{ + auto opening = std::move(owner); + throw std::runtime_error("Scripted native open failed."); + }catch (const std::runtime_error&){} + } + if (scenario == LeaseScenario::RELEASE){ + owner.reset(); + } + if (scenario == LeaseScenario::LONG_RUNNING){ + QFile lock(QString::fromStdString(path)); + if (!lock.open(QIODevice::ReadOnly) || !lock.setFileTime(QDateTime::currentDateTimeUtc().addDays(-2), QFileDevice::FileModificationTime)){ + throw std::runtime_error("Could not age the lock fixture."); + } + } + return owner; +} + +std::string check_lease(const LeaseCase& test){ + QTemporaryDir directory; + if (!directory.isValid()){ + return "Cannot create temporary lease directory."; + } + LabDevicePolicy policy{"lab-B", {"port-A", "port-B"}, {"port-A"}, {}}; + LabDeviceIdentity first{LabDeviceKind::SERIAL, "port-A"}; + if (test.input.scenario == LeaseScenario::PROFILE){ + first.kind = LabDeviceKind::PROFILE; + } + const std::string root = directory.path().toStdString(); + const std::string path = lab_device_lock_path(root, first); + QProcess child; + try{ + auto owner = prepare_owner(test.input.scenario, path, child); + LabDeviceIdentity second = first; + if (test.input.scenario == LeaseScenario::DIFFERENT_DEVICE){ + second.id = "port-B"; + } + if (test.input.scenario == LeaseScenario::DIFFERENT_KIND){ + second.kind = LabDeviceKind::VIDEO; + } + bool opened = false; + try{ + auto lease = second.kind == LabDeviceKind::PROFILE + ? std::make_unique(lab_device_lock_path(root, second)) + : acquire_lab_device(policy, root, second); + opened = true; + }catch (const std::runtime_error&){} + child.kill(); + child.waitForFinished(5000); + if (opened != test.expectation.second_opens){ + return "Expected second opens " + std::to_string(test.expectation.second_opens) + "; read " + std::to_string(opened); + } + }catch (const std::runtime_error& error){ + child.kill(); + child.waitForFinished(5000); + return error.what(); + } + return {}; +} + +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +class SerialTest : public UnitTest{ +public: + explicit SerialTest(const SerialCase& test) : UnitTest("LabDevices/Serial/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_serial(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const SerialCase& m_test; +}; + +class PolicyTest : public UnitTest{ +public: + explicit PolicyTest(const PolicyCase& test) : UnitTest("LabDevices/Policy/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_policy(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const PolicyCase& m_test; +}; + +class LeaseTest : public UnitTest{ +public: + explicit LeaseTest(const LeaseCase& test) : UnitTest("LabDevices/Lease/" + std::string(test.input.name)), m_test(test){} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + const std::string error = check_lease(m_test); + return error.empty() ? UnitTestResult(true) : UnitTestResult(error); + } +private: + const LeaseCase& m_test; +}; +#endif + +} // namespace + +#ifndef PA_LAB_DEVICE_STANDALONE_TESTS +void add_tests_LabDeviceAccess(UnitTestDatabase& database){ + for (const SerialCase& test : SERIAL_CASES){ + database.add(test); + } + for (const PolicyCase& test : POLICY_CASES){ + database.add(test); + } + for (const LeaseCase& test : LEASE_CASES){ + database.add(test); + } +} +#endif + +} // namespace PokemonAutomation + +#ifdef PA_LAB_DEVICE_STANDALONE_TESTS +// Small Qt-only runner/child process. It never links the GUI or device backends. +int main(int argc, char* argv[]){ + std::unique_ptr early_lease; + if (argc == 3 && std::string_view(argv[1]) == "--lease-before-qt"){ + early_lease = std::make_unique(argv[2]); + } + QCoreApplication application(argc, argv); + if (early_lease){ + std::cout << "LOCKED" << std::endl; + return application.exec(); + }else if (argc == 3 && std::string_view(argv[1]) == "--lease-child"){ + PokemonAutomation::LabDeviceLease lease(argv[2]); + std::cout << "LOCKED" << std::endl; + return application.exec(); + } + if (argc != 1){ + return 2; + } + int failed = 0; + for (const auto& test : PokemonAutomation::SERIAL_CASES){ + const std::string error = PokemonAutomation::check_serial(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + for (const auto& test : PokemonAutomation::POLICY_CASES){ + const std::string error = PokemonAutomation::check_policy(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + for (const auto& test : PokemonAutomation::LEASE_CASES){ + const std::string error = PokemonAutomation::check_lease(test); + std::cout << test.input.name << ": " << (error.empty() ? "PASS" : error) << std::endl; + failed += !error.empty(); + } + return failed; +} +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h new file mode 100644 index 0000000000..73a5d516a8 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceAccessQt_Tests.h @@ -0,0 +1,16 @@ +/* Lab Device Access Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceAccessQt_Tests_H +#define PokemonAutomation_LabDeviceAccessQt_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register offline policy, lifecycle and cross-process lease regression cases. +void add_tests_LabDeviceAccess(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp new file mode 100644 index 0000000000..d6a7d8af2c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.cpp @@ -0,0 +1,98 @@ +/* Lab Device Runtime + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "LabDeviceRuntime.h" + +namespace PokemonAutomation{ +namespace{ + +#if defined(__APPLE__) +// Shared by all guarded apps for this user, independent of their profile/bundle. +std::string shared_lease_directory(){ + return (QDir::homePath() + "/Library/Application Support/PokemonAutomation/LabDeviceLeases").toStdString(); +} + +std::string current_serial_identity(std::string_view port){ + const QSerialPortInfo selected(QString::fromUtf8(port.data(), (qsizetype)port.size())); + std::vector devices; + for (const QSerialPortInfo& info : QSerialPortInfo::availablePorts()){ + if (info.hasVendorIdentifier() && info.hasProductIdentifier()){ + devices.emplace_back(LabSerialDevice{ + info.systemLocation().toStdString(), info.vendorIdentifier(), + info.productIdentifier(), info.serialNumber().toStdString() + }); + } + } + return resolve_lab_serial_identity(devices, selected.systemLocation().toStdString()); +} + +const LabDevicePolicy& runtime_policy(){ + static const LabDevicePolicy policy = []{ + const auto path = Filesystem::application_scratch_path() / "lab-devices.json"; + QFile file(QString::fromStdString(path.string())); + if (!file.open(QIODevice::ReadOnly)){ + throw std::runtime_error("Cannot read lab-devices.json; hardware access is disabled."); + } + return parse_lab_device_policy(file.readAll().toStdString(), Filesystem::STARTUP_PROFILE()); + }(); + return policy; +} +#endif + +} // namespace + +std::unique_ptr reserve_lab_profile(){ +#if defined(__APPLE__) + const std::string directory = shared_lease_directory(); + if (!QDir().mkpath(QString::fromStdString(directory))){ + throw std::runtime_error("Cannot create lab profile lease directory."); + } + return std::make_unique(lab_device_lock_path( + directory, {LabDeviceKind::PROFILE, Filesystem::STARTUP_PROFILE()} + )); +#else + return nullptr; +#endif +} + +std::unique_ptr acquire_lab_serial(Logger& logger, std::string_view port){ +#if defined(__APPLE__) + try{ + // Do not trust the UI's cached port list after unplug/replug. + return acquire_lab_device(runtime_policy(), shared_lease_directory(), { + LabDeviceKind::SERIAL, current_serial_identity(port) + }); + }catch (const std::runtime_error& error){ + throw UserSetupError(logger, error.what()); + } +#else + (void)logger; + (void)port; + return nullptr; +#endif +} + +std::unique_ptr acquire_lab_capture(Logger& logger, LabDeviceKind kind, std::string_view id){ +#if defined(__APPLE__) + try{ + return acquire_lab_device(runtime_policy(), shared_lease_directory(), {kind, std::string(id)}); + }catch (const std::runtime_error& error){ + throw UserSetupError(logger, error.what()); + } +#else + (void)logger; + (void)kind; + (void)id; + return nullptr; +#endif +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h new file mode 100644 index 0000000000..5dd5fb88ca --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Devices/LabDeviceRuntime.h @@ -0,0 +1,26 @@ +/* Lab Device Runtime + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabDeviceRuntime_H +#define PokemonAutomation_LabDeviceRuntime_H + +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "LabDeviceAccessQt.h" + +namespace PokemonAutomation{ + +// Reserve a macOS GUI profile before loading settings; throw std::runtime_error on failure. +// The returned lease must outlive the GUI. Other platforms return no reservation. +std::unique_ptr reserve_lab_profile(); + +// Resolve a fresh USB identity and reserve it before opening the requested port. +// Missing policy/identity or contention throws UserSetupError. Non-macOS is unchanged. +std::unique_ptr acquire_lab_serial(Logger& logger, std::string_view port); + +// Reserve a camera/audio input by the lowercase hex encoding of its Qt device ID. +// Caller holds the lease until after closing the input. Non-macOS is unchanged. +std::unique_ptr acquire_lab_capture(Logger& logger, LabDeviceKind kind, std::string_view id); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h index f9f6f5d1ff..dfc5ead1b4 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation.h @@ -7,12 +7,14 @@ #ifndef PokemonAutomation_HardwareValidation_H #define PokemonAutomation_HardwareValidation_H +#include "CommonFramework/Startup/SetupSettings.h" + namespace PokemonAutomation{ // Check user hardware. -// If the hardware is too old, send a QMessageBox to tell user and return false. -// If the hardware may not be powerful enough, send a QMessageBox to inform user. -bool check_hardware(); +// If the hardware is too old, send a QMessageBox (when `dialogs == SHOW`) and return false. +// If the hardware may not be powerful enough, send a QMessageBox (when `dialogs == SHOW`). +bool check_hardware(StartupDialogs dialogs = StartupDialogs::SHOW); } diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp index 7f716e8136..1d184838d6 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_arm64.tpp @@ -14,7 +14,8 @@ namespace PokemonAutomation{ -bool check_hardware(){ +bool check_hardware(StartupDialogs dialogs){ + (void)dialogs; // TODO: write computer power check for Apple M-series computers return true; diff --git a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp index 361ea8765c..d34f5a3361 100644 --- a/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp +++ b/SerialPrograms/Source/CommonFramework/Environment/HardwareValidation_x86.tpp @@ -11,36 +11,47 @@ #include "HardwareValidation.h" namespace PokemonAutomation{ +namespace{ + +void show_hardware_warning(const QString& message, StartupDialogs dialogs){ + if (dialogs == StartupDialogs::LOG_ONLY){ + return; + } + QMessageBox box; + box.warning(nullptr, "Warning", message); +} + +} // namespace -bool check_hardware(){ +bool check_hardware(StartupDialogs dialogs){ if (!CPU_CAPABILITY_NATIVE.OK_08_Nehalem){ - QMessageBox box; - box.critical( - nullptr, - "Error", - "This computer is too old to run this program.

" - "Please try a much newer computer.

" - "(Reason: SSE4.2 is required to run this program.)" - ); + if (dialogs == StartupDialogs::SHOW){ + QMessageBox box; + box.critical( + nullptr, + "Error", + "This computer is too old to run this program.

" + "Please try a much newer computer.

" + "(Reason: SSE4.2 is required to run this program.)" + ); + } return false; } if (!CPU_CAPABILITY_NATIVE.OK_13_Haswell){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; str += "(i.e. Increased error rate. Fail to reliably detect shinies, etc...)

"; str += "(Reason: Computers that lack AVX2 are likely to be too old and too slow to handle this program.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } ProcessorSpecs specs = get_processor_specs(); if (specs.threads < 4){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -49,11 +60,10 @@ bool check_hardware(){ str += QString::number(specs.threads); str += " vcores is a weak CPU.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } if (specs.base_frequency < 2'000'000'000){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -62,7 +72,7 @@ bool check_hardware(){ str += QString::fromStdString(tostr_fixed(specs.base_frequency / 1000000000., 3)); str += " GHz which is very slow.)

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } #if (defined _WIN32) || (defined __linux) @@ -71,7 +81,6 @@ bool check_hardware(){ : 0; double efreq = (specs.cores + 0.2 * vcores) * specs.base_frequency; if (efreq < 8'000'000'000){ - QMessageBox box; QString str; str += "This computer may not be powerful enough to run this program.

"; str += "You can continue, but the program may not work correctly.
"; @@ -87,7 +96,7 @@ bool check_hardware(){ str += QString::fromStdString(tostr_fixed(efreq / 1000000000., 3)); str += " GHz

"; str += "Recommendation: Use a more powerful computer."; - box.warning(nullptr, "Warning", str); + show_hardware_warning(str, dialogs); } #endif diff --git a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp index df38989010..37f37828cf 100644 --- a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp +++ b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.cpp @@ -119,6 +119,12 @@ GlobalSettings::GlobalSettings() LockMode::UNLOCK_WHILE_RUNNING, false ) + , CONSOLE_SETTINGS_COLLAPSED( + "Start with console settings collapsed:
" + "Each program's \"Console N Settings\" box starts folded up. Click its title to open it.", + LockMode::UNLOCK_WHILE_RUNNING, + false + ) , KEYBOARD_CONTROLS_LAYOUT( CONSTRUCT_TOKEN, "System Keyboard Layout:
" @@ -205,6 +211,7 @@ GlobalSettings::GlobalSettings() PA_ADD_OPTION(WINDOW_SIZE); PA_ADD_OPTION(LOG_WINDOW_SIZE); PA_ADD_OPTION(LOG_WINDOW_STARTUP); + PA_ADD_OPTION(CONSOLE_SETTINGS_COLLAPSED); PA_ADD_OPTION(KEYBOARD_CONTROLS_LAYOUT); PA_ADD_OPTION(DEVICE_ALIASES); diff --git a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h index 29ef1fff03..2afea6a082 100644 --- a/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h +++ b/SerialPrograms/Source/CommonFramework/GlobalSettingsPanel.h @@ -92,6 +92,7 @@ class GlobalSettings : public BatchOption, private ConfigOption::Listener, priva Pimpl WINDOW_SIZE; Pimpl LOG_WINDOW_SIZE; BooleanCheckBoxOption LOG_WINDOW_STARTUP; + BooleanCheckBoxOption CONSOLE_SETTINGS_COLLAPSED; Pimpl KEYBOARD_CONTROLS_LAYOUT; diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp new file mode 100644 index 0000000000..7afe27818f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.cpp @@ -0,0 +1,345 @@ +/* Headless Host + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_WIN32) +#include +#include +#endif +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/PersistentSettings.h" +#include "CommonFramework/ProgramSession.h" +#include "GameConsole/Framework/ConsoleProgramSession.h" +#include "GameConsole/Framework/ConsoleSystemSession.h" +#include "GameConsole/Framework/MultiConsoleProgramSession.h" +#include "GameConsole/Framework/MultiConsoleSystemSession.h" +#include "PanelCatalog.h" +#include "HeadlessHost.h" + +namespace PokemonAutomation{ +namespace{ + +constexpr std::chrono::seconds READINESS_TIMEOUT{10}; + +#if !defined(_WIN32) +constexpr std::chrono::seconds STOP_TIMEOUT{5}; +struct SignalState{ + std::atomic write_fd{-1}; + std::atomic last_signal{0}; +}; + +SignalState& signal_state(){ + static SignalState state; + return state; +} + +void posix_signal_handler(int sig){ + SignalState& s = signal_state(); + s.last_signal.store(sig, std::memory_order_relaxed); + const int fd = s.write_fd.load(std::memory_order_relaxed); + if (fd >= 0){ + const char byte = static_cast(sig); + const ssize_t ignored = ::write(fd, &byte, /*nbyte=*/1); + (void)ignored; + } +} + +class SignalPipe final{ +public: + SignalPipe(Logger& logger, std::function on_signal){ + if (::pipe(m_fds) != 0){ + logger.log("Failed to create signal pipe; SIGINT/SIGTERM will terminate immediately.", COLOR_ORANGE); + return; + } + for (int fd : m_fds){ + const int flags = ::fcntl(fd, F_GETFL, /*arg=*/0); + if (flags >= 0){ + ::fcntl(fd, F_SETFL, flags | O_NONBLOCK); + } + } + signal_state().write_fd.store(m_fds[1], std::memory_order_release); + struct sigaction sa{}; + sa.sa_handler = posix_signal_handler; + sigemptyset(&sa.sa_mask); + m_has_int = (::sigaction(SIGINT, &sa, &m_old_int) == 0); + m_has_term = (::sigaction(SIGTERM, &sa, &m_old_term) == 0); + + m_notifier = std::make_unique(m_fds[0], QSocketNotifier::Read); + QObject::connect(m_notifier.get(), &QSocketNotifier::activated, m_notifier.get(), [this, on_signal = std::move(on_signal)]{ + std::array buf; + while (::read(m_fds[0], buf.data(), buf.size()) > 0){ + } + on_signal(signal_state().last_signal.load(std::memory_order_relaxed)); + }); + } + + ~SignalPipe(){ + if (m_has_int){ + ::sigaction(SIGINT, &m_old_int, nullptr); + } + if (m_has_term){ + ::sigaction(SIGTERM, &m_old_term, nullptr); + } + signal_state().write_fd.store(-1, std::memory_order_release); + m_notifier.reset(); + for (int& fd : m_fds){ + if (fd >= 0){ + ::close(fd); + fd = -1; + } + } + } + +private: + int m_fds[2] = {-1, -1}; + struct sigaction m_old_int{}; + struct sigaction m_old_term{}; + bool m_has_int = false; + bool m_has_term = false; + std::unique_ptr m_notifier; +}; +#endif + +bool system_ready(GameConsole::ConsoleSystemSession& system){ + for (size_t i = 0; i < system.controllers(); i++){ + if (!system.controller(i).ready()){ + return false; + } + } + return true; +} + +bool controller_ready(ProgramSession& program){ + if (auto* single = dynamic_cast(&program)){ + return system_ready(single->system()); + } + if (auto* multi = dynamic_cast(&program)){ + for (size_t i = 0; i < multi->system().active_consoles(); i++){ + if (!system_ready(multi->system()[i])){ + return false; + } + } + } + return true; +} + +class HeadlessProgramObserver final : public ProgramSession::Listener{ +public: + HeadlessProgramObserver(ProgramSession& program, QObject& context, std::function on_stopped) + : m_program(program), m_context(context), m_on_stopped(std::move(on_stopped)) + {} + + bool saw_error() const{ return m_saw_error.load(std::memory_order_acquire); } + + virtual void state_change(ProgramState state) override{ + if (state == ProgramState::STOPPED){ + QMetaObject::invokeMethod(&m_context, [this]{ + if (m_program.current_state() == ProgramState::STOPPED){ + m_on_stopped(); + } + }, Qt::QueuedConnection); + } + } + virtual void stats_update(const StatsTracker*, const StatsTracker*) override{} + virtual void error(const std::string&) override{ m_saw_error.store(true, std::memory_order_release); } + virtual void download_error(const std::string&) override{} + virtual void download_added(std::shared_ptr) override{} + virtual void all_downloads_done() override{} + +private: + ProgramSession& m_program; + QObject& m_context; + std::function m_on_stopped; + std::atomic m_saw_error{false}; +}; + +std::expected merge_options(const JsonValue& base, const JsonValue& over){ + const JsonObject* over_obj = over.to_object(); + if (over_obj == nullptr){ + return std::unexpected("Options file must contain a JSON object."); + } + if (over_obj->get_value("ConsoleSetup") != nullptr){ + return std::unexpected("--options cannot override ConsoleSetup."); + } + JsonObject merged; + if (const JsonObject* base_obj = base.to_object()){ + merged = base_obj->clone(); + } + for (const auto& [key, value] : *over_obj){ + merged[key] = value.clone(); + } + return JsonValue(std::move(merged)); +} + +HeadlessExit load_headless_panel( + const std::vector>& lists, + const HeadlessRunOptions& options, + HeadlessHost& host, + Logger& logger +){ + std::optional match = find_panel(lists, options.program_id); + if (!match){ + logger.log(std::format("Unknown program identifier: \"{}\".", options.program_id), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + std::shared_ptr descriptor(std::move(match->entry.descriptor)); + + JsonValue json; + if (options.base_panel_json){ + json = options.base_panel_json->clone(); + }else if (const JsonValue* saved = PERSISTENT_SETTINGS().panels.get_value(descriptor->identifier())){ + json = saved->clone(); + } + if (options.options_override){ + auto merged = merge_options(json, *options.options_override); + if (!merged){ + logger.log(merged.error(), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + json = std::move(*merged); + } + + try{ + load_panel_session(descriptor, json, host); + }catch (const UserSetupError& e){ + logger.log(std::format("Device setup error: {}", e.to_str()), COLOR_RED); + return HeadlessExit::DEVICE; + }catch (const Exception& e){ + logger.log(std::format("Failed to load program: {}", e.to_str()), COLOR_RED); + return HeadlessExit::BAD_REQUEST; + } + return HeadlessExit::OK; +} + +bool wait_for_controller_ready(ProgramSession& program){ + if (controller_ready(program)){ + return true; + } + QEventLoop loop; + QTimer poll; + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&poll, &QTimer::timeout, &loop, [&]{ + if (controller_ready(program)){ + loop.quit(); + } + }); + QObject::connect(&timeout, &QTimer::timeout, &loop, [&]{ loop.quit(); }); + poll.start(std::chrono::milliseconds(50)); + timeout.start(READINESS_TIMEOUT); + loop.exec(); + return controller_ready(program); +} + +HeadlessExit run_program_event_loop(ProgramSession& program, bool handle_signals, Logger& logger){ + QEventLoop loop; + std::optional signal_exit; + QTimer stop_timer; + stop_timer.setSingleShot(true); + + HeadlessProgramObserver observer(program, loop, [&]{ loop.quit(); }); + program.add_listener(observer); + +#if !defined(_WIN32) + std::unique_ptr pipe; + if (handle_signals){ + pipe = std::make_unique(logger, [&](int sig){ + const HeadlessExit code = sig == SIGTERM ? HeadlessExit::SIGNAL_TERM : HeadlessExit::SIGNAL_INT; + if (signal_exit){ + logger.log("Second signal received; exiting immediately.", COLOR_RED); + std::_Exit(static_cast(code)); + } + signal_exit = code; + logger.log("Signal received; stopping program...", COLOR_ORANGE); + program.stop_program(ActionSource::CLI, std::nullopt); + QObject::connect(&stop_timer, &QTimer::timeout, &loop, [&logger, code]{ + logger.log("Stop timeout expired; exiting immediately.", COLOR_RED); + std::_Exit(static_cast(code)); + }); + stop_timer.start(STOP_TIMEOUT); + }); + } +#else + (void)handle_signals; +#endif + + const std::string err = program.start_program(ActionSource::CLI, std::nullopt); + if (!err.empty()){ + logger.log(std::format("Failed to start program: {}", err), COLOR_RED); + program.remove_listener(observer); + return HeadlessExit::DEVICE; + } + if (program.current_state() != ProgramState::STOPPED){ + loop.exec(); + } + program.remove_listener(observer); + if (signal_exit){ + return *signal_exit; + } + const std::optional last = program.last_transition(); + const bool program_error = observer.saw_error() || (last && last->source == ActionSource::PROGRAM_ERROR); + return program_error ? HeadlessExit::PROGRAM_ERROR : HeadlessExit::OK; +} + +} // namespace + + +void HeadlessHost::clear_panel(){ + m_current_panel.reset(); + m_current_panel_descriptor.reset(); +} + +void HeadlessHost::load_panel( + std::shared_ptr descriptor, + std::unique_ptr panel +){ + clear_panel(); + m_current_panel_descriptor = std::move(descriptor); + m_current_panel = std::move(panel); +} + +HeadlessExit run_headless_program( + const std::vector>& lists, + const HeadlessRunOptions& options +){ + Logger& logger = global_logger_tagged(); + HeadlessHost host; + const HeadlessExit load_status = load_headless_panel(lists, options, host, logger); + if (load_status != HeadlessExit::OK){ + return load_status; + } + ProgramSession* program = dynamic_cast(host.current_panel()); + if (program == nullptr){ + logger.log(std::format("Panel \"{}\" is not a runnable program.", options.program_id), COLOR_RED); + host.clear_panel(); + return HeadlessExit::BAD_REQUEST; + } + if (!wait_for_controller_ready(*program)){ + logger.log("Controller did not become ready before timeout.", COLOR_RED); + host.clear_panel(); + return HeadlessExit::DEVICE; + } + const HeadlessExit ret = run_program_event_loop(*program, options.handle_signals, logger); + host.clear_panel(); + return ret; +} + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h new file mode 100644 index 0000000000..28753ea339 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost.h @@ -0,0 +1,72 @@ +/* Headless Host + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_HeadlessHost_H +#define PokemonAutomation_HeadlessHost_H + +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/Panels/PanelList.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "CommonFramework/Panels/PanelTools.h" + +namespace PokemonAutomation{ + + +// Process exit codes returned by --headless --run. +enum class HeadlessExit : int{ + OK = 0, + PROGRAM_ERROR = 1, + STARTUP = 3, + BAD_REQUEST = 4, + DEVICE = 5, + SIGNAL_INT = 130, + SIGNAL_TERM = 143, +}; + + +// Widgetless PanelHolder for --headless runs. Never calls make_ui_component() +// or writes persistent settings. +class HeadlessHost final : public PanelHolder{ +public: + PanelSession* current_panel() const{ return m_current_panel.get(); } + + void clear_panel(); + + virtual bool report_new_panel_intent(const PanelDescriptor&) override{ return true; } + virtual void load_panel( + std::shared_ptr descriptor, + std::unique_ptr panel + ) override; + virtual void on_busy() override{} + virtual void on_idle() override{} + +private: + std::shared_ptr m_current_panel_descriptor; + std::unique_ptr m_current_panel; +}; + + +struct HeadlessRunOptions{ + std::string program_id; + std::optional options_override = std::nullopt; + std::optional base_panel_json = std::nullopt; + bool handle_signals = true; +}; + + +// Run one program from `lists` to completion under QCoreApplication::exec(). +HeadlessExit run_headless_program( + const std::vector>& lists, + const HeadlessRunOptions& options +); + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp new file mode 100644 index 0000000000..e7ef7d9d99 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.cpp @@ -0,0 +1,138 @@ +/* Headless Host Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/Panels/PanelDescriptor.h" +#include "CommonFramework/Panels/PanelList.h" +#include "CommonFramework/Panels/PanelSession.h" +#include "ComputerPrograms/ComputerProgram.h" +#include "HeadlessHost.h" +#include "HeadlessHost_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +enum class FakeBehavior{SUCCEED, REPORT_ERROR, REFUSE_START}; + +class FakeInstance final : public ComputerProgramInstance{ +public: + explicit FakeInstance(FakeBehavior b) : m_b(b) {} + virtual std::string check_validity() const override{ + return m_b == FakeBehavior::REFUSE_START ? "invalid configuration" : ""; + } + virtual void program(ProgramEnvironment&, CancellableScope&) override{ + if (m_b == FakeBehavior::REPORT_ERROR){ + throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "simulated failure"); + } + } +private: + FakeBehavior m_b; +}; + +class FakeProgramDescriptor final : public ComputerProgramDescriptor{ +public: + FakeProgramDescriptor(std::string id, FakeBehavior b) + : ComputerProgramDescriptor(std::move(id), "Fake", "FakeProgram", /*doc_link=*/"", /*description=*/"", PanelQualityStanding::DEVELOPER) + , m_b(b) + {} + virtual std::unique_ptr make_instance() const override{ + return std::make_unique(m_b); + } +private: + FakeBehavior m_b; +}; + +class FakeSettingsDescriptor final : public PanelDescriptor{ +public: + explicit FakeSettingsDescriptor(std::string id) + : PanelDescriptor(std::move(id), "Fake", "FakeSettings", /*doc_link=*/"", /*description=*/"", PanelQualityStanding::DEVELOPER, /*restore_defaults_button=*/false) + {} + virtual std::unique_ptr make_panel() const override{ + return std::make_unique(*this); + } +}; + +class FakeListDescriptor final : public PanelListDescriptor{ +public: + FakeListDescriptor() : PanelListDescriptor("Fake") {} +protected: + virtual std::vector make_panels() const override{ + std::vector ret; + ret.emplace_back(std::make_unique("Fake:Ok", FakeBehavior::SUCCEED)); + ret.emplace_back(std::make_unique("Fake:Err", FakeBehavior::REPORT_ERROR)); + ret.emplace_back(std::make_unique("Fake:Refused", FakeBehavior::REFUSE_START)); + ret.emplace_back(std::make_unique("Fake:Settings")); + return ret; + } +}; + +struct RunInput{ + std::string_view program_id; + std::optional options_json = std::nullopt; +}; + +struct RunExpectation{ + HeadlessExit exit_code; +}; + +struct RunCase{ + std::string_view name; + RunInput input; + RunExpectation expectation; +}; + +const RunCase RUN_CASES[] = { + {.name = "clean program finishes 0", .input = {.program_id = "Fake:Ok"}, .expectation = {.exit_code = HeadlessExit::OK}}, + {.name = "program error finishes 1", .input = {.program_id = "Fake:Err"}, .expectation = {.exit_code = HeadlessExit::PROGRAM_ERROR}}, + {.name = "unknown identifier finishes 4", .input = {.program_id = "Fake:Missing"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "settings panel finishes 4", .input = {.program_id = "Fake:Settings"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "refused start finishes 5", .input = {.program_id = "Fake:Refused"}, .expectation = {.exit_code = HeadlessExit::DEVICE}}, + {.name = "ConsoleSetup override finishes 4", .input = {.program_id = "Fake:Ok", .options_json = "{\"ConsoleSetup\": {}}"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, + {.name = "non-object override finishes 4", .input = {.program_id = "Fake:Ok", .options_json = "42"}, .expectation = {.exit_code = HeadlessExit::BAD_REQUEST}}, +}; + +UnitTestResult check_run_case(const RunCase& test){ + std::vector> lists; + lists.push_back(std::make_unique()); + HeadlessRunOptions options{ + .program_id = std::string(test.input.program_id), + .base_panel_json = JsonValue(), + .handle_signals = false, + }; + if (test.input.options_json){ + options.options_override = parse_json(std::string(*test.input.options_json)); + } + const HeadlessExit code = run_headless_program(lists, options); + if (code != test.expectation.exit_code){ + return std::format("Expected exit code {}; got {}.", static_cast(test.expectation.exit_code), static_cast(code)); + } + return true; +} + +class RunTest final : public UnitTest{ +public: + explicit RunTest(const RunCase& test) + : UnitTest(std::format("HeadlessHost/Run/{}", test.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_run_case(m_test); + } +private: + const RunCase& m_test; +}; + +} // namespace + +void add_tests_HeadlessHost(UnitTestDatabase& database){ + for (const RunCase& test : RUN_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h new file mode 100644 index 0000000000..6107fb361f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Headless/HeadlessHost_Tests.h @@ -0,0 +1,17 @@ +/* Headless Host Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_HeadlessHost_Tests_H +#define PokemonAutomation_HeadlessHost_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +void add_tests_HeadlessHost(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Main.cpp b/SerialPrograms/Source/CommonFramework/Main.cpp index c493f48b54..372dae0189 100644 --- a/SerialPrograms/Source/CommonFramework/Main.cpp +++ b/SerialPrograms/Source/CommonFramework/Main.cpp @@ -1,5 +1,9 @@ #include +#include +#include +#include +#include //#include #include #include "Common/Cpp/Logging/LastLogTracker.h" @@ -17,7 +21,9 @@ #include "Common/Cpp/ScopeExit.h" #include "Common/Qt/GlobalThreadPoolsQt.h" #include "StaticRegistration.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" #include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Headless/HeadlessHost.h" #include "CommonFramework/Tools/GlobalThreadPools.h" #include "CommonFramework/Panels/OptionsPanel.h" #include "CommonFramework/VideoPipeline/Backends/CameraImplementations.h" @@ -41,17 +47,21 @@ #include "Logging/Logger.h" #include "Logging/OutputRedirector.h" #include "Startup/SetupSettings.h" +#include "Startup/StartupArguments.h" +#include "Startup/LabStartupPolicy.h" #include "Startup/NewVersionCheck.h" #include "Windows/MainWindow.h" #include "GlobalSettingsPanel.h" #include "GlobalServices.h" #include "PersistentSettings.h" +#include "PanelCatalog.h" #include "NintendoSwitch/NintendoSwitch_SettingsPanel.h" -using namespace PokemonAutomation; - Q_DECLARE_METATYPE(std::string) +namespace PokemonAutomation{ +namespace{ + void set_working_directory(){ const std::string app_dir = Filesystem::application_binary_directory().string_slash_normalized(); const std::string bundle_dir = "/Contents/MacOS"; @@ -66,10 +76,6 @@ void set_working_directory(){ } } -namespace PokemonAutomation{ - - - FileLogger& global_file_logger(){ static FileLogger logger( GlobalThreadPools::unlimited_normal(), @@ -80,19 +86,47 @@ FileLogger& global_file_logger(){ return logger; } -} +bool init_common_runtime(Logger& logger, StartupDialogs dialogs, bool fail_on_settings_error){ + qRegisterMetaType("size_t"); + qRegisterMetaType("uint8_t"); + qRegisterMetaType("std::string"); + qRegisterMetaType("Resolution"); + + Filesystem::create_directories(Filesystem::Path(SETTINGS_PATH())); + Filesystem::create_directories(Filesystem::Path(SCREENSHOTS_PATH())); + + // Several novice developers struggled to build and run the program due to missing Resources folder. + // Add this check to pop a message box when Resources folder is missing. + if (!check_resource_folder(logger, dialogs)){ + return false; + } + + // Read program settings from json file: SerialPrograms-Settings.json. + try{ + PERSISTENT_SETTINGS().read(); + }catch (const FileException& error){ + logger.log(error.message(), COLOR_RED); + if (fail_on_settings_error){ + return false; + } + }catch (const ParseException& error){ + logger.log(error.message(), COLOR_RED); + if (fail_on_settings_error){ + return false; + } + } + + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + return true; +} int run_program(int argc, char *argv[]){ // qputenv("QT_RHI_BACKEND", "opengl"); -#if defined(__APPLE__) - Filesystem::set_startup_profile(argc, argv); - QApplication application(argc, argv); -#else QApplication application(argc, argv); -#endif #ifdef PA_DPP // Configure OpenSSL before DPP or background tasks create a TLS context. @@ -164,26 +198,18 @@ int run_program(int argc, char *argv[]){ logger.log(dpp_cert_message, dpp_cert_error ? COLOR_RED : Color()); #endif - qRegisterMetaType("size_t"); - qRegisterMetaType("uint8_t"); - qRegisterMetaType("std::string"); - qRegisterMetaType("Resolution"); - #if defined(__linux) || defined(__APPLE__) // By default Qt uses native menubar but this only works on Windows. // We use menubar in our ButtonDiagram window to choose which controller's button mapping image to show. // So we fix it by don't using native menubar on non-Windows OS. QCoreApplication::setAttribute(Qt::AA_DontUseNativeMenuBar); #endif - -//#if QT_VERSION_MAJOR == 5 // AA_EnableHighDpiScaling is deprecated in Qt6 -// QApplication::setAttribute(Qt::AA_EnableHighDpiScaling); -//#endif - - Filesystem::create_directories(Filesystem::Path(SETTINGS_PATH())); - Filesystem::create_directories(Filesystem::Path(SCREENSHOTS_PATH())); - - + if (!init_common_runtime(logger, StartupDialogs::SHOW, /*fail_on_settings_error=*/false)){ + return 1; + } + if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){ + return run_command_line_tests(false); + } ScopeExit cleanup([]{ #ifdef PA_SDL @@ -193,49 +219,10 @@ int run_program(int argc, char *argv[]){ GlobalMediaServices::instance().stop(); }); - // Several novice developers struggled to build and run the program due to missing Resources folder. - // Add this check to pop a message box when Resources folder is missing. - if (!check_resource_folder(logger)){ - return 1; - } - - // Read program settings from json file: SerialPrograms-Settings.json. - try{ - PERSISTENT_SETTINGS().read(); - }catch (const FileException& error){ - logger.log(error.message(), COLOR_RED); - }catch (const ParseException& error){ - logger.log(error.message(), COLOR_RED); - } - - // Load the Switch framework settings now, not only when their panel is opened. OptionsPanelWrapper().make_panel()->load_json_from_global(); - - // Preload a bunch of stuff now so they are ready later. SerialPortPoller::instance().ports(); get_all_cameras(); - // Force all the Qt thread pools to be constructed now on the main thread. - GlobalThreadPools::qt_worker_threadpool(); - GlobalThreadPools::qt_event_threadpool(); - - bool augment_test_color_distortions = false; - for (size_t i = 0; i < argc; i++){ - constexpr const char* force_run_tests = "--command-line-test-mode"; - constexpr const char* distort_test_images = "--augment-color-distortions"; - - if (strcmp(argv[i], force_run_tests) == 0){ - GlobalSettings::instance().COMMAND_LINE_TEST_MODE = true; - } - if (strcmp(argv[i], distort_test_images) == 0){ - augment_test_color_distortions = true; - } - } - - if (GlobalSettings::instance().COMMAND_LINE_TEST_MODE){ - return run_command_line_tests(augment_test_color_distortions); - } - // Check whether the hardware is powerful enough to run this program. if (!check_hardware()){ return 1; @@ -283,29 +270,8 @@ int run_program(int argc, char *argv[]){ } -int main(int argc, char *argv[]){ -#if 0 - // Retrieve and store program name - set_program_path(argv[0]); -#endif - -#if defined(__linux__) - // Qt multimedia, default to gstreamer to prevent flickering - // Easier than the alternative which is compiling qt6multimedia with QT_DEFAULT_MEDIA_BACKEND - // See: https://doc.qt.io/qt-6.5/qtmultimedia-index.html - if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){ - qputenv("QT_MEDIA_BACKEND", "gstreamer"); - } -#endif - - // So far, this is only needed on Mac where static initialization is fucked up. - PokemonAutomation::register_all_statics(); - - setup_crash_handler(); - - int ret = run_program(argc, argv); - - +// Stop GUI services after run_program() returns, preserving the existing order. +void stop_application_services(){ #ifdef PA_SOCIAL_SDK Integration::DiscordSocialSDK::DiscordSocial::instance().stop(); #endif @@ -328,6 +294,47 @@ int main(int argc, char *argv[]){ global_watchdog().stop(); global_file_logger().stop(); +} + +// Join the worker pools after service shutdown and the Windows Qt workaround. +void stop_thread_pools(){ + // Force stop the thread pools. + // This is where all the threads in the program are joined. + PokemonAutomation::GlobalThreadPools::computation_realtime().stop(); + PokemonAutomation::GlobalThreadPools::computation_normal().stop(); + PokemonAutomation::GlobalThreadPools::unlimited_realtime().stop(); + PokemonAutomation::GlobalThreadPools::unlimited_normal().stop(); + PokemonAutomation::global_dispatcher.stop(); + + GlobalThreadPools::qt_worker_threadpool().stop(); + GlobalThreadPools::qt_event_threadpool().stop(); + +} + +int run_gui_application(int argc, char *argv[]){ +#if 0 + // Retrieve and store program name + set_program_path(argv[0]); +#endif + +#if defined(__linux__) + // Qt multimedia, default to gstreamer to prevent flickering + // Easier than the alternative which is compiling qt6multimedia with QT_DEFAULT_MEDIA_BACKEND + // See: https://doc.qt.io/qt-6.5/qtmultimedia-index.html + if (qEnvironmentVariableIsEmpty("QT_MEDIA_BACKEND")){ + qputenv("QT_MEDIA_BACKEND", "gstreamer"); + } +#endif + + // So far, this is only needed on Mac where static initialization is fucked up. + PokemonAutomation::register_all_statics(); + + setup_crash_handler(); + + int ret = run_program(argc, argv); + + + stop_application_services(); // // Workaround Qt 6.9 thread-adoption bug on Windows. // https://github.com/PokemonAutomation/Arduino-Source/issues/570 @@ -340,18 +347,162 @@ int main(int argc, char *argv[]){ std::quick_exit(ret); #endif - // Force stop the thread pools. - // This is where all the threads in the program are joined. - PokemonAutomation::GlobalThreadPools::computation_realtime().stop(); - PokemonAutomation::GlobalThreadPools::computation_normal().stop(); - PokemonAutomation::GlobalThreadPools::unlimited_realtime().stop(); - PokemonAutomation::GlobalThreadPools::unlimited_normal().stop(); - PokemonAutomation::global_dispatcher.stop(); - - GlobalThreadPools::qt_worker_threadpool().stop(); - GlobalThreadPools::qt_event_threadpool().stop(); + stop_thread_pools(); std::cout << "Exiting main()..." << std::endl; return ret; } + +bool load_headless_options(const StartupPlan& plan, Logger& logger, std::optional& out){ + if (!plan.options_path){ + return true; + } + try{ + out = load_json_file(*plan.options_path); + return true; + }catch (const Exception& e){ + logger.log("Failed to read --options \"" + *plan.options_path + "\": " + e.to_str(), COLOR_RED); + return false; + } +} + +int run_headless_application(QApplication& application, const StartupPlan& plan){ + ScopeExit service_cleanup([]{ + stop_application_services(); + stop_thread_pools(); + }); + + GlobalOutputRedirector redirect_stdout(std::cout, "stdout", Color()); + GlobalOutputRedirector redirect_stderr(std::cerr, "stderr", COLOR_RED); + { + MultiOutputLogger& logger = global_multi_logger(); + logger.add_listener(global_last_log_history()); + logger.add_listener(global_file_logger()); + } + + Logger& logger = global_logger_tagged(); + logger.log("================================================================================"); + logger.log("Starting Headless Program: " + *plan.run_program); + + std::optional options_override; + if (!load_headless_options(plan, logger, options_override)){ + return static_cast(HeadlessExit::BAD_REQUEST); + } + + if (!init_common_runtime(logger, StartupDialogs::LOG_ONLY, /*fail_on_settings_error=*/true)){ + return static_cast(HeadlessExit::STARTUP); + } + + ScopeExit device_cleanup([]{ + SerialPortPoller::instance().stop(); + GlobalMediaServices::instance().stop(); + }); + OptionsPanelWrapper().make_panel()->load_json_from_global(); + if (!check_hardware(StartupDialogs::LOG_ONLY)){ + return static_cast(HeadlessExit::STARTUP); + } + + set_working_directory(); + set_permissions(application); + return static_cast(run_headless_program( + make_all_panel_lists(), + HeadlessRunOptions{ + .program_id = *plan.run_program, + .options_override = std::move(options_override), + .handle_signals = true, + } + )); +} + +// Explicit tests use QCoreApplication and never enter GUI/settings/device startup. +int run_test_application(int argc, char* argv[], const StartupPlan& plan){ +#if defined(__APPLE__) + Filesystem::set_startup_profile("unit-tests"); +#endif + QCoreApplication application(argc, argv); + register_all_statics(); + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + const int ret = run_command_line_tests(plan.augment_color_distortions); + OCR::clear_ocr_cache(); + global_periodic_runner().stop(); + global_watchdog().stop(); +#ifdef PA_ENABLE_QT_ADOPTION_WORKAROUND + std::quick_exit(ret); +#endif + stop_thread_pools(); + return ret; +} + +int run_with_profile_lease( + const PokemonAutomation::StartupPlan& selected, + int lease_error_exit, + const std::function& launch +){ +#if defined(__APPLE__) + PokemonAutomation::Filesystem::set_startup_profile(*selected.profile); +#else + (void)selected; +#endif + std::unique_ptr profile_lease; + try{ + profile_lease = PokemonAutomation::reserve_lab_profile(); + }catch (const std::runtime_error& error){ + std::cerr << error.what() << std::endl; + return lease_error_exit; + } + return launch(); +} + +} // namespace +} // namespace PokemonAutomation + +// Validate before even constructing Qt: a typo must never become a hardware launch. +int main(int argc, char* argv[]){ + std::vector arguments; + for (int i = 1; i < argc; i++){ + arguments.emplace_back(argv[i]); + } + PokemonAutomation::LabStartupPolicy policy; +#if defined(__APPLE__) + policy.profiles = PokemonAutomation::LabProfileMode::REQUIRED; +#ifdef PA_LAB_PROFILE + policy.bound_profile = PA_LAB_PROFILE; +#endif +#endif + const PokemonAutomation::StartupPlan plan = PokemonAutomation::apply_lab_startup_policy(PokemonAutomation::parse_startup_arguments(arguments), policy); + if (plan.mode == PokemonAutomation::StartupMode::REJECTED){ + std::cerr << plan.error << std::endl; + } + // Keep backing storage alive for Qt, which may mutate argc and argv. + std::vector qt_storage = plan.qt_arguments; + qt_storage.insert(qt_storage.begin(), argv[0]); + std::vector qt_argv; + for (std::string& arg : qt_storage){ + qt_argv.push_back(arg.data()); + } + int qt_argc = (int)qt_argv.size(); + qt_argv.push_back(nullptr); + return PokemonAutomation::dispatch_startup(plan, { + .gui = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_with_profile_lease(selected, 2, [&]{ + return PokemonAutomation::run_gui_application(qt_argc, qt_argv.data()); + }); + }, + .headless = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_with_profile_lease( + selected, static_cast(PokemonAutomation::HeadlessExit::STARTUP), + [&]{ + PokemonAutomation::register_all_statics(); + PokemonAutomation::setup_crash_handler(); + QApplication application(qt_argc, qt_argv.data()); + return PokemonAutomation::run_headless_application(application, selected); + } + ); + }, + .tests = [&](const PokemonAutomation::StartupPlan& selected){ + return PokemonAutomation::run_test_application(qt_argc, qt_argv.data(), selected); + }, + }); +} diff --git a/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp b/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp index ae16768c86..0ebce8d7e7 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp +++ b/SerialPrograms/Source/CommonFramework/Panels/PanelList.cpp @@ -6,6 +6,7 @@ #include "Common/Cpp/Exceptions.h" #include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/Panels/PanelSession.h" #include "PanelList.h" //#include @@ -33,6 +34,19 @@ PanelListDescriptor::PanelListDescriptor( } } +bool load_panel_session( + const std::shared_ptr& descriptor, + const JsonValue& panel_json, + PanelHolder& holder +){ + if (!holder.report_new_panel_intent(*descriptor)){ + return false; + } + std::unique_ptr panel = descriptor->make_panel(); + panel->load_json(panel_json); + holder.load_panel(descriptor, std::move(panel)); + return true; +} diff --git a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h index 995b028763..0ab5f36dca 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/PanelList.h +++ b/SerialPrograms/Source/CommonFramework/Panels/PanelList.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_PanelList_H #define PokemonAutomation_PanelList_H +#include #include #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/Panels/PanelTools.h" @@ -35,6 +36,9 @@ class PanelListDescriptor{ PanelListWidget* make_QWidget(QWidget& parent, PanelHolder& holder) const; + // The entries of the list, as the widget shows them. + std::vector panels() const{ return make_panels(); } + protected: virtual std::vector make_panels() const = 0; @@ -45,6 +49,16 @@ class PanelListDescriptor{ }; +// Instantiate `descriptor` (non-null), load `panel_json` into it, and install +// it in `holder`. Returns false, loading nothing, if +// `holder.report_new_panel_intent()` declines. +bool load_panel_session( + const std::shared_ptr& descriptor, + const JsonValue& panel_json, + PanelHolder& holder +); + + } #endif diff --git a/SerialPrograms/Source/CommonFramework/Panels/ProgramDescriptor.h b/SerialPrograms/Source/CommonFramework/Panels/ProgramDescriptor.h index b8dcbc06f1..8bd01f8ffa 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/ProgramDescriptor.h +++ b/SerialPrograms/Source/CommonFramework/Panels/ProgramDescriptor.h @@ -7,6 +7,9 @@ #ifndef PokemonAutomation_CommonFramework_ProgramDescriptor_H #define PokemonAutomation_CommonFramework_ProgramDescriptor_H +#include +#include +#include #include "Common/Cpp/Options/BatchOption.h" #include "CommonFramework/Notifications/EventNotificationOption.h" #include "PanelDescriptor.h" @@ -30,6 +33,30 @@ enum class AllowCommandsWhenRunning{ ENABLE_COMMANDS, }; +// One check a program runs before it starts, as it is listed without +// running it: its stable name (e.g. "free_party_slots"), one line saying +// what the program needs, and whether the user can turn it off. +struct PrecheckSummary{ + std::string name; + std::string needs; + bool skippable = false; +}; + +// How one precheck ended in a checks-only run. +enum class PrecheckReportStatus{ + PASSED, + FAILED, + SKIPPED, +}; + +// What one precheck found in a checks-only run. On FAILED, the message +// tells the user what to fix. +struct PrecheckReport{ + std::string name = {}; + PrecheckReportStatus status = PrecheckReportStatus::PASSED; + std::string message = {}; +}; + class ProgramDescriptor : public PanelDescriptor{ @@ -87,6 +114,12 @@ class ProgramInstance{ ProgramEnvironment& env ){} + // The program's prechecks in the order it runs them, or nothing when it + // declares no precheck list. Never touches the console. + virtual std::optional> precheck_summaries() const{ + return std::nullopt; + } + protected: BatchOption m_options; void add_option(ConfigOption& option, std::string serialization_string); diff --git a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.cpp b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.cpp index a61d84b49c..bbbc9675dd 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.cpp +++ b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.cpp @@ -247,19 +247,7 @@ RunnablePanelActionBar::RunnablePanelActionBar( connect( m_start_button, &QPushButton::clicked, this, [&](bool){ - std::string error; - switch (program_session.current_state()){ - case ProgramState::STOPPED: - error = program_session.start_program(); - break; - case ProgramState::RUNNING: - error = program_session.stop_program(); - break; - default:; - } - if (!error.empty()){ - program_session.report_error(error); - } + start_or_stop_program(program_session); } ); connect( @@ -277,6 +265,21 @@ RunnablePanelActionBar::RunnablePanelActionBar( } ); } +void start_or_stop_program(ProgramSession& program_session){ + std::string error; + switch (program_session.current_state()){ + case ProgramState::STOPPED: + error = program_session.start_program(ActionSource::UI, std::nullopt); + break; + case ProgramState::RUNNING: + error = program_session.stop_program(ActionSource::UI, std::nullopt); + break; + default:; + } + if (!error.empty()){ + program_session.report_error(error); + } +} void RunnablePanelActionBar::set_state(ProgramState state){ m_last_known_state = state; switch (state){ diff --git a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.h b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.h index 550417e0e8..553cc4fed0 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.h +++ b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelElements.h @@ -59,6 +59,10 @@ class StatsBar : public QLabel{ +// What clicking Start/Stop Program does: start the program if it is stopped, +// stop it if it is running, and report any error to the program's listeners. +void start_or_stop_program(ProgramSession& program_session); + class RunnablePanelActionBar : public QGroupBox{ public: RunnablePanelActionBar( diff --git a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp index 387ca75331..277bdfebba 100644 --- a/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/Panels/UI/PanelListWidget.cpp @@ -93,18 +93,16 @@ void PanelListWidget::handle_panel_clicked(const std::string& text){ PERSISTENT_SETTINGS().panels[JSON_PROGRAM_PANEL] = ""; return; } - std::shared_ptr& descriptor = iter->second; + const std::shared_ptr& descriptor = iter->second; // cout << descriptor->display_name() << endl; - if (!m_panel_holder.report_new_panel_intent(*descriptor)){ - return; - } try{ - std::unique_ptr panel = descriptor->make_panel(); -// try{ - panel->load_json(PERSISTENT_SETTINGS().panels[descriptor->identifier()]); -// }catch (ParseException&){} - m_panel_holder.load_panel(descriptor, std::move(panel)); - + if (!load_panel_session( + descriptor, + PERSISTENT_SETTINGS().panels[descriptor->identifier()], + m_panel_holder + )){ + return; + } PERSISTENT_SETTINGS().panels[JSON_PROGRAM_PANEL] = iter->first; }catch (const Exception& error){ QMessageBox box; diff --git a/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.cpp b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.cpp new file mode 100644 index 0000000000..8b378de3ad --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.cpp @@ -0,0 +1,255 @@ +/* Program Precheck Run + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "ProgramPrecheckRun.h" + +namespace PokemonAutomation{ + +namespace{ + + +std::string key_path(std::string_view prefix, std::string_view key){ + return prefix.empty() ? std::string(key) : std::format("{}.{}", prefix, key); +} + +// Puts the values of `candidate` into `target`, an object, recursing into +// groups. `prefix` is the key path of `target`. +std::optional merge_into(const JsonObject& candidate, std::string_view prefix, JsonObject& target){ + for (const auto& item : candidate){ + const std::string path = key_path(prefix, item.first); + JsonValue* slot = target.get_value(item.first); + if (slot == nullptr){ + return RefusedOption{path, std::format("The program has no option {}.", path)}; + } + JsonObject* group = slot->to_object(); + const JsonObject* values = item.second.to_object(); + if (group != nullptr && values != nullptr){ + std::optional refused = merge_into(*values, path, *group); + if (refused){ + return refused; + } + continue; + } + if (group != nullptr || values != nullptr){ + return RefusedOption{path, std::format("Option {} is a group: its value is an object of its own options.", path)}; + } + *slot = item.second.clone(); + } + return std::nullopt; +} + +bool same_value(const JsonValue& x, const JsonValue& y){ + if (x.is_float() && y.is_float()){ + return x.to_double_default() == y.to_double_default(); + } + return x == y; +} + +std::optional find_refused_in(const JsonObject& candidate, const JsonObject& loaded, std::string_view prefix){ + for (const auto& item : candidate){ + const std::string path = key_path(prefix, item.first); + const JsonValue* now = loaded.get_value(item.first); + if (now == nullptr){ + return RefusedOption{path, std::format("The program has no option {}.", path)}; + } + const JsonObject* values = item.second.to_object(); + const JsonObject* group = now->to_object(); + if (values != nullptr && group != nullptr){ + std::optional refused = find_refused_in(*values, *group, path); + if (refused){ + return refused; + } + continue; + } + if (!same_value(item.second, *now)){ + return RefusedOption{ + path, + std::format("The program holds {} for {}, not {}.", now->dump(-1), path, item.second.dump(-1)) + }; + } + } + return std::nullopt; +} + + +// `current` (a program's option JSON) with the values of `candidate` put in. +std::expected merge_candidate_options( + const JsonValue& current, const JsonValue& candidate +){ + const JsonObject* values = candidate.to_object(); + if (values == nullptr){ + return std::unexpected(RefusedOption{"", "The options to check must be a JSON object."}); + } + JsonValue merged = current.clone(); + JsonObject* target = merged.to_object(); + if (target == nullptr){ + return std::unexpected(RefusedOption{"", "The program's options are not a JSON object."}); + } + std::optional refused = merge_into(*values, "", *target); + if (refused){ + return std::unexpected(std::move(*refused)); + } + return merged; +} + +// The first option of `candidate` whose value `loaded` (the options read +// back after loading the merged candidate) does not hold. +std::optional find_refused_option( + const JsonValue& candidate, const JsonValue& loaded +){ + const JsonObject* values = candidate.to_object(); + const JsonObject* now = loaded.to_object(); + if (values == nullptr || now == nullptr){ + return RefusedOption{"", "The options to check must be a JSON object."}; + } + return find_refused_in(*values, *now, ""); +} + + + + + +// The top-level keys of `x` and `y` (option JSON objects) whose values +// differ or that only one has, in key order. +std::vector changed_keys(const JsonValue& x, const JsonValue& y){ + std::vector keys; + const JsonObject* a = x.to_object(); + const JsonObject* b = y.to_object(); + if (a == nullptr || b == nullptr){ + return keys; + } + for (const auto& item : *a){ + const JsonValue* other = b->get_value(item.first); + if (other == nullptr || *other != item.second){ + keys.push_back(item.first); + } + } + for (const auto& item : *b){ + if (a->get_value(item.first) == nullptr){ + keys.push_back(item.first); + } + } + return keys; +} + +std::string key_list(const JsonValue& candidate){ + std::string keys; + const JsonObject* object = candidate.to_object(); + if (object == nullptr){ + return keys; + } + for (const auto& item : *object){ + keys += keys.empty() ? item.first : std::format(", {}", item.first); + } + return keys; +} + +// The options a checks-only run staged and must put back. +struct StagedOptions{ + std::optional prior = {}; + std::optional staged = {}; +}; + +// Step 1 of run_prechecks_only(), on the options thread. +void stage_options(const JsonValue& candidate, PrecheckOnlyTarget& target, StagedOptions& options, PrecheckOnlyRun& run){ + options.prior = target.options(); + std::expected merged = merge_candidate_options(*options.prior, candidate); + if (!merged){ + run.refused = std::move(merged.error()); + return; + } + target.load_options(*merged); + options.staged = target.options(); + run.refused = find_refused_option(candidate, *options.staged); + if (run.refused){ + return; + } + std::string error = target.check_validity(); + if (!error.empty()){ + run.refused = RefusedOption{"", std::move(error)}; + } +} + +// Step 3 of run_prechecks_only(). Never throws. +void restore_options( + const StagedOptions& options, const OptionsThreadCall& on_options_thread, + PrecheckOnlyTarget& target, Logger& logger, PrecheckOnlyRun& run +){ + run.restored = false; + if (!options.prior){ + return; + } + try{ + on_options_thread([&]{ + const JsonValue current = target.options(); + if (options.staged && current != *options.staged){ + run.edited_keys = changed_keys(*options.staged, current); + return; + } + target.load_options(*options.prior); + run.restored = target.options() == *options.prior; + }); + }catch (Exception& e){ + logger.log(std::format("Start checks only: the prior options did not load back: {}", e.message()), COLOR_RED); + return; + }catch (std::exception& e){ + logger.log(std::format("Start checks only: the prior options did not load back: {}", e.what()), COLOR_RED); + return; + } + if (!run.edited_keys.empty()){ + logger.log("Start checks only: options changed during the checks, kept as they are.", COLOR_RED); + }else if (!run.restored){ + logger.log("Start checks only: the prior options did not load back.", COLOR_RED); + } +} + + +} // namespace + + +PrecheckOnlyRun run_prechecks_only( + const JsonValue& candidate, + const OptionsThreadCall& on_options_thread, + PrecheckOnlyTarget& target, + Logger& logger +){ + logger.log(std::format("Start checks only, with options: {}", key_list(candidate))); + PrecheckOnlyRun run; + StagedOptions options; + try{ + on_options_thread([&]{ + stage_options(candidate, target, options, run); + }); + if (!run.refused){ + run.checks = target.run_checks(); + } + }catch (OperationCancelledException&){ + logger.log("Start checks only: cancelled.", COLOR_RED); + run.checks.reset(); + run.cancelled = true; + }catch (...){ + restore_options(options, on_options_thread, target, logger, run); + throw; + } + if (!options.staged){ + // Refused before anything was loaded. + run.restored = true; + return run; + } + restore_options(options, on_options_thread, target, logger, run); + return run; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.h b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.h new file mode 100644 index 0000000000..0150a4ce65 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun.h @@ -0,0 +1,110 @@ +/* Program Precheck Run + * + * From: https://github.com/PokemonAutomation/ + * + * A checks-only run of a program (ProgramSession::Prechecks::run_only()): + * candidate options are loaded over the current ones, the program's + * prechecks run without starting it, and the prior options are loaded back + * unless a user changed them meanwhile. Options are loaded and read back on + * the thread that owns them (the UI thread in the app); the checks run on + * the calling thread. + * The lab's "preflight" command uses it to find a setup mistake before a run + * resets or saves anything. + * + */ + +#ifndef PokemonAutomation_CommonFramework_ProgramPrecheckRun_H +#define PokemonAutomation_CommonFramework_ProgramPrecheckRun_H + +#include +#include +#include +#include +#include "Common/Cpp/Json/JsonValue.h" +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "CommonFramework/Panels/ProgramDescriptor.h" + +namespace PokemonAutomation{ + + +// A candidate option the program refused, and why. The key is a dotted key +// path ("NOTIFICATIONS.Enabled"); it is empty when the program refused the +// combination of options (check_validity()), which names no key. +struct RefusedOption{ + std::string key = {}; + std::string message = {}; +}; + +// What a checks-only run found. +struct PrecheckOnlyRun{ + // Set when the program refused the candidate options; no check ran. + std::optional refused = {}; + // The checks in the order they ran, up to the first FAILED one. Nothing + // when the program declares no precheck list, or refused the options. + std::optional> checks = {}; + // Whether the program holds the prior options again afterwards. + bool restored = false; + // The top-level option keys a user changed while the checks ran. The + // run then leaves the options as the user left them, and `restored` is + // false. + std::vector edited_keys = {}; + // Set when the checks were cancelled (OperationCancelledException) + // before they finished; `checks` is then nothing. + bool cancelled = false; +}; + + +// What a checks-only run acts on: a program's options and its prechecks. +class PrecheckOnlyTarget{ +public: + virtual ~PrecheckOnlyTarget() = default; + + // The program's option JSON. Called on the options thread. + virtual JsonValue options() const = 0; + // Loads option JSON into the program. Called on the options thread. + virtual void load_options(const JsonValue& options) = 0; + // Why the program refuses the loaded options as a whole, or "" when it + // takes them (ProgramInstance::check_validity()). Called on the options + // thread. + virtual std::string check_validity() const = 0; + // Runs the program's prechecks on the loaded options, on the calling + // thread. Nothing when the program declares no precheck list. + virtual std::optional> run_checks() = 0; +}; + +// Runs `body` on the thread that owns the program's options and returns +// once it has run. Throws what `body` throws, or when it cannot run it. +// It must still run `body` once the checks are cancelled: that is how the +// prior options get back after a cancel. +using OptionsThreadCall = std::function& body)>; + +// A checks-only run of `target` with `candidate`, an object of option keys +// to change; a group's value is an object of the group's own keys, of which +// only those given change: +// 1. On the options thread: loads the candidate over the current options. +// Refuses a key the program does not have, a candidate that is not an +// object, a value the program does not hold once loaded (a number +// clamped to its bounds, a disabled or unknown choice ignored; 2 and +// 2.0 are the same) and a combination it refuses (check_validity()). +// A refused key is a dotted key path. +// 2. Unless refused, runs the checks on the calling thread. +// 3. On the options thread: loads the prior options back, also when a +// check throws, unless the options differ from what step 1 loaded (a +// user edited them): those are kept and named in `edited_keys`. +// A failed restore is logged and leaves `restored` false; it never throws. +// Cancelled checks return the run with `cancelled` set, after the restore, +// so the caller can still say whether the prior options are back; any +// other exception from a check is thrown after the restore. +PrecheckOnlyRun run_prechecks_only( + const JsonValue& candidate, + const OptionsThreadCall& on_options_thread, + PrecheckOnlyTarget& target, + Logger& logger +); + + + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.cpp b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.cpp new file mode 100644 index 0000000000..bdf28d2ae7 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.cpp @@ -0,0 +1,297 @@ +/* Program Precheck Run - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * A checks-only run against a fake program: the candidate merged over the + * current options (groups keep the keys not given, unknown keys are + * refused), a value the program does not hold refused, options loaded only + * on the options thread and put back after the checks, also when one + * throws or is cancelled, and kept when a user edits them during the + * checks. + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Json/JsonObject.h" +#include "Common/Cpp/Json/JsonValue.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "ProgramPrecheckRun.h" +#include "ProgramPrecheckRun_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + + +constexpr std::string_view CURRENT = R"({"COUNT":5,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})"; + +// What the fake program's precheck does while it runs. +enum class CheckAction{ + PASS, + USER_SETS_COUNT_TO_9, // A user edits COUNT in the panel meanwhile. + THROW, + CANCEL, // The panel changes: the checks' scope is cancelled. +}; + +// A program whose options are CURRENT. Loading clamps COUNT to 10, stores +// SPEED as a double, and ignores NOTIFICATIONS.Screenshot (as false) while +// NOTIFICATIONS.Enabled is false; check_validity() refuses COUNT 0. Counts +// option calls made off the options thread. +class FakeProgram : public PrecheckOnlyTarget{ +public: + explicit FakeProgram(CheckAction action) + : m_options(parse_json(std::string(CURRENT))) + , m_action(action) + {} + + bool on_options_thread = false; + mutable size_t off_thread_calls = 0; + std::optional seen_by_checks = {}; + + virtual JsonValue options() const override{ + count_thread(); + return m_options.clone(); + } + virtual void load_options(const JsonValue& options) override{ + count_thread(); + m_options = options.clone(); + JsonObject& object = *m_options.to_object(); + JsonValue& count = object["COUNT"]; + if (count.to_integer_default() > 10){ + count = JsonValue(int64_t(10)); + } + object["SPEED"] = JsonValue(object["SPEED"].to_double_default()); + JsonObject& notifications = *object["NOTIFICATIONS"].to_object(); + if (!notifications["Enabled"].to_boolean_default()){ + notifications["Screenshot"] = JsonValue(false); + } + } + virtual std::string check_validity() const override{ + count_thread(); + return m_options.to_object()->get_value("COUNT")->to_integer_default() == 0 ? "COUNT must be at least 1." : ""; + } + virtual std::optional> run_checks() override{ + seen_by_checks = m_options.dump(-1); + switch (m_action){ + case CheckAction::PASS: + break; + case CheckAction::USER_SETS_COUNT_TO_9: + (*m_options.to_object())["COUNT"] = JsonValue(int64_t(9)); + break; + case CheckAction::THROW: + throw std::runtime_error("the console did not answer"); + case CheckAction::CANCEL: + throw OperationCancelledException(); + } + return std::vector{PrecheckReport{.name = "black_border", .status = PrecheckReportStatus::PASSED, .message = ""}}; + } + std::string held() const{ + return m_options.dump(-1); + } + +private: + void count_thread() const{ + if (!on_options_thread){ + off_thread_calls++; + } + } + + JsonValue m_options; + CheckAction m_action; +}; + +struct RunInput{ + std::string candidate = {}; + CheckAction action = CheckAction::PASS; + bool restore_call_fails = false; // The options thread refuses its second call. +}; +struct RunExpectation{ + std::optional refused_key = {}; + std::optional seen_by_checks = {}; // None: no check ran. + bool threw = false; + bool cancelled = false; + bool restored = false; + std::vector edited_keys = {}; + std::string held = {}; // The options the program holds afterwards. +}; +struct RunCase{ + std::string name; + RunInput input; + RunExpectation expectation; +}; + +UnitTestResult check(const RunCase& c, Logger& logger){ + FakeProgram program(c.input.action); + size_t options_thread_calls = 0; + const OptionsThreadCall on_options_thread = [&](const std::function& body){ + options_thread_calls++; + if (c.input.restore_call_fails && options_thread_calls == 2){ + throw std::runtime_error("the options thread is gone"); + } + program.on_options_thread = true; + try{ + body(); + }catch (...){ + program.on_options_thread = false; + throw; + } + program.on_options_thread = false; + }; + PrecheckOnlyRun run; + bool threw = false; + try{ + run = run_prechecks_only(parse_json(c.input.candidate), on_options_thread, program, logger); + }catch (std::runtime_error&){ + threw = true; + } + const RunExpectation& e = c.expectation; + const std::optional refused_key = run.refused ? std::optional(run.refused->key) : std::nullopt; + if (program.off_thread_calls != 0){ + return std::format("{} option calls off the options thread", program.off_thread_calls); + } + if (threw != e.threw || refused_key != e.refused_key || program.seen_by_checks != e.seen_by_checks){ + return std::format("threw {}, refused {}, checks saw {}; expected threw {}, refused {}, checks saw {}", + threw, refused_key.value_or("nothing"), program.seen_by_checks.value_or("nothing"), + e.threw, e.refused_key.value_or("nothing"), e.seen_by_checks.value_or("nothing")); + } + if (!threw && (run.cancelled != e.cancelled || run.restored != e.restored || run.edited_keys != e.edited_keys)){ + return std::format("cancelled {}, restored {}, {} edited keys; expected cancelled {}, restored {}, {} edited keys", + run.cancelled, run.restored, run.edited_keys.size(), e.cancelled, e.restored, e.edited_keys.size()); + } + if (program.held() != e.held){ + return std::format("the program holds\n{}\nexpected\n{}", program.held(), e.held); + } + return true; +} + +class Test_RunPrechecksOnly : public UnitTest{ +public: + explicit Test_RunPrechecksOnly(RunCase test_case) + : UnitTest(std::format("CommonFramework::ProgramPrecheckRun - run - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + return check(m_case, logger); + } + +private: + RunCase m_case; +}; + +const RunCase RUN_CASES[] = { + { + .name = "a value replaces the current one, the rest stay", + .input = {.candidate = R"({"COUNT":7})", .action = CheckAction::PASS}, + .expectation = {.seen_by_checks = R"({"COUNT":7,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a group keeps the keys not given", + .input = {.candidate = R"({"NOTIFICATIONS":{"Screenshot":true}})", .action = CheckAction::PASS}, + .expectation = {.seen_by_checks = R"({"COUNT":5,"NOTIFICATIONS":{"Enabled":true,"Screenshot":true},"SPEED":0.5})", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "an empty candidate changes nothing", + .input = {.candidate = R"({})", .action = CheckAction::PASS}, + .expectation = {.seen_by_checks = R"({"COUNT":5,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "an integer for a number option is the same value", + .input = {.candidate = R"({"SPEED":2})", .action = CheckAction::PASS}, + .expectation = {.seen_by_checks = R"({"COUNT":5,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":2.0})", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "an unknown key is refused", + .input = {.candidate = R"({"COUNT":7,"SPEEED":1.0})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "SPEEED", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "an unknown key in a group is refused with its key path", + .input = {.candidate = R"({"NOTIFICATIONS":{"Sound":true}})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "NOTIFICATIONS.Sound", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a value for a group is refused", + .input = {.candidate = R"({"NOTIFICATIONS":false})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "NOTIFICATIONS", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a candidate that is not an object is refused", + .input = {.candidate = R"([1,2])", .action = CheckAction::PASS}, + .expectation = {.refused_key = "", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a number the program clamps is refused", + .input = {.candidate = R"({"COUNT":70})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "COUNT", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a choice the program ignores is refused with its key path", + .input = {.candidate = R"({"NOTIFICATIONS":{"Enabled":false,"Screenshot":true}})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "NOTIFICATIONS.Screenshot", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "a combination check_validity refuses is refused", + .input = {.candidate = R"({"COUNT":0})", .action = CheckAction::PASS}, + .expectation = {.refused_key = "", .restored = true, .held = std::string(CURRENT)}, + }, + { + .name = "an option a user edits during the checks is kept and named", + .input = {.candidate = R"({"SPEED":0.75})", .action = CheckAction::USER_SETS_COUNT_TO_9}, + .expectation = { + .seen_by_checks = R"({"COUNT":5,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.75})", + .restored = false, + .edited_keys = {"COUNT"}, + .held = R"({"COUNT":9,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.75})", + }, + }, + { + .name = "a check that throws still puts the prior options back", + .input = {.candidate = R"({"COUNT":7})", .action = CheckAction::THROW}, + .expectation = { + .seen_by_checks = R"({"COUNT":7,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", + .threw = true, + .held = std::string(CURRENT), + }, + }, + { + .name = "cancelled checks put the prior options back and report it", + .input = {.candidate = R"({"COUNT":7})", .action = CheckAction::CANCEL}, + .expectation = { + .seen_by_checks = R"({"COUNT":7,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", + .cancelled = true, + .restored = true, + .held = std::string(CURRENT), + }, + }, + { + .name = "cancelled checks whose restore fails report the options not restored", + .input = {.candidate = R"({"COUNT":7})", .action = CheckAction::CANCEL, .restore_call_fails = true}, + .expectation = { + .seen_by_checks = R"({"COUNT":7,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", + .cancelled = true, + .restored = false, + .held = R"({"COUNT":7,"NOTIFICATIONS":{"Enabled":true,"Screenshot":false},"SPEED":0.5})", + }, + }, +}; + + +} // namespace + + + +void add_tests_ProgramPrecheckRun(UnitTestDatabase& database){ + for (const RunCase& c : RUN_CASES){ + database.add(c); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.h b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.h new file mode 100644 index 0000000000..401f3461f0 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramPrecheckRun_Tests.h @@ -0,0 +1,21 @@ +/* Program Precheck Run - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_ProgramPrecheckRun_Tests_H +#define PokemonAutomation_CommonFramework_ProgramPrecheckRun_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Adds the tests of the checks-only run's option staging. +void add_tests_ProgramPrecheckRun(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp index f4176b9199..1be06a246f 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.cpp +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.cpp @@ -43,6 +43,20 @@ ProgramSession::RunningProgramScope::RunningProgramScope(ProgramSession& session } session.m_scope = &m_scope; } +ProgramSession::RunningProgramScope::RunningProgramScope(ProgramSession& session, CancellableScope& parent) + : m_session(session) + , m_scope(parent) +{ + std::lock_guard lg(session.m_lock); + const ProgramState state = session.current_state(); + if (state != ProgramState::STOPPED){ + throw UserSetupError(session.m_logger, "Stop the program before checking it."); + } + if (session.m_scope != nullptr){ + throw UserSetupError(session.m_logger, "The program is already running its start checks."); + } + session.m_scope = &m_scope; +} ProgramSession::RunningProgramScope::~RunningProgramScope(){ { std::lock_guard lg(m_session.m_lock); @@ -95,6 +109,56 @@ void ProgramSession::join_program_thread() noexcept{ const std::string& ProgramSession::identifier() const{ return m_descriptor.identifier(); } +std::optional> ProgramSession::Prechecks::summaries() const{ + if (m_session.m_instance == nullptr){ + return std::nullopt; + } + return m_session.m_instance->precheck_summaries(); +} +PrecheckOnlyRun ProgramSession::Prechecks::run_only( + const JsonValue& candidate, CancellableScope& parent, const OptionsThreadCall& on_options_thread +){ + if (m_session.m_instance == nullptr){ + throw InternalProgramError(&m_session.m_logger, PA_CURRENT_FUNCTION, "The program has no instance."); + } + RunningProgramScope running_scope(m_session, parent); + + // A local class has the access of this function: make_env() and + // internal_run_prechecks(). + class SessionTarget : public PrecheckOnlyTarget{ + public: + explicit SessionTarget(ProgramSession& session) : m_session(session) {} + virtual JsonValue options() const override{ + return m_session.m_instance->to_json(); + } + virtual void load_options(const JsonValue& options) override{ + m_session.m_instance->load_json(options); + } + virtual std::string check_validity() const override{ + return m_session.m_instance->check_validity(); + } + virtual std::optional> run_checks() override{ + if (!m_session.m_instance->precheck_summaries()){ + return std::nullopt; + } + const ProgramInfo program_info( + m_session.identifier(), + m_session.m_descriptor.category(), + m_session.m_descriptor.display_name(), + current_time() + ); + std::unique_ptr env = m_session.make_env(program_info); + return m_session.internal_run_prechecks(*env); + } + private: + ProgramSession& m_session; + }; + SessionTarget target(m_session); + return run_prechecks_only(candidate, on_options_thread, target, m_session.m_logger); +} +std::optional> ProgramSession::internal_run_prechecks(ProgramEnvironment&){ + return std::nullopt; +} std::string ProgramSession::current_stats() const{ std::lock_guard lg(m_lock); if (m_current_stats){ @@ -177,6 +241,15 @@ void ProgramSession::set_state(ProgramState state){ m_state.store(state, std::memory_order_relaxed); m_listeners.run_method(&Listener::state_change, state); } +void ProgramSession::mark_started(ActionSource source, std::optional request_id){ + const WallClock now = current_time(); + m_last_state_change.store(now, std::memory_order_relaxed); + m_transitions.started(source, std::move(request_id), now); +} +void ProgramSession::set_finished(){ + m_transitions.finished(current_time()); + set_state(ProgramState::STOPPED); +} void ProgramSession::push_stats(){ m_listeners.run_method( &Listener::stats_update, @@ -185,6 +258,7 @@ void ProgramSession::push_stats(){ ); } void ProgramSession::push_error(const std::string& message){ + m_transitions.error_reported(); m_listeners.run_method(&Listener::error, message); } @@ -227,7 +301,7 @@ void ProgramSession::update_historical_stats_with_current(){ } -std::string ProgramSession::start_program(){ +std::string ProgramSession::start_program(ActionSource source, std::optional request_id){ m_logger.log("Received Program Start Request"); std::lock_guard lg(m_lock); @@ -238,7 +312,7 @@ std::string ProgramSession::start_program(){ return "Program is not ready."; case ProgramState::STOPPED:{ - std::string error = check_validity(); + std::string error = m_scope != nullptr ? "Program is running its start checks." : check_validity(); if (!error.empty()){ m_logger.log(error, COLOR_RED); return error; @@ -249,7 +323,7 @@ std::string ProgramSession::start_program(){ // Now start the program. m_logger.log("Starting program..."); - m_last_state_change.store(current_time(), std::memory_order_relaxed); + mark_started(source, std::move(request_id)); set_state(ProgramState::RUNNING); m_program_thread = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( [this]{ @@ -284,7 +358,7 @@ std::string ProgramSession::start_program(){ "Invalid State: " + std::to_string((int)state) ); } -std::string ProgramSession::stop_program(){ +std::string ProgramSession::stop_program(ActionSource source, std::optional request_id){ m_logger.log("Received Stop Request"); { std::lock_guard lg(m_lock); @@ -299,6 +373,7 @@ std::string ProgramSession::stop_program(){ return "Program is already stopped."; case ProgramState::RUNNING: m_logger.log("Stopping program..."); + m_transitions.stop_requested(source, std::move(request_id), current_time()); set_state(ProgramState::STOPPING); break; case ProgramState::STOPPING: @@ -449,7 +524,7 @@ void ProgramSession::run_program(){ std::lock_guard lg(m_lock); push_stats(); update_historical_stats_with_current(); - set_state(ProgramState::STOPPED); + set_finished(); } } diff --git a/SerialPrograms/Source/CommonFramework/ProgramSession.h b/SerialPrograms/Source/CommonFramework/ProgramSession.h index af12d8c820..5ec1ca8e70 100644 --- a/SerialPrograms/Source/CommonFramework/ProgramSession.h +++ b/SerialPrograms/Source/CommonFramework/ProgramSession.h @@ -17,6 +17,7 @@ #define PokemonAutomation_CommonFramework_ProgramSession_H #include +#include #include #include #include "Common/Cpp/Logging/TaggedLogger.h" @@ -29,6 +30,8 @@ #include "CommonFramework/Globals.h" //#include "CommonFramework/Logging/Logger.h" #include "CommonFramework/Panels/ProgramDescriptor.h" +#include "CommonFramework/ProgramPrecheckRun.h" +#include "CommonFramework/ProgramTransition.h" #include "CommonFramework/Tools/ProgramEnvironment.h" // #include "CommonFramework/ResourceDownload/ProgramMissingResourceTracker.h" #include "Integrations/ProgramTrackerInterfaces.h" @@ -77,6 +80,33 @@ class ProgramSession : public TrackableProgram{ uint64_t instance_id() const{ return m_instance_id; } Logger& logger(){ return m_logger; } + // The program's start checks, without starting the program. + class Prechecks{ + public: + explicit Prechecks(ProgramSession& session) : m_session(session) {} + + // The precheck list (ProgramInstance::precheck_summaries()), read + // without running it. Nothing when it declares none. + std::optional> summaries() const; + + // A checks-only run under `parent` (run_prechecks_only()): loads + // `candidate` (option keys to change, run_prechecks_only()) over + // the current options through `on_options_thread`, runs the + // program's prechecks on the calling thread without starting it + // (internal_run_prechecks()), and loads the prior options back + // through `on_options_thread`, also when a check throws, unless a + // user changed them meanwhile. Start refuses while it runs, and + // nothing is saved to the settings file. Throws if the program is + // not STOPPED. + PrecheckOnlyRun run_only( + const JsonValue& candidate, CancellableScope& parent, const OptionsThreadCall& on_options_thread + ); + + private: + ProgramSession& m_session; + }; + Prechecks prechecks(){ return Prechecks(*this); } + // - return struct that contains the list of missing resources and boolean upgrade_warning. // - upgrade_warning: is true if we need to warn the user to upgrade CC. @@ -97,6 +127,9 @@ class ProgramSession : public TrackableProgram{ std::string historical_stats() const; virtual WallClock last_state_change() const final; + // The last start or stop and who asked for it, or nullopt before the first start. + std::optional last_transition() const{ return m_transitions.last(); } + // Temporary for migration. StatsTracker* current_stats_tracker(){ return m_current_stats.get(); } const StatsTracker* historical_stats_tracker() const{ return m_historical_stats.get(); } @@ -109,18 +142,25 @@ class ProgramSession : public TrackableProgram{ // Returns empty string on success. Otherwise returns error string. // Note that these are asynchronous and will return before they are finished. + // `source` is who asks; `request_id` the lab request's id when the lab asks. + // last_transition() reports them. - std::string start_program(); // Returns before the program is fully started up. - std::string stop_program(); // Returns before the program is completely stopped. + // Returns before the program is fully started up. + std::string start_program(ActionSource source, std::optional request_id); + // Returns before the program is completely stopped. + std::string stop_program(ActionSource source, std::optional request_id); // Temporary - virtual void async_start() override{ start_program(); } - virtual void async_stop() override{ stop_program(); } + virtual void async_start() override{ start_program(ActionSource::UI, std::nullopt); } + virtual void async_stop() override{ stop_program(ActionSource::UI, std::nullopt); } protected: virtual std::unique_ptr make_env(const ProgramInfo& program_info) = 0; virtual void internal_run_program(ProgramEnvironment& env) = 0; + // The program's prechecks for Prechecks::run_only(), under m_scope, up to + // and including the first FAILED one; nothing when it declares none. + virtual std::optional> internal_run_prechecks(ProgramEnvironment& env); virtual void internal_stop_program(); // virtual void restore_defaults(){ return; } @@ -145,6 +185,10 @@ class ProgramSession : public TrackableProgram{ // Everything here must be called under the lock. void set_state(ProgramState state); + // Records the start time and who asked, before the state goes RUNNING. + void mark_started(ActionSource source, std::optional request_id); + // The run ended: records who ended it, unless a stop was asked for, and goes STOPPED. + void set_finished(); void push_stats(); void push_error(const std::string& message); void push_download_error(const std::string& message); @@ -168,6 +212,7 @@ class ProgramSession : public TrackableProgram{ std::atomic m_last_state_change; std::atomic m_state; + ProgramTransitionLatch m_transitions; // ProgramMissingResourceTracker m_missing_resource_tracker; @@ -186,6 +231,9 @@ class ProgramSession : public TrackableProgram{ class RunningProgramScope{ public: RunningProgramScope(ProgramSession& session); + // For a checks-only run: attached to `parent`, and only while the + // program is STOPPED. + RunningProgramScope(ProgramSession& session, CancellableScope& parent); ~RunningProgramScope(); private: diff --git a/SerialPrograms/Source/CommonFramework/ProgramTransition.cpp b/SerialPrograms/Source/CommonFramework/ProgramTransition.cpp new file mode 100644 index 0000000000..840ab2a71f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramTransition.cpp @@ -0,0 +1,42 @@ +/* Program Transition + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "ProgramTransition.h" + +namespace PokemonAutomation{ + + +void ProgramTransitionLatch::started(ActionSource source, std::optional request_id, WallClock now){ + std::lock_guard lg(m_lock); + m_stop_requested = false; + m_error = false; + m_last = ProgramTransition{ProgramAction::START, source, std::move(request_id), now}; +} +void ProgramTransitionLatch::stop_requested(ActionSource source, std::optional request_id, WallClock now){ + std::lock_guard lg(m_lock); + m_stop_requested = true; + m_last = ProgramTransition{ProgramAction::STOP, source, std::move(request_id), now}; +} +void ProgramTransitionLatch::error_reported(){ + std::lock_guard lg(m_lock); + m_error = true; +} +void ProgramTransitionLatch::finished(WallClock now){ + std::lock_guard lg(m_lock); + if (m_stop_requested){ + return; + } + const ActionSource source = m_error ? ActionSource::PROGRAM_ERROR : ActionSource::PROGRAM; + m_last = ProgramTransition{ProgramAction::STOP, source, std::nullopt, now}; +} +std::optional ProgramTransitionLatch::last() const{ + std::lock_guard lg(m_lock); + return m_last; +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/ProgramTransition.h b/SerialPrograms/Source/CommonFramework/ProgramTransition.h new file mode 100644 index 0000000000..34ec365338 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramTransition.h @@ -0,0 +1,75 @@ +/* Program Transition + * + * From: https://github.com/PokemonAutomation/ + * + * Who started or stopped a program: the Start/Stop button, a lab request + * (with the lab server's id for it), or the program itself when it ends. + * ProgramSession passes the asker in with each start and stop and keeps + * the last transition here, so a reader such as the lab channel never has + * to guess it from the state changes. Free of Qt and of ProgramSession, so + * it is unit tested. + * + */ + +#ifndef PokemonAutomation_CommonFramework_ProgramTransition_H +#define PokemonAutomation_CommonFramework_ProgramTransition_H + +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/Concurrency/Mutex.h" + +namespace PokemonAutomation{ + + +// Whether a transition started or stopped the program. +enum class ProgramAction{ + START, + STOP, +}; + +// Who asked for a start or stop. +enum class ActionSource{ + UI, // The Start/Stop button, or an integration (e.g. Discord). + LAB, // A lab channel request. + CLI, // A --headless --run command-line launch. + PROGRAM, // The program finished by itself. + PROGRAM_ERROR, // The program ended with an error. +}; + +// One start or stop of a program. +struct ProgramTransition{ + ProgramAction action; + ActionSource source; + std::optional request_id; // The lab request's id, if a lab request asked. + WallClock at; +}; + + +// The last transition of one program. A stop keeps its asker from STOPPING +// through STOPPED: an error or the program's own end after a stop request +// does not overwrite who stopped it. Thread-safe. +class ProgramTransitionLatch{ +public: + // The program went RUNNING because `source` asked. + void started(ActionSource source, std::optional request_id, WallClock now); + // The program went STOPPING because `source` asked. + void stop_requested(ActionSource source, std::optional request_id, WallClock now); + // The running program reported an error. + void error_reported(); + // The program went STOPPED. + void finished(WallClock now); + + std::optional last() const; + +private: + mutable Mutex m_lock; + bool m_stop_requested = false; + bool m_error = false; + std::optional m_last; +}; + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.cpp b/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.cpp new file mode 100644 index 0000000000..0e644fcb54 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.cpp @@ -0,0 +1,172 @@ +/* Program Transition - Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Who each start and stop of a program is said to come from, as + * ProgramSession reports its steps to ProgramTransitionLatch. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "ProgramTransition.h" +#include "ProgramTransition_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + +using namespace std::chrono_literals; + + +constexpr std::string_view REQUEST_ID = "5f0c2a8e-3d41-4b7a-9c55-0e6a1f2b7c10"; + +// One step ProgramSession reports. +enum class Step{ + UI_START, + LAB_START, + UI_STOP, + LAB_STOP, + ERROR_REPORTED, + FINISHED, +}; +struct LatchInput{ + std::vector steps; +}; +struct LatchExpectation{ + std::optional action; // None: no transition yet. + ActionSource source = ActionSource::UI; + bool has_request_id = false; + size_t at_step = 0; // The step (from 1) whose time the transition has. +}; +struct LatchCase{ + std::string name; + LatchInput input; + LatchExpectation expectation; +}; + +WallClock step_time(size_t step){ + return WallClock{} + Seconds(1000 + step); +} +void apply(Step step, WallClock now, ProgramTransitionLatch& latch){ + switch (step){ + case Step::UI_START: + latch.started(ActionSource::UI, std::nullopt, now); + return; + case Step::LAB_START: + latch.started(ActionSource::LAB, std::string(REQUEST_ID), now); + return; + case Step::UI_STOP: + latch.stop_requested(ActionSource::UI, std::nullopt, now); + return; + case Step::LAB_STOP: + latch.stop_requested(ActionSource::LAB, std::string(REQUEST_ID), now); + return; + case Step::ERROR_REPORTED: + latch.error_reported(); + return; + case Step::FINISHED: + latch.finished(now); + return; + } +} +UnitTestResult check(const LatchCase& c){ + ProgramTransitionLatch latch; + for (size_t i = 0; i < c.input.steps.size(); i++){ + apply(c.input.steps[i], step_time(i + 1), latch); + } + const std::optional last = latch.last(); + const LatchExpectation& expected = c.expectation; + if (!expected.action){ + return last ? UnitTestResult("expected no transition") : UnitTestResult(true); + } + if (!last || last->action != *expected.action || last->source != expected.source){ + return "wrong action or source"; + } + if (last->request_id.has_value() != expected.has_request_id){ + return "a request id goes with a lab request only"; + } + if (last->at != step_time(expected.at_step)){ + return "the transition has the time of the wrong step"; + } + return true; +} +class Test_ProgramTransitionLatch : public UnitTest{ +public: + explicit Test_ProgramTransitionLatch(LatchCase test_case) + : UnitTest(std::format("CommonFramework::ProgramTransition - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check(m_case); + } + +private: + LatchCase m_case; +}; +const LatchCase LATCH_CASES[] = { + {.name = "nothing yet", .input = {.steps = {}}, .expectation = {.action = std::nullopt}}, + { + .name = "UI start", + .input = {.steps = {Step::UI_START}}, + .expectation = {.action = ProgramAction::START, .source = ActionSource::UI, .at_step = 1}, + }, + { + .name = "lab start", + .input = {.steps = {Step::LAB_START}}, + .expectation = {.action = ProgramAction::START, .source = ActionSource::LAB, .has_request_id = true, .at_step = 1}, + }, + { + .name = "UI stop after a lab start", + .input = {.steps = {Step::LAB_START, Step::UI_STOP, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::UI, .at_step = 2}, + }, + { + .name = "lab stop is latched through the end of the run", + .input = {.steps = {Step::UI_START, Step::LAB_STOP, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::LAB, .has_request_id = true, .at_step = 2}, + }, + { + .name = "error while stopping keeps who stopped", + .input = {.steps = {Step::UI_START, Step::LAB_STOP, Step::ERROR_REPORTED, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::LAB, .has_request_id = true, .at_step = 2}, + }, + { + .name = "program finished by itself", + .input = {.steps = {Step::LAB_START, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::PROGRAM, .at_step = 2}, + }, + { + .name = "program ended with an error", + .input = {.steps = {Step::UI_START, Step::ERROR_REPORTED, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::PROGRAM_ERROR, .at_step = 3}, + }, + { + .name = "error of an earlier run is forgotten", + .input = {.steps = {Step::UI_START, Step::ERROR_REPORTED, Step::FINISHED, Step::UI_START, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::PROGRAM, .at_step = 5}, + }, + { + .name = "stop of an earlier run is forgotten", + .input = {.steps = {Step::UI_START, Step::LAB_STOP, Step::FINISHED, Step::UI_START, Step::FINISHED}}, + .expectation = {.action = ProgramAction::STOP, .source = ActionSource::PROGRAM, .at_step = 5}, + }, +}; + + +} // namespace + + +void add_tests_ProgramTransition(UnitTestDatabase& database){ + for (const LatchCase& c : LATCH_CASES){ + database.add(c); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.h b/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.h new file mode 100644 index 0000000000..8aa4419356 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/ProgramTransition_Tests.h @@ -0,0 +1,21 @@ +/* Program Transition - Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_CommonFramework_ProgramTransition_Tests_H +#define PokemonAutomation_CommonFramework_ProgramTransition_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Adds the tests of ProgramTransitionLatch. +void add_tests_ProgramTransition(UnitTestDatabase& database); + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp new file mode 100644 index 0000000000..710ece1b0c --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.cpp @@ -0,0 +1,35 @@ +/* Lab Startup Policy + * + * From: https://github.com/PokemonAutomation/ + */ +#include "LabStartupPolicy.h" + +namespace PokemonAutomation{ + +StartupPlan apply_lab_startup_policy(StartupPlan plan, const LabStartupPolicy& policy){ + if (plan.mode != StartupMode::GUI && plan.mode != StartupMode::HEADLESS){ + return plan; + } + if (policy.profiles == LabProfileMode::UNSUPPORTED){ + if (plan.mode == StartupMode::HEADLESS){ + return {.error = "--headless is supported only on macOS."}; + } + if (plan.profile || policy.bound_profile){ + return {.error = "Settings profiles are supported only on macOS."}; + } + return plan; + } + if (policy.bound_profile){ + if (!valid_startup_profile(*policy.bound_profile) || + (plan.profile && plan.profile != policy.bound_profile)){ + return {.error = "The requested profile does not match this lab build."}; + } + plan.profile = policy.bound_profile; + } + if (!plan.profile){ + return {.error = "Lab launch requires --profile NAME or a build configured with PA_LAB_PROFILE."}; + } + return plan; +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h new file mode 100644 index 0000000000..f8566c3998 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy.h @@ -0,0 +1,25 @@ +/* Lab Startup Policy + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabStartupPolicy_H +#define PokemonAutomation_LabStartupPolicy_H +#include "StartupArguments.h" + +namespace PokemonAutomation{ + +// Profiles are supported by the macOS settings backend, and required for lab GUIs. +enum class LabProfileMode{UNSUPPORTED, REQUIRED}; + +// Lab-only restrictions, independent of the general application argument grammar. +struct LabStartupPolicy{ + LabProfileMode profiles = LabProfileMode::UNSUPPORTED; + std::optional bound_profile = std::nullopt; +}; + +// Apply lab identity restrictions after parsing. Rejected plans stay rejected; +// tests bypass console identity, while GUI launches must satisfy the lab policy. +StartupPlan apply_lab_startup_policy(StartupPlan plan, const LabStartupPolicy& policy); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp new file mode 100644 index 0000000000..411fda9c49 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.cpp @@ -0,0 +1,150 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + * + * Regression context: fork PR #226, 01e665911, lab controller/permissions lesson. + * A typo in a test command must never restore a saved controller panel. + */ +#include +#include +#include "LabStartupPolicy.h" +#include "LabStartupPolicy_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +struct LabStartupInput{ + std::string_view name; + std::vector arguments; + LabStartupPolicy policy; +}; + +struct LabStartupExpectation{ + StartupMode mode; + std::optional profile = std::nullopt; + bool augment = false; + int gui_calls = 0; + int headless_calls = 0; + int test_calls = 0; + int exit_code = 2; + std::vector qt_arguments = {}; +}; + +struct LabStartupCase{ + LabStartupInput input; + LabStartupExpectation expectation; +}; + +const LabStartupCase LAB_STARTUP_CASES[] = { + {.input = {.name = "unbound relaunch", .arguments = {}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "bound relaunch", .arguments = {}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "explicit identity", .arguments = {"--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "matching identity", .arguments = {"--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "wrong console", .arguments = {"--profile", "lab-A"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "missing profile", .arguments = {"--profile"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "empty profile", .arguments = {"--profile", ""}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "duplicate profile", .arguments = {"--profile", "lab-B", "--profile", "lab-B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "path traversal", .arguments = {"--profile", "../lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "absolute profile", .arguments = {"--profile", "/lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "space in profile", .arguments = {"--profile", "lab B"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unicode profile", .arguments = {"--profile", "lab-é"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "option as profile", .arguments = {"--profile", "--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "test typo from incident", .arguments = {"--test", "bdsp"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown after test flag", .arguments = {"--command-line-test-mode", "bdsp"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unsupported filter", .arguments = {"--command-line-test-mode", "--test-filter=bdsp"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "test bypasses profile requirement", .arguments = {"--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::TESTS, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "test bypasses bound console", .arguments = {"--command-line-test-mode"}, .policy = {LabProfileMode::REQUIRED, "lab-B"}}, + .expectation = {.mode = StartupMode::TESTS, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "augmented tests", .arguments = {"--augment-color-distortions", "--command-line-test-mode"}, .policy = {}}, + .expectation = {.mode = StartupMode::TESTS, .augment = true, .test_calls = 1, .exit_code = 23}}, + {.input = {.name = "augmentation without tests", .arguments = {"--augment-color-distortions"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "tests cannot select live profile", .arguments = {"--command-line-test-mode", "--profile", "lab-A"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "non mac GUI unchanged", .arguments = {}, .policy = {}}, + .expectation = {.mode = StartupMode::GUI, .gui_calls = 1, .exit_code = 17}}, + {.input = {.name = "unsupported profile platform", .arguments = {"--profile", "lab-B"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "invalid build identity", .arguments = {}, .policy = {LabProfileMode::REQUIRED, "../lab-A"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless with profile on mac", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::HEADLESS, .profile = "emu-1", .headless_calls = 1, .exit_code = 19}}, + {.input = {.name = "headless without profile on mac", .arguments = {"--headless", "--run", "UnitTestRunner"}, .policy = {LabProfileMode::REQUIRED}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless rejected off mac", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}, .policy = {}}, + .expectation = {.mode = StartupMode::REJECTED}}, +}; + +// The GUI callback represents settings load, window restoration and hardware discovery. +// A rejected or test launch must never reach it, even when a saved panel has devices. +UnitTestResult check_lab_startup_case(const LabStartupCase& test){ + const StartupPlan plan = apply_lab_startup_policy(parse_startup_arguments(test.input.arguments), test.input.policy); + const LabStartupExpectation& expected = test.expectation; + if (plan.mode != expected.mode || plan.profile != expected.profile || + plan.augment_color_distortions != expected.augment){ + return std::format("Expected mode {}, profile {}, augment {}; read mode {}, profile {}, augment {}.", + (int)expected.mode, expected.profile.value_or(""), expected.augment, + (int)plan.mode, plan.profile.value_or(""), plan.augment_color_distortions); + } + if (plan.qt_arguments != expected.qt_arguments){ + return "Expected Qt argument sequence differs from the preserved sequence."; + } + if ((plan.mode == StartupMode::REJECTED) != !plan.error.empty()){ + return "Expected rejection reason exactly when startup is rejected."; + } + int gui_calls = 0; + int headless_calls = 0; + int test_calls = 0; + const int code = dispatch_startup(plan, { + .gui = [&](const StartupPlan&){ gui_calls++; return 17; }, + .headless = [&](const StartupPlan&){ headless_calls++; return 19; }, + .tests = [&](const StartupPlan&){ test_calls++; return 23; }, + }); + if (gui_calls != expected.gui_calls || headless_calls != expected.headless_calls || + test_calls != expected.test_calls || code != expected.exit_code){ + return std::format("Expected GUI calls {}, headless calls {}, test calls {}, exit {}; read GUI calls {}, headless calls {}, test calls {}, exit {}.", + expected.gui_calls, expected.headless_calls, expected.test_calls, expected.exit_code, + gui_calls, headless_calls, test_calls, code); + } + return true; +} + +class LabStartupTest : public UnitTest{ +public: + explicit LabStartupTest(const LabStartupCase& test) + : UnitTest("Startup/LabPolicy/" + std::string(test.input.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_lab_startup_case(m_test); + } +private: + const LabStartupCase& m_test; +}; + +} // namespace + +void add_tests_LabStartupPolicy(UnitTestDatabase& database){ + for (const LabStartupCase& test : LAB_STARTUP_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h new file mode 100644 index 0000000000..786ab3fdc9 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/LabStartupPolicy_Tests.h @@ -0,0 +1,15 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_LabStartupPolicy_Tests_H +#define PokemonAutomation_LabStartupPolicy_Tests_H +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register lab-policy and startup-dispatch regressions; no Qt or hardware is opened. +void add_tests_LabStartupPolicy(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp index 615379189d..e01e073fa3 100644 --- a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp +++ b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.cpp @@ -76,25 +76,27 @@ void set_permissions(QObject& object){ } -bool check_resource_folder(Logger& logger){ +bool check_resource_folder(Logger& logger, StartupDialogs dialogs){ if (Filesystem::exists(Filesystem::Path(RESOURCE_PATH()))){ logger.log("Found Resource folder at " + RESOURCE_PATH()); return true; } logger.log("No Resources/ folder at " + RESOURCE_PATH()); - QMessageBox box; - box.setTextFormat(Qt::RichText); - box.critical( - nullptr, - "No Resources Folder!", - QString::fromStdString( - "No Resources Folder:

" - "Make sure you download Resources folder from latest release at
" - "https://github.com/PokemonAutomation/ComputerControl/releases/

" - "The Resources folder is part of PA-SerialPrograms-[VERSION]-[DATE].zip

" - "Extract the Resources folder from the zip file and place it at following path:

" + RESOURCE_PATH() - ) - ); + if (dialogs == StartupDialogs::SHOW){ + QMessageBox box; + box.setTextFormat(Qt::RichText); + box.critical( + nullptr, + "No Resources Folder!", + QString::fromStdString( + "No Resources Folder:

" + "Make sure you download Resources folder from latest release at
" + "https://github.com/PokemonAutomation/ComputerControl/releases/

" + "The Resources folder is part of PA-SerialPrograms-[VERSION]-[DATE].zip

" + "Extract the Resources folder from the zip file and place it at following path:

" + RESOURCE_PATH() + ) + ); + } return false; } diff --git a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h index b9e41d2baf..c9cd695b14 100644 --- a/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h +++ b/SerialPrograms/Source/CommonFramework/Startup/SetupSettings.h @@ -15,13 +15,19 @@ namespace PokemonAutomation{ class Logger; +// Controls whether startup checks open modal QMessageBox dialogs or only log. +enum class StartupDialogs{ + SHOW, + LOG_ONLY, +}; + // Use Qt to setup permissions of cameras and microphones on macOS. // See https://www.qt.io/blog/permission-apis-in-qt-6.5 void set_permissions(QObject& object); // Return true if there is the required Resources/ folder. -// When the folder is missing, it will also pop a critical error message box -bool check_resource_folder(Logger& logger); +// When the folder is missing and `dialogs == StartupDialogs::SHOW`, pops a critical error box. +bool check_resource_folder(Logger& logger, StartupDialogs dialogs = StartupDialogs::SHOW); } #endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp new file mode 100644 index 0000000000..061cd37ffc --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.cpp @@ -0,0 +1,132 @@ +/* Application Startup Arguments + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include "StartupArguments.h" + +namespace PokemonAutomation{ +// Profile names are single ASCII path components; never silently rewrite an identity. +bool valid_startup_profile(std::string_view profile){ + return !profile.empty() && profile.front() != '-' && std::ranges::all_of(profile, [](char c){ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '-'; + }); +} + +namespace{ + +// Qt owns option values; recognize only its documented application option spellings. +// References: doc.qt.io/qt-6/{qapplication,qguiapplication}.html constructors. +enum class QtArgument{UNKNOWN, ACCEPTED, INVALID}; +QtArgument consume_qt_argument(std::span arguments, size_t& index, StartupPlan& plan){ + const std::string_view arg = arguments[index]; + if (arg == "-reverse" || arg == "-widgetcount"){ + plan.qt_arguments.emplace_back(arg); + return QtArgument::ACCEPTED; + } + constexpr std::string_view VALUE_OPTIONS[] = { + "-platform", "-platformpluginpath", "-platformtheme", "-plugin", + "-qwindowgeometry", "-qwindowicon", "-qwindowtitle", "-session", + "-display", "-geometry", "-style", "-stylesheet", + }; + if (std::ranges::find(VALUE_OPTIONS, arg) != std::end(VALUE_OPTIONS)){ + if (index + 1 == arguments.size() || arguments[index + 1].empty() || + arguments[index + 1].starts_with('-')){ + return QtArgument::INVALID; + } + plan.qt_arguments.emplace_back(arg); + plan.qt_arguments.emplace_back(arguments[++index]); + return QtArgument::ACCEPTED; + } + for (std::string_view prefix : {"-style=", "-stylesheet=", "-qmljsdebugger="}){ + if (arg.starts_with(prefix)){ + if (arg.size() == prefix.size()){ + return QtArgument::INVALID; + } + plan.qt_arguments.emplace_back(arg); + return QtArgument::ACCEPTED; + } + } + return QtArgument::UNKNOWN; +} + +StartupPlan validate_startup_plan(StartupPlan plan){ + if (plan.run_program && plan.mode != StartupMode::HEADLESS){ + return {.error = "--run requires --headless."}; + } + if (plan.options_path && !plan.run_program){ + return {.error = "--options requires --run."}; + } + if (plan.mode == StartupMode::HEADLESS && !plan.run_program){ + return {.error = "--headless requires --run ."}; + } + if (plan.augment_color_distortions && plan.mode != StartupMode::TESTS){ + return {.error = "--augment-color-distortions requires --command-line-test-mode."}; + } + if (plan.mode == StartupMode::TESTS && plan.profile){ + return {.error = "Test launches cannot select a console profile."}; + } + return plan; +} + +} // namespace + +StartupPlan parse_startup_arguments(std::span arguments){ + StartupPlan plan{.mode = StartupMode::GUI}; + for (size_t i = 0; i < arguments.size(); i++){ + const std::string_view arg = arguments[i]; + if (arg == "--profile"){ + if (plan.profile || i + 1 == arguments.size() || !valid_startup_profile(arguments[i + 1])){ + return {.error = "--profile requires one unique name containing only ASCII letters, digits, '_' or '-' (not starting with '-')."}; + } + plan.profile = arguments[++i]; + }else if (arg == "--headless"){ + if (plan.mode == StartupMode::TESTS){ + return {.error = "--headless and --command-line-test-mode cannot be combined."}; + } + plan.mode = StartupMode::HEADLESS; + }else if (arg == "--run"){ + if (plan.run_program || i + 1 == arguments.size() || arguments[i + 1].empty() || + arguments[i + 1].starts_with('-')){ + return {.error = "--run requires one unique program identifier."}; + } + plan.run_program = arguments[++i]; + }else if (arg == "--options"){ + if (plan.options_path || i + 1 == arguments.size() || arguments[i + 1].empty() || + arguments[i + 1].starts_with('-')){ + return {.error = "--options requires one unique file path."}; + } + plan.options_path = arguments[++i]; + }else if (arg == "--command-line-test-mode"){ + if (plan.mode == StartupMode::HEADLESS){ + return {.error = "--headless and --command-line-test-mode cannot be combined."}; + } + plan.mode = StartupMode::TESTS; + }else if (arg == "--augment-color-distortions"){ + plan.augment_color_distortions = true; + }else{ + const QtArgument result = consume_qt_argument(arguments, i, plan); + if (result != QtArgument::ACCEPTED){ + return {.error = "Unknown or incomplete startup argument: " + std::string(arg)}; + } + } + } + return validate_startup_plan(std::move(plan)); +} + +int dispatch_startup(const StartupPlan& plan, const StartupHandlers& handlers){ + switch (plan.mode){ + case StartupMode::GUI: + return handlers.gui(plan); + case StartupMode::HEADLESS: + return handlers.headless(plan); + case StartupMode::TESTS: + return handlers.tests(plan); + case StartupMode::REJECTED: + return 2; + } + return 2; +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h new file mode 100644 index 0000000000..d3470ba6b6 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments.h @@ -0,0 +1,52 @@ +/* Application Startup Arguments + * + * From: https://github.com/PokemonAutomation/ + * + * Parse before Qt, settings, logging or device discovery. Example: + * open -n SerialPrograms.app --args --profile lab-B + */ +#ifndef PokemonAutomation_StartupArguments_H +#define PokemonAutomation_StartupArguments_H + +#include +#include +#include +#include +#include +#include + +namespace PokemonAutomation{ + +// Rejected launches must not enter any application entry point. +enum class StartupMode{REJECTED, GUI, HEADLESS, TESTS}; + +// Validated startup intent; error describes a rejected launch. +struct StartupPlan{ + StartupMode mode = StartupMode::REJECTED; + std::optional profile = std::nullopt; + std::optional run_program = std::nullopt; + std::optional options_path = std::nullopt; + bool augment_color_distortions = false; + std::string error = {}; + std::vector qt_arguments = {}; +}; + +// Check a single ASCII profile path component without rewriting its identity. +bool valid_startup_profile(std::string_view profile); + +// Parse argv without argv[0]; preserve documented Qt options for Qt to interpret. +// Unknown or malformed application/Qt arguments fail before settings or devices. +StartupPlan parse_startup_arguments(std::span arguments); + +// Application entry points: the GUI and headless callbacks own all settings and hardware startup. +struct StartupHandlers{ + std::function gui; + std::function headless; + std::function tests; +}; + +// Dispatch a validated plan. Rejection returns 2 without invoking either callback. +int dispatch_startup(const StartupPlan& plan, const StartupHandlers& handlers); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp new file mode 100644 index 0000000000..2f88d67df1 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.cpp @@ -0,0 +1,110 @@ +/* Application Startup Argument Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#include +#include "StartupArguments.h" +#include "StartupArguments_Tests.h" + +namespace PokemonAutomation{ +namespace{ + +struct ArgumentInput{ + std::string_view name; + std::vector arguments; +}; + +struct ArgumentExpectation{ + StartupMode mode; + std::optional profile = std::nullopt; + std::vector qt_arguments = {}; +}; + +struct ArgumentCase{ + ArgumentInput input; + ArgumentExpectation expectation; +}; + +const ArgumentCase ARGUMENT_CASES[] = { + {.input = {.name = "ordinary GUI", .arguments = {}}, + .expectation = {.mode = StartupMode::GUI}}, + {.input = {.name = "optional profile", .arguments = {"--profile", "lab-B"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B"}}, + {.input = {.name = "test dispatch", .arguments = {"--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::TESTS}}, + {.input = {.name = "Qt platform", .arguments = {"-platform", "offscreen"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-platform", "offscreen"}}}, + {.input = {.name = "Qt style assignment", .arguments = {"-style=Fusion"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-style=Fusion"}}}, + {.input = {.name = "Qt style pair", .arguments = {"-style", "Fusion"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-style", "Fusion"}}}, + {.input = {.name = "Qt stylesheet assignment", .arguments = {"-stylesheet=some path.qss"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-stylesheet=some path.qss"}}}, + {.input = {.name = "Qt flags and application profile", .arguments = {"-reverse", "--profile", "lab-B", "-widgetcount"}}, + .expectation = {.mode = StartupMode::GUI, .profile = "lab-B", .qt_arguments = {"-reverse", "-widgetcount"}}}, + {.input = {.name = "repeated Qt plugin", .arguments = {"-plugin", "one", "-plugin", "two"}}, + .expectation = {.mode = StartupMode::GUI, .qt_arguments = {"-plugin", "one", "-plugin", "two"}}}, + {.input = {.name = "Qt missing value", .arguments = {"-platform"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt empty value", .arguments = {"-style", ""}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt empty assignment", .arguments = {"-style="}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt must not swallow application option", .arguments = {"-platform", "--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "Qt must not swallow typo", .arguments = {"-platform", "--test", "bdsp"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown Qt-looking option", .arguments = {"-platfrom", "offscreen"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "unknown after valid Qt option", .arguments = {"-reverse", "--test", "bdsp"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless run", .arguments = {"--headless", "--profile", "emu-1", "--run", "UnitTestRunner"}}, + .expectation = {.mode = StartupMode::HEADLESS, .profile = "emu-1"}}, + {.input = {.name = "headless run with options", .arguments = {"--headless", "--run", "UnitTestRunner", "--options", "opts.json"}}, + .expectation = {.mode = StartupMode::HEADLESS}}, + {.input = {.name = "headless without run", .arguments = {"--headless", "--profile", "emu-1"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "run without headless", .arguments = {"--run", "UnitTestRunner"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "options without run", .arguments = {"--headless", "--options", "opts.json"}}, + .expectation = {.mode = StartupMode::REJECTED}}, + {.input = {.name = "headless with tests", .arguments = {"--headless", "--run", "UnitTestRunner", "--command-line-test-mode"}}, + .expectation = {.mode = StartupMode::REJECTED}}, +}; + +UnitTestResult check_argument_case(const ArgumentCase& test){ + const StartupPlan plan = parse_startup_arguments(test.input.arguments); + const ArgumentExpectation& expected = test.expectation; + if (plan.mode != expected.mode || plan.profile != expected.profile){ + return std::format("Expected mode {} and profile {}; read mode {} and profile {}.", + (int)expected.mode, expected.profile.value_or(""), + (int)plan.mode, plan.profile.value_or("")); + } + if (plan.qt_arguments != expected.qt_arguments){ + return "Expected the complete Qt argument sequence to be preserved in order."; + } + return true; +} + +class ArgumentTest : public UnitTest{ +public: + explicit ArgumentTest(const ArgumentCase& test) + : UnitTest("Startup/Arguments/" + std::string(test.input.name)) + , m_test(test) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return check_argument_case(m_test); + } +private: + const ArgumentCase& m_test; +}; + +} // namespace + +void add_tests_StartupArguments(UnitTestDatabase& database){ + for (const ArgumentCase& test : ARGUMENT_CASES){ + database.add(test); + } +} + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h new file mode 100644 index 0000000000..665e0bed7a --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Startup/StartupArguments_Tests.h @@ -0,0 +1,15 @@ +/* Lab Startup Tests + * + * From: https://github.com/PokemonAutomation/ + */ +#ifndef PokemonAutomation_StartupArguments_Tests_H +#define PokemonAutomation_StartupArguments_Tests_H +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + +// Register parser and startup-dispatch regressions; no Qt or hardware is opened. +void add_tests_StartupArguments(UnitTestDatabase& database); + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp index fc6ceeceaf..401dcd5425 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.5.cpp @@ -141,21 +141,14 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_capture_session.reset(new QMediaCaptureSession()); m_capture_session->setCamera(&m_camera->camera()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() == QCamera::NoError){ - return; - } - m_logger.log("QCamera error: " + m_camera->errorString().toStdString(), COLOR_RED); - }); - - m_camera->start(); -#endif } diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp index 13b6dff18d..c801cb6067 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6.cpp @@ -137,19 +137,15 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_video_sink.reset(new QVideoSink()); m_capture.reset(new QMediaCaptureSession()); m_capture->setCamera(&m_camera->camera()); m_capture->setVideoSink(m_video_sink.get()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() != QCamera::NoError){ - m_logger.log("QCamera error: " + m_camera->errorString().toStdString()); - } - }); -#endif m_metaobject->connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, &m_camera->camera(), [&](const QVideoFrame& frame){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp index 5264a3aea9..0160ac7aeb 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/CameraWidgetQt6_QQuickWidget.cpp @@ -139,19 +139,15 @@ void CameraVideoSource::init( ", FPS: " + std::to_string(m_fps) ); - m_camera.reset(new QCameraThread(m_logger, *device, format)); + m_camera = QCameraThread::try_create(m_logger, *device, format); + if (!m_camera){ + return; + } m_video_sink.reset(new QVideoSink()); m_capture.reset(new QMediaCaptureSession()); m_capture->setCamera(&m_camera->camera()); m_capture->setVideoSink(m_video_sink.get()); -#if 0 - connect(m_camera.get(), &QCamera::errorOccurred, this, [&](){ - if (m_camera->error() != QCamera::NoError){ - m_logger.log("QCamera error: " + m_camera->errorString().toStdString()); - } - }); -#endif m_metaobject->connect( m_video_sink.get(), &QVideoSink::videoFrameChanged, &m_camera->camera(), [&](const QVideoFrame& frame){ diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h index 9db0137e9d..2c6235f32c 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Backends/QCameraThread.h @@ -8,8 +8,10 @@ #define PokemonAutomation_VideoPipeline_QCameraThread_H #include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Qt/GlobalThreadPoolsQt.h" +#include "CommonFramework/Devices/LabDeviceRuntime.h" //#include //using std::cout; @@ -18,14 +20,25 @@ namespace PokemonAutomation{ +// Own the camera and its device lease across the Qt event-thread lifetime. class QCameraThread : public QObject{ public: + // A denied input must not throw across run_on_main_thread_and_wait's Qt callback. + static std::unique_ptr try_create(Logger& logger, QCameraDevice device, QCameraFormat format){ + try{ + return std::make_unique(logger, std::move(device), std::move(format)); + }catch (UserSetupError&){ + return nullptr; // UserSetupError already logged the refusal. + } + } + QCameraThread( Logger& logger, QCameraDevice device, QCameraFormat format ) - : m_logger(logger) + : m_device_lease(acquire_lab_capture(logger, LabDeviceKind::VIDEO, device.id().toHex().toStdString())) + , m_logger(logger) , m_device(std::move(device)) , m_format(std::move(format)) , m_camera(nullptr) @@ -63,6 +76,7 @@ class QCameraThread : public QObject{ private: + std::unique_ptr m_device_lease; Logger& m_logger; QCameraDevice m_device; QCameraFormat m_format; @@ -71,5 +85,5 @@ class QCameraThread : public QObject{ -} +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.cpp new file mode 100644 index 0000000000..56d6ffd21f --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.cpp @@ -0,0 +1,339 @@ +/* Video Source (mGBA over MgbaLink) + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Color.h" +#include "Common/Cpp/Exceptions.h" +#include "Controllers/MgbaLink/MgbaLink_SharedLink.h" +#include "CommonFramework/ImageTypes/ImageRGB32_Qt.h" +#include "CommonFramework/Tools/GlobalThreadPools.h" +#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +#include "VideoSource_MgbaLink.h" + +namespace PokemonAutomation{ + +using namespace MgbaLink; + + +namespace{ + +// Re-anchor the frame clock when a frame maps further than this before +// the time it was read (mGBA paused, or slower than real time). +constexpr std::chrono::milliseconds MAX_CLOCK_LAG(500); + +constexpr std::chrono::milliseconds RECONNECT_DELAY(2000); + +// One GBA frame, rounded down: polling faster only repeats a frame number. +constexpr std::chrono::milliseconds POLL_PERIOD(16); + + +// Forwards to "logger" every line but a repeat of the last one forwarded: +// a bridge that stays away fails the same way on every retry. +class RepeatSuppressingLogger : public Logger{ +public: + explicit RepeatSuppressingLogger(Logger& logger) + : m_logger(logger) + {} + + virtual void log(const std::string& msg, Color color) override{ + { + std::lock_guard lg(m_lock); + if (msg == m_last){ + return; + } + m_last = msg; + } + m_logger.log(msg, color); + } + +private: + Logger& m_logger; + Mutex m_lock; + std::string m_last; +}; + +bool is_local_host(std::string_view host){ + return host == "127.0.0.1" || host == "localhost"; +} + +} // namespace + + + +ImageRGB32 compose_mgba_frame(const ImageViewRGB32& gba_frame, const MgbaComposeGeometry& geometry){ + if (geometry.box_x + geometry.box_width > geometry.canvas_width || + geometry.box_y + geometry.box_height > geometry.canvas_height + ){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + std::format( + "mGBA box {}x{} at ({}, {}) does not fit in a {}x{} canvas.", + geometry.box_width, geometry.box_height, geometry.box_x, geometry.box_y, + geometry.canvas_width, geometry.canvas_height + ) + ); + } + ImageRGB32 canvas(geometry.canvas_width, geometry.canvas_height); + canvas.fill(0xff000000); + const size_t src_width = gba_frame.width(); + const size_t src_height = gba_frame.height(); + if (src_width == 0 || src_height == 0){ + return canvas; + } + std::vector src_x(geometry.box_width); + for (size_t x = 0; x < geometry.box_width; x++){ + src_x[x] = (2 * x + 1) * src_width / (2 * geometry.box_width); + } + for (size_t y = 0; y < geometry.box_height; y++){ + const size_t sy = (2 * y + 1) * src_height / (2 * geometry.box_height); + for (size_t x = 0; x < geometry.box_width; x++){ + canvas.pixel(geometry.box_x + x, geometry.box_y + y) = gba_frame.pixel(src_x[x], sy); + } + } + return canvas; +} + + +std::optional> parse_mgba_address(std::string_view address){ + const size_t colon = address.rfind(':'); + if (colon == std::string_view::npos || colon == 0){ + return std::nullopt; + } + std::string_view port_text = address.substr(colon + 1); + uint16_t port = 0; + std::from_chars_result result = std::from_chars(port_text.data(), port_text.data() + port_text.size(), port); + if (port_text.empty() || result.ec != std::errc() || result.ptr != port_text.data() + port_text.size() || port == 0){ + return std::nullopt; + } + std::string_view host = address.substr(0, colon); + if (!is_local_host(host)){ + return std::nullopt; + } + return std::make_pair(std::string(host), port); +} + + + +std::string VideoSourceDescriptor_MgbaLink::address() const{ + ReadSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + return m_address; +} +bool VideoSourceDescriptor_MgbaLink::operator==(const VideoSourceDescriptor& x) const{ + if (typeid(*this) != typeid(x)){ + return false; + } + return address() == static_cast(x).address(); +} +std::string VideoSourceDescriptor_MgbaLink::display_name() const{ + return "mGBA (" + address() + ")"; +} +JsonValue VideoSourceDescriptor_MgbaLink::to_json() const{ + return address(); +} +void VideoSourceDescriptor_MgbaLink::load_json(const JsonValue& json){ + const std::string* address = json.to_string(); + if (address != nullptr && parse_mgba_address(*address)){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + m_address = *address; + } +} +std::unique_ptr VideoSourceDescriptor_MgbaLink::make_VideoSource( + Logger& logger, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps +) const{ + return std::make_unique(logger, address()); +} + + + +VideoSource_MgbaLink::VideoSource_MgbaLink(Logger& logger, std::string address) + : VideoSource(logger, false) + , m_logger(logger) + , m_address(std::move(address)) +{ + m_formats[{1920, 1080}][VideoFormat::OTHER] = {60}; + m_poller = GlobalThreadPools::unlimited_realtime().dispatch_now_blocking( + [this]{ poll_loop(); } + ); +} +VideoSource_MgbaLink::~VideoSource_MgbaLink(){ + { + std::lock_guard lg(m_lock); + m_stopping = true; + if (m_connection != nullptr){ + m_connection->abort(); + } + } + m_cv.notify_all(); + m_poller.wait_and_ignore_exceptions(); +} + + +VideoSnapshot VideoSource_MgbaLink::snapshot_latest_blocking(){ + std::lock_guard lg(m_lock); + return m_last; +} +VideoSnapshot VideoSource_MgbaLink::snapshot_recent_nonblocking(WallClock min_time){ + std::lock_guard lg(m_lock); + if (m_last.timestamp < min_time){ + return VideoSnapshot(); + } + return m_last; +} + + +bool VideoSource_MgbaLink::wait_or_stop(std::chrono::milliseconds duration){ + std::unique_lock lg(m_lock); + return m_cv.wait_for(lg, duration, [this]{ return m_stopping; }); +} + +void VideoSource_MgbaLink::poll_loop(){ + std::optional> address = parse_mgba_address(m_address); + if (!address){ + m_logger.log( + "MgbaLink: invalid address \"" + m_address + "\"; expected 127.0.0.1: or localhost:.", + COLOR_RED + ); + return; + } + m_logger.log(std::format( + "MgbaLink: polling {}; retrying every {} s while it is away.", + m_address, std::chrono::duration_cast(RECONNECT_DELAY).count() + )); + RepeatSuppressingLogger logger(m_logger); + do{ + try{ + run_connection(logger, address->first, address->second); + }catch (Exception&){ + // Already logged (once) by the exception; retry. + } + }while (!wait_or_stop(RECONNECT_DELAY)); +} + +void VideoSource_MgbaLink::run_connection(Logger& logger, const std::string& host, uint16_t port){ + // Shared: a controller for the same address sends its commands on it. + std::shared_ptr connection = std::make_shared( + logger, GlobalThreadPools::unlimited_realtime(), host, port + ); + { + std::lock_guard lg(m_lock); + if (m_stopping){ + return; + } + m_connection = connection.get(); + } + try{ + connection->open(); + LinkRegistry::Publication publication = link_registry().publish(m_address, connection); + poll_connected(logger, *connection); + }catch (...){ + std::lock_guard lg(m_lock); + m_connection = nullptr; + m_last = VideoSnapshot(); + throw; + } + std::lock_guard lg(m_lock); + m_connection = nullptr; + m_last = VideoSnapshot(); +} + +void VideoSource_MgbaLink::poll_connected(Logger& logger, Connection& connection){ + std::optional clock; + uint64_t last_frame = 0; + do{ + FrameReply reply = connection.frame(); + WallClock now = current_time(); + if (clock && reply.frame == last_frame){ + continue; + } + if (!clock){ + clock.emplace(now, reply.frame); + } + FrameClock::Stamp stamp = clock->stamp(now, reply.frame, MAX_CLOCK_LAG); + if (stamp.reanchored_after_lag){ + m_logger.log(std::format("MgbaLink: frame clock re-anchored at frame {} (mGBA paused or slow).", reply.frame)); + } + last_frame = reply.frame; + + // Read the PNG before the next FRAME overwrites it. + ImageRGB32 gba_frame(reply.path); + if (gba_frame.width() != reply.width || gba_frame.height() != reply.height){ + throw ConnectionException(&logger, std::format( + "MgbaLink: FRAME {} said {}x{}, but {} is {}x{}.", + reply.frame, reply.width, reply.height, reply.path, gba_frame.width(), gba_frame.height() + )); + } + publish_frame(compose_mgba_frame(gba_frame), stamp.time); + }while (!wait_or_stop(POLL_PERIOD)); +} + +void VideoSource_MgbaLink::publish_frame(ImageRGB32 frame, WallClock timestamp){ + VideoSnapshot snapshot(std::move(frame), timestamp); + { + std::lock_guard lg(m_lock); + m_last = snapshot; + } + + // For the FPS readout and stream history. The QImage shares the + // snapshot's pixels and keeps them alive. + const ImageRGB32& pixels = *snapshot.frame; + QImage image( + (const uchar*)pixels.data(), + (int)pixels.width(), + (int)pixels.height(), + (qsizetype)pixels.bytes_per_row(), + QImage::Format_ARGB32, + [](void* owner){ delete static_cast*>(owner); }, + new std::shared_ptr(snapshot.frame) + ); + report_source_frame(std::make_shared(timestamp, QVideoFrame(image))); +} + + + +class VideoWidget_MgbaLink : public QWidget{ +public: + VideoWidget_MgbaLink(QWidget* parent, VideoSource_MgbaLink& source) + : QWidget(parent) + , m_source(source) + { + QTimer* timer = new QTimer(this); + connect(timer, &QTimer::timeout, this, [this]{ update(); }); + timer->start(33); + } + +private: + virtual void paintEvent(QPaintEvent* event) override{ + QWidget::paintEvent(event); + VideoSnapshot snapshot = m_source.snapshot_latest_blocking(); + if (!snapshot.frame || snapshot.frame->width() == 0){ + return; + } + QPainter painter(this); + painter.drawImage(QRect(0, 0, width(), height()), to_QImage_ref(*snapshot.frame)); + m_source.report_rendered_frame(current_time()); + } + +private: + VideoSource_MgbaLink& m_source; +}; + + +QWidget* VideoSource_MgbaLink::make_display_QtWidget(QWidget* parent){ + return new VideoWidget_MgbaLink(parent, *this); +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.h new file mode 100644 index 0000000000..d0e1318402 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink.h @@ -0,0 +1,156 @@ +/* Video Source (mGBA over MgbaLink) + * + * From: https://github.com/PokemonAutomation/ + * + * Shows the picture of an mGBA instance running "bridge.lua" (MgbaLink v1). + * Each new emulated frame is composed into a 1920x1080 frame at FRLG's + * GAME_BOX with nearest-neighbour scaling, so detectors see console + * geometry. While mGBA runs no faster than 59.7275 fps, a frame is stamped + * with the time its frame number maps to; when it runs ahead (e.g. synced + * to a 60 Hz display), with the time it was read. Never later than the + * read time. The source reconnects itself, so it opts out of the session + * no-video watchdog. Developer mode only (fork lab tooling). + * + */ + +#ifndef PokemonAutomation_VideoPipeline_VideoSource_MgbaLink_H +#define PokemonAutomation_VideoPipeline_VideoSource_MgbaLink_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/AsyncTask.h" +#include "Common/Cpp/Concurrency/ConditionVariable.h" +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "Controllers/MgbaLink/MgbaLink_Connection.h" +#include "CommonFramework/VideoPipeline/VideoSourceDescriptor.h" +#include "CommonFramework/VideoPipeline/VideoSource.h" + +namespace PokemonAutomation{ + + +// Where the GBA picture goes: GAME_BOX {0.09375, 0.00462963, 0.8125, +// 0.962963} of a 1920x1080 frame, i.e. x = 180..1740, y = 5..1045 (6.5x). +struct MgbaComposeGeometry{ + size_t canvas_width = 1920; + size_t canvas_height = 1080; + size_t box_x = 180; + size_t box_y = 5; + size_t box_width = 1560; + size_t box_height = 1040; +}; + +// Draws "gba_frame" into a black canvas at the box, nearest-neighbour with +// pixel-centre sampling (as Pillow's NEAREST, which the geometry gate used): +// canvas pixel x takes source column floor((2 (x - box_x) + 1) w / (2 box_width)). +// Throws InternalProgramError if the box does not fit inside the canvas. +ImageRGB32 compose_mgba_frame( + const ImageViewRGB32& gba_frame, + const MgbaComposeGeometry& geometry = MgbaComposeGeometry() +); + + +// "host:port" -> (host, port). nullopt if the port is missing or invalid, +// or the host is not this machine ("127.0.0.1" or "localhost"): FRAME +// names a PNG on the bridge's disk, which this app opens as a local path. +std::optional> parse_mgba_address(std::string_view address); + + + +// Selects an mGBA bridge by "host:port" (saved in the profile). +class VideoSourceDescriptor_MgbaLink : public VideoSourceDescriptor{ +public: + static constexpr std::string_view DEFAULT_ADDRESS = "127.0.0.1:47801"; + + VideoSourceDescriptor_MgbaLink() + : VideoSourceDescriptor(VideoSourceType::MgbaLink) + , m_address(DEFAULT_ADDRESS) + {} + + std::string address() const; + + virtual bool operator==(const VideoSourceDescriptor& x) const override; + virtual std::string display_name() const override; + + virtual JsonValue to_json() const override; + virtual void load_json(const JsonValue& json) override; + + virtual std::unique_ptr make_VideoSource( + Logger& logger, + Resolution resolution, + VideoFormat format, + FramesPerSecond fps + ) const override; + +private: + mutable SpinLock m_lock; + std::string m_address; +}; + + + +// Polls FRAME on its own connection and keeps the latest composed frame; +// drops it when the connection is lost, so a dead bridge shows no video. +// +// The bridge holds one client at a time, so this source takes its only +// slot. While open, the connection is published in MgbaLink::link_registry() +// under the source's address; the mGBA controller for the same address +// sends KEYS and GRANT on it (Connection serializes commands from several +// threads). +class VideoSource_MgbaLink : public VideoSource{ +public: + VideoSource_MgbaLink(Logger& logger, std::string address); + ~VideoSource_MgbaLink(); + + virtual Resolution current_resolution() const override{ + return Resolution(1920, 1080); + } + virtual VideoFormat current_format() const override{ + return VideoFormat::OTHER; + } + virtual FramesPerSecond current_fps() const override{ + return 60; + } + virtual const VideoFormatSet& supported_formats() const override{ + return m_formats; + } + + virtual VideoSnapshot snapshot_latest_blocking() override; + virtual VideoSnapshot snapshot_recent_nonblocking(WallClock min_time) override; + + virtual QWidget* make_display_QtWidget(QWidget* parent) override; + +private: + friend class VideoWidget_MgbaLink; + + void poll_loop(); + void run_connection(Logger& logger, const std::string& host, uint16_t port); + void poll_connected(Logger& logger, MgbaLink::Connection& connection); + void publish_frame(ImageRGB32 frame, WallClock timestamp); + bool wait_or_stop(std::chrono::milliseconds duration); + +private: + Logger& m_logger; + const std::string m_address; + VideoFormatSet m_formats; + + Mutex m_lock; + ConditionVariable m_cv; + bool m_stopping = false; + VideoSnapshot m_last; + // The poller's open connection, if any; the destructor aborts it so it + // never waits out a reply timeout. + MgbaLink::Connection* m_connection = nullptr; + + AsyncTask m_poller; +}; + + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.cpp new file mode 100644 index 0000000000..f0ce9056f9 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.cpp @@ -0,0 +1,202 @@ +/* Video Source (mGBA over MgbaLink) Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "VideoSource_MgbaLink.h" +#include "VideoSource_MgbaLink_Tests.h" + +namespace PokemonAutomation{ + +namespace{ + + +// A 240x160 frame whose pixels name their own coordinates. +ImageRGB32 make_gba_frame(){ + ImageRGB32 frame(240, 160); + for (size_t y = 0; y < 160; y++){ + for (size_t x = 0; x < 240; x++){ + frame.pixel(x, y) = 0xff000000 | (uint32_t)(x << 8) | (uint32_t)y; + } + } + return frame; +} + +struct Point{ + size_t x; + size_t y; +}; + + +// Which GBA pixel lands on a canvas pixel of the composed 1920x1080 frame. +struct ComposeInput{ + Point canvas; +}; +struct ComposeExpectation{ + std::optional gba_pixel; // nullopt: black border. +}; +struct ComposeCase{ + std::string name; + ComposeInput input; + ComposeExpectation expectation; +}; + +class Test_Compose : public UnitTest{ +public: + Test_Compose(ComposeCase test_case) + : UnitTest(std::format("CommonFramework::VideoSource_MgbaLink - compose - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + ImageRGB32 canvas = compose_mgba_frame(make_gba_frame()); + if (canvas.width() != 1920 || canvas.height() != 1080){ + return std::format("canvas is {}x{}, expected 1920x1080", canvas.width(), canvas.height()); + } + const std::optional& gba = m_case.expectation.gba_pixel; + uint32_t expected = gba ? 0xff000000 | (uint32_t)(gba->x << 8) | (uint32_t)gba->y : 0xff000000; + uint32_t actual = canvas.pixel(m_case.input.canvas.x, m_case.input.canvas.y); + if (actual != expected){ + return std::format("pixel {:08x}, expected {:08x}", actual, expected); + } + return true; + } + +private: + ComposeCase m_case; +}; + + +// "host:port" parsing for the descriptor. +struct AddressInput{ + std::string address; +}; +struct AddressExpectation{ + std::optional> host_and_port; +}; +struct AddressCase{ + std::string name; + AddressInput input; + AddressExpectation expectation; +}; + +class Test_Address : public UnitTest{ +public: + Test_Address(AddressCase test_case) + : UnitTest(std::format("CommonFramework::VideoSource_MgbaLink - address - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + std::optional> parsed = parse_mgba_address(m_case.input.address); + if (parsed != m_case.expectation.host_and_port){ + return std::format( + "parsed {}, expected {}", + describe(parsed), describe(m_case.expectation.host_and_port) + ); + } + return true; + } + +private: + static std::string describe(const std::optional>& host_and_port){ + return host_and_port + ? std::format("<{}> port {}", host_and_port->first, host_and_port->second) + : std::string("nothing"); + } + + AddressCase m_case; +}; + + +// compose_mgba_frame() refuses a box that does not fit the canvas. +struct GeometryInput{ + MgbaComposeGeometry geometry; +}; +struct GeometryExpectation{ + bool accepted = false; +}; +struct GeometryCase{ + std::string name; + GeometryInput input; + GeometryExpectation expectation; +}; + +class Test_Geometry : public UnitTest{ +public: + Test_Geometry(GeometryCase test_case) + : UnitTest(std::format("CommonFramework::VideoSource_MgbaLink - geometry - {}", test_case.name)) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + bool accepted = true; + try{ + compose_mgba_frame(make_gba_frame(), m_case.input.geometry); + }catch (InternalProgramError&){ + accepted = false; + } + if (accepted != m_case.expectation.accepted){ + return std::format("accepted={}, expected {}", accepted, m_case.expectation.accepted); + } + return true; + } + +private: + GeometryCase m_case; +}; + +} // namespace + + +void add_tests_VideoSource_MgbaLink(UnitTestDatabase& database){ + // GAME_BOX at 1080p is x = 180..1739, y = 5..1044; 6.5x, so source + // columns alternate 6 and 7 canvas pixels. + const ComposeCase COMPOSE_CASES[] = { + {.name = "box top-left", .input = {{180, 5}}, .expectation = {Point{0, 0}}}, + {.name = "box bottom-right", .input = {{1739, 1044}}, .expectation = {Point{239, 159}}}, + {.name = "left of the box", .input = {{179, 500}}, .expectation = {std::nullopt}}, + {.name = "right of the box", .input = {{1740, 500}}, .expectation = {std::nullopt}}, + {.name = "above the box", .input = {{960, 4}}, .expectation = {std::nullopt}}, + {.name = "below the box", .input = {{960, 1045}}, .expectation = {std::nullopt}}, + {.name = "last pixel of column 0", .input = {{185, 5}}, .expectation = {Point{0, 0}}}, + {.name = "first pixel of column 1", .input = {{186, 5}}, .expectation = {Point{1, 0}}}, + {.name = "last pixel of column 1", .input = {{192, 5}}, .expectation = {Point{1, 0}}}, + {.name = "first pixel of column 2", .input = {{193, 5}}, .expectation = {Point{2, 0}}}, + {.name = "first pixel of row 1", .input = {{180, 11}}, .expectation = {Point{0, 1}}}, + {.name = "centre", .input = {{960, 525}}, .expectation = {Point{120, 80}}}, + }; + for (const ComposeCase& test_case : COMPOSE_CASES){ + database.add(test_case); + } + + const AddressCase ADDRESS_CASES[] = { + {.name = "default bridge", .input = {"127.0.0.1:47801"}, .expectation = {std::make_pair(std::string("127.0.0.1"), uint16_t(47801))}}, + {.name = "host name", .input = {"localhost:47802"}, .expectation = {std::make_pair(std::string("localhost"), uint16_t(47802))}}, + {.name = "no port", .input = {"127.0.0.1"}, .expectation = {std::nullopt}}, + {.name = "empty port", .input = {"127.0.0.1:"}, .expectation = {std::nullopt}}, + {.name = "port 0", .input = {"127.0.0.1:0"}, .expectation = {std::nullopt}}, + {.name = "port too large", .input = {"127.0.0.1:70000"}, .expectation = {std::nullopt}}, + {.name = "no host", .input = {":47801"}, .expectation = {std::nullopt}}, + {.name = "remote host refused", .input = {"192.168.1.20:47801"}, .expectation = {std::nullopt}}, + }; + for (const AddressCase& test_case : ADDRESS_CASES){ + database.add(test_case); + } + + const GeometryCase GEOMETRY_CASES[] = { + {.name = "GAME_BOX at 1080p fits", .input = {MgbaComposeGeometry()}, .expectation = {true}}, + {.name = "box fills the canvas", .input = {{.canvas_width = 240, .canvas_height = 160, .box_x = 0, .box_y = 0, .box_width = 240, .box_height = 160}}, .expectation = {true}}, + {.name = "box past the right edge", .input = {{.box_x = 400}}, .expectation = {false}}, + {.name = "box past the bottom edge", .input = {{.box_y = 50}}, .expectation = {false}}, + }; + for (const GeometryCase& test_case : GEOMETRY_CASES){ + database.add(test_case); + } +} + + + +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.h new file mode 100644 index 0000000000..18ceaa14e6 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/Sources/VideoSource_MgbaLink_Tests.h @@ -0,0 +1,20 @@ +/* Video Source (mGBA over MgbaLink) Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_VideoPipeline_VideoSource_MgbaLink_Tests_H +#define PokemonAutomation_VideoPipeline_VideoSource_MgbaLink_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ + + +// Compose geometry (nearest-neighbour into GAME_BOX) and address parsing. +void add_tests_VideoSource_MgbaLink(UnitTestDatabase& database); + + +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp index 10957abbd6..efddea1c0f 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/UI/VideoSourceSelectorWidget.cpp @@ -9,6 +9,7 @@ #include #include #include "Common/Qt/NoWheelComboBox.h" +#include "CommonFramework/StaticGlobals.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/VideoPipeline/VideoPipelineOptions.h" #include "CommonFramework/Panels/ConsoleSettingsStretch.h" @@ -16,6 +17,7 @@ #include "CommonFramework/VideoPipeline/Sources/VideoSource_Null.h" #include "CommonFramework/VideoPipeline/Sources/VideoSource_Camera.h" +#include "Controllers/MgbaLink/MgbaLink_SharedLink.h" #include "VideoSourceSelectorWidget.h" @@ -138,6 +140,9 @@ void VideoSourceSelectorWidget::update_source_list(){ m_session.save(option); m_sources.emplace_back(option.get_descriptor_from_cache(VideoSourceType::None)); m_sources.emplace_back(option.get_descriptor_from_cache(VideoSourceType::StillImage)); + if (STATIC_GLOBALS.DEVELOPER_MODE || MgbaLink::lab_opt_in()){ + m_sources.emplace_back(option.get_descriptor_from_cache(VideoSourceType::MgbaLink)); + } } // Now add all the cameras. diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.h b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.h index 24db4ea985..ab874a6df1 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.h +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceDescriptor.h @@ -23,6 +23,7 @@ enum class VideoSourceType{ StillImage, VideoPlayback, Camera, + MgbaLink, // Developer mode only (fork lab tooling). }; diff --git a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceOption.cpp b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceOption.cpp index 6c11a5daca..d8252ea03d 100644 --- a/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceOption.cpp +++ b/SerialPrograms/Source/CommonFramework/VideoPipeline/VideoSourceOption.cpp @@ -8,10 +8,13 @@ #include "Common/Cpp/Json/JsonValue.h" #include "Common/Cpp/Json/JsonArray.h" #include "Common/Cpp/Json/JsonObject.h" +#include "CommonFramework/StaticGlobals.h" #include "Sources/VideoSource_Null.h" #include "Sources/VideoSource_StillImage.h" #include "Sources/VideoSource_Camera.h" +#include "Sources/VideoSource_MgbaLink.h" +#include "Controllers/MgbaLink/MgbaLink_SharedLink.h" #include "VideoSourceOption.h" @@ -27,9 +30,31 @@ const EnumStringMap VIDEO_TYPE_STRINGS{ {VideoSourceType::StillImage, "Still Image"}, {VideoSourceType::VideoPlayback, "Video Playback"}, {VideoSourceType::Camera, "Camera"}, + {VideoSourceType::MgbaLink, "mGBA Link"}, }; +namespace{ + +// Loads the saved parameters of one source type into the cache, if present. +template +void load_cached_descriptor( + std::map>& cache, + const JsonObject& obj, + VideoSourceType type +){ + const JsonValue* params = obj.get_value(VIDEO_TYPE_STRINGS.get_string(type)); + if (params == nullptr){ + return; + } + auto descriptor = std::make_unique(); + descriptor->load_json(*params); + cache[type] = std::move(descriptor); +} + +} // namespace + + VideoSourceOption::VideoSourceOption() : m_resolution(1920, 1080) , m_format(VideoFormat::OTHER) @@ -57,6 +82,9 @@ std::shared_ptr VideoSourceOption::get_descriptor_from_ca case VideoSourceType::Camera: descriptor.reset(new VideoSourceDescriptor_Camera()); break; + case VideoSourceType::MgbaLink: + descriptor = std::make_shared(); + break; default:; descriptor.reset(new VideoSourceDescriptor_Null()); } @@ -119,22 +147,10 @@ void VideoSourceOption::load_json(const JsonValue& json){ } #endif - const JsonValue* params; - params = obj->get_value(VIDEO_TYPE_STRINGS.get_string(VideoSourceType::StillImage)); - if (params != nullptr){ - auto x = std::make_unique(); - x->load_json(*params); - m_descriptor_cache[VideoSourceType::StillImage] = std::move(x); - } - params = obj->get_value(VIDEO_TYPE_STRINGS.get_string(VideoSourceType::VideoPlayback)); - if (params != nullptr){ -// m_descriptor_cache[VideoSourceType::VideoPlayback] = - } - params = obj->get_value(VIDEO_TYPE_STRINGS.get_string(VideoSourceType::Camera)); - if (params != nullptr){ - auto x = std::make_unique(); - x->load_json(*params); - m_descriptor_cache[VideoSourceType::Camera] = std::move(x); + load_cached_descriptor(m_descriptor_cache, *obj, VideoSourceType::StillImage); + load_cached_descriptor(m_descriptor_cache, *obj, VideoSourceType::Camera); + if (STATIC_GLOBALS.DEVELOPER_MODE || MgbaLink::lab_opt_in()){ + load_cached_descriptor(m_descriptor_cache, *obj, VideoSourceType::MgbaLink); } auto iter = m_descriptor_cache.find(VIDEO_TYPE_STRINGS.get_enum(*type, VideoSourceType::None)); diff --git a/SerialPrograms/Source/CommonFramework/Windows/Lab.md b/SerialPrograms/Source/CommonFramework/Windows/Lab.md new file mode 100644 index 0000000000..36e84d9c9a --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Windows/Lab.md @@ -0,0 +1,489 @@ +# Lab Channel + +Lets a script drive a running SerialPrograms app through files, with no +window focus and no click: select a program, set its options, start and stop +it, read its state, and press buttons or check detectors on the console +between runs. Meant for a lab that drives one app per console from the shell. + +The channel is one folder per app instance, independent of the selected +program. Requests are text files; replies are text files. Nothing else is +needed on the script side. + +## Turning it on + +Create the folder `Lab/` inside the profile's user-files folder (the folder +that holds `SerialPrograms.log`, `Screenshots/` and the settings; with +`--profile NAME` that is `Profiles/NAME/`) **before launching the app**. An +app that finds it at startup polls it every 500 ms from a worker and creates +`inbox/`, `outbox/` and `done/` inside it; an app that does not find it +starts nothing, so normal users see no change. Creating the folder later +takes effect at the next launch. + +## Files + +| Path | Direction | Meaning | +|---|---|---| +| `Lab/inbox/.txt` | script writes | A request: one command per line, run in order. Requests run one at a time, in name order; a request of `snapshot` lines only runs beside them (see [Snapshot](#snapshot)). | +| `Lab/done/.txt` | app moves | The request, moved here when it starts. | +| `Lab/outbox/.out` | app writes | The request's journal (below). It appears only once the request has finished; `.tmp` shows the progress so far. | +| `Lab/outbox/_