refactor: unify runner execution and weight residency

docs/backend.md

@@ -160,10 +160,12 @@ the runner's graph-cut capacity checks.
 Runtime capacity checks also leave 512 MiB of currently free device memory for
 backend scratch buffers and pipelines, including with explicit backend assignments.
 They cap free-memory reports by the device's total memory minus tracked
-resident allocations. Vulkan reports exceeding total memory are rejected because
-its heap-budget subtraction can underflow. Other backends use the cap instead of
-treating such reports as zero free memory. Failed checks log the reported free and
-total memory alongside tracked weight and runtime allocations.
+resident allocations. Vulkan reports exceeding total memory are rejected on
+discrete GPUs because its heap-budget subtraction can underflow. On integrated
+GPUs, a wrapped report triggers reclamation before the tracked estimate can be
+accepted, subject to the byte checks and configured budgets. Other backends use
+the cap instead of treating such reports as zero free memory. Failed checks log
+the reported free and total memory alongside tracked weight and runtime allocations.
 With `--mmap`, device-backed mappings count toward these budgets at their full
 mapped-file size, once per device buffer even when multiple parameter blocks
 share it. Mappings retained in the loader cache continue to count.
@@ -181,8 +183,18 @@ GPU cache space follows the same component priority. Before a lower-priority
 component can become permanently resident, the planner leaves room for the full
 weights and estimated compute space of higher-priority offloaded components.
 If offloaded diffusion already needs the entire main GPU budget, TE and VAE also
-use offloaded parameters. Their GPU copies can then be released after their
-phases, leaving more room to reuse diffusion weights across sampling steps.
+use offloaded parameters. GPU copies remain cached across sampling steps while
+a component's execution is active and are released under memory pressure.
+Ending a component's execution releases its staged GPU copies, disk-backed
+weight buffers, temporary compute buffers and execution caches. Parameters
+assigned permanently to a backend remain resident. Hires model upscaling shares
+the runtime budget and can
+reclaim idle weight copies from other components.
+Reclamation releases idle compute buffers first, then cached weight allocations
+until the current work fits. Weights used by active computation or pending
+transfers stay protected.
+For CPU computation with disk-backed parameters, cached weights are released
+when execution ends so they do not accumulate in RAM across components.
 CPU parameter residency allows GPU weight caching; it does not force every
 weight to be copied again at every step.
 

src/core/execution_engine.h

@@ -0,0 +1,107 @@
+#ifndef __SD_CORE_EXECUTION_ENGINE_H__
+#define __SD_CORE_EXECUTION_ENGINE_H__
+
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "core/compute_workspace.h"
+#include "core/ggml_graph_cut.h"
+#include "core/runner_cache.h"
+#include "core/tensor.hpp"
+#include "device_residency_manager.h"
+
+namespace sd {
+    struct ComputeTask {
+        std::string description;
+        ggml_cgraph* graph  = nullptr;
+        ggml_tensor* output = nullptr;
+        std::vector<ggml_tensor*> weights;
+        std::map<ggml_tensor*, const void*> inputs;
+        std::vector<std::pair<ggml_tensor*, std::string>> debug_tensors;
+        int n_threads  = 1;
+        bool no_return = false;
+        std::function<bool()> read_outputs;
+    };
+
+    struct ComputeResult {
+        ggml_status status = GGML_STATUS_FAILED;
+        Tensor<float> output;
+    };
+
+    class ExecutionSession {
+        friend class ExecutionEngine;
+        friend class GraphExecution;
+
+        ggml_backend_t backend_;
+        std::weak_ptr<DeviceResidencyManager> residency_manager_;
+        RunnerCache cache_;
+        GraphCutTensorCache cut_cache_;
+        ComputeWorkspace workspace_;
+        std::unordered_set<ggml_tensor*> params_;
+        std::vector<ggml_backend_t> extra_backends_;
+        size_t max_backend_bytes_ = 0;
+        bool layer_split_enabled_ = false;
+        std::vector<size_t> backend_limits_;
+        ggml_graph_cut::PlanCache plan_cache_;
+        std::unordered_map<const ggml_tensor*, ggml_backend_t> weight_assignments_;
+        std::unordered_map<const ggml_tensor*, ggml_backend_t> node_assignments_;
+        bool primary_notice_logged_ = false;
+        std::map<ggml_backend_t, size_t> logged_compute_bytes_;
+        size_t logged_segment_count_ = 0;
+        bool started_                = false;
+        bool executing_              = false;
+
+        size_t resident_bytes(ggml_backend_t backend) const;
+        uintptr_t owner_id() const { return reinterpret_cast<uintptr_t>(this); }
+
+    public:
+        ExecutionSession(ggml_backend_t backend, const std::shared_ptr<DeviceResidencyManager>& manager);
+        ExecutionSession(const ExecutionSession&)            = delete;
+        ExecutionSession& operator=(const ExecutionSession&) = delete;
+
+        RunnerCache& cache() { return cache_; }
+        bool started() const { return started_; }
+        size_t compute_bytes() const { return workspace_.bytes(backend_); }
+        void clear_plan_cache() { plan_cache_.graph_cut_plans.clear(); }
+        void set_max_backend_bytes(size_t bytes) { max_backend_bytes_ = bytes; }
+        void set_layer_split_enabled(bool enabled);
+        void set_backend_limits(const std::vector<size_t>& limits);
+        void set_runtime_backends(const std::vector<ggml_backend_t>& backends);
+    };
+
+    class ExecutionEngine {
+        friend class GraphExecution;
+
+        std::map<uintptr_t, std::unique_ptr<ExecutionSession>> sessions_;
+        DeviceResidencyManager* manager_;
+
+        void sync_residency(ExecutionSession& session);
+
+    public:
+        explicit ExecutionEngine(DeviceResidencyManager* manager = nullptr)
+            : manager_(manager) {}
+        ~ExecutionEngine();
+        ExecutionSession& create_session(ggml_backend_t backend, const std::shared_ptr<DeviceResidencyManager>& manager);
+        void release_session(ExecutionSession& session);
+        bool start(ExecutionSession& session);
+        void end(ExecutionSession& session);
+        void clear_cache(ExecutionSession& session);
+        void discard_graph(ExecutionSession& session);
+        bool ensure_capacity(DeviceResidencyManager& manager,
+                             const DeviceMemoryRequest& request,
+                             const std::vector<ggml_tensor*>& required_params,
+                             const std::vector<std::vector<ggml_tensor*>>& preferred_eviction_order,
+                             const std::vector<ggml_tensor*>& protected_params);
+        // Graphs, input data and output callbacks remain borrowed until execute returns.
+        ComputeResult execute(ExecutionSession& session, const ComputeTask& task);
+    };
+}
+
+#endif  // __SD_CORE_EXECUTION_ENGINE_H__