From 73f1a19b7fb0d177663213dbab73a12c99b3947f Mon Sep 17 00:00:00 2001 From: Marc Alvarez Date: Thu, 22 Oct 2026 00:51:05 -0600 Subject: [PATCH] Warm the MySQL ready pool before EnsureTemplate returns The first GOMAXPROCS tests of a package used to find an empty pool and clone on demand. EnsureTemplate now waits for the pool to fill once, which costs one round of parallel clones, so every test from the first takes a prepared database; WithReadyWarmup(false) skips the wait. A ctx that ends during the warm-up releases the template on the way out. Also says, in the package and clone docs, what a clone deliberately does not copy: events (a scheduler job firing inside a test's clone would make it nondeterministic) and grants (tests connect as the admin user); that DEFINER clauses are dropped; and that a foreign key to another database keeps pointing there. The package doc no longer says templates are unavailable. --- mysql/clone.go | 6 ++++ mysql/mysql.go | 4 ++- mysql/pool.go | 24 +++++++++++++ mysql/template.go | 28 ++++++++++++--- mysql/template_benchmark_test.go | 2 +- mysql/template_test.go | 61 +++++++++++++++++++++++++++++++- 6 files changed, 118 insertions(+), 7 deletions(-) diff --git a/mysql/clone.go b/mysql/clone.go index fb32480..affb2e3 100644 --- a/mysql/clone.go +++ b/mysql/clone.go @@ -16,6 +16,12 @@ import ( // then views, triggers, procedures and functions from their SHOW CREATE // forms. Tables named _testdb_* hold testdb's own bookkeeping inside a // template and are not copied. +// +// Deliberately not copied: events (a scheduler job firing inside a test's +// clone would make the test nondeterministic) and grants (tests connect as +// the admin user). DEFINER clauses are dropped, so objects belong to +// whoever runs the clone. A foreign key that references a table in another +// database keeps pointing at that database. // bookkeepingPrefix names the tables testdb keeps inside a template. const bookkeepingPrefix = "_testdb_" diff --git a/mysql/mysql.go b/mysql/mysql.go index 279f575..88749c9 100644 --- a/mysql/mysql.go +++ b/mysql/mysql.go @@ -19,7 +19,9 @@ // or a mysql://user:pass@host:port/db?params URL. The user needs CREATE and // DROP DATABASE and the right to kill other sessions (CONNECTION_ADMIN). // -// Templates are not available for MySQL yet; see testdb.TemplateProvider. +// Templates (EnsureTemplate) build the schema once and hand each test a +// clone prepared ahead of time, which is where MySQL's speed comes from; +// see Template and WithReadyDatabases. package mysql import ( diff --git a/mysql/pool.go b/mysql/pool.go index 884f1b4..a393bd5 100644 --- a/mysql/pool.go +++ b/mysql/pool.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "sync" + "time" "github.com/bashhack/testdb" ) @@ -61,6 +62,29 @@ func startReadyPool(tpl *Template, size, workers int) (*readyPool, error) { return pool, nil } +// waitFull blocks until the pool holds its full number of prepared +// databases, the workers have given up, or ctx ends. +func (p *readyPool) waitFull(ctx context.Context) error { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + if len(p.ready) == cap(p.ready) { + return nil + } + p.mu.Lock() + failed := p.failure != nil + p.mu.Unlock() + if failed { + return nil // already logged; tests will clone on demand + } + select { + case <-ticker.C: + case <-ctx.Done(): + return ctx.Err() + } + } +} + // take returns a prepared database, or false without waiting. func (p *readyPool) take() (string, bool) { select { diff --git a/mysql/template.go b/mysql/template.go index 61a8928..5760afa 100644 --- a/mysql/template.go +++ b/mysql/template.go @@ -71,6 +71,7 @@ type templateConfig struct { ready int readyWorkers int readyPrefix string + warmup bool } // WithFingerprint identifies the template's content, typically a hash of the @@ -103,13 +104,24 @@ func WithTemplateAdminDSN(dsn string) TemplateOption { // default is runtime.GOMAXPROCS(0), which is also go test's default // -parallel. 0 disables the pool: every test clones on demand. // -// The pool keeps up as long as tests take clones no faster than the -// workers (see WithReadyWorkers) can make them. When it runs dry, a test -// clones on demand rather than wait. +// EnsureTemplate fills the pool once before returning (see +// WithReadyWarmup), so the first tests find it full. After that it keeps +// up as long as tests take clones no faster than the workers (see +// WithReadyWorkers) can make them; the pool size is also the number of +// back-to-back quick tests it can absorb. When it runs dry, a test clones +// on demand rather than wait. func WithReadyDatabases(n int) TemplateOption { return func(c *templateConfig) { c.ready = n } } +// WithReadyWarmup sets whether EnsureTemplate waits for the pool to fill +// before returning. On by default: it costs one round of parallel clones, +// once, and means every test from the first finds a prepared database. +// Turn it off to start tests immediately and let the pool fill alongside. +func WithReadyWarmup(on bool) TemplateOption { + return func(c *templateConfig) { c.warmup = on } +} + // WithReadyWorkers sets how many clones are made in parallel to keep the // pool full. The server handles concurrent clones well, so the default is // the pool size. @@ -154,7 +166,7 @@ func EnsureTemplate(ctx context.Context, name string, setup SetupFunc, opts ...T if err := validateTemplateName(name); err != nil { return nil, fmt.Errorf("mysql.EnsureTemplate: %w", err) } - cfg := templateConfig{ready: runtime.GOMAXPROCS(0), readyPrefix: "test"} + cfg := templateConfig{ready: runtime.GOMAXPROCS(0), readyPrefix: "test", warmup: true} for _, opt := range opts { opt(&cfg) } @@ -180,6 +192,14 @@ func EnsureTemplate(ctx context.Context, name string, setup SetupFunc, opts ...T return tpl, nil } tpl.pool = pool + if cfg.warmup { + if err := pool.waitFull(ctx); err != nil { + // ctx ended mid warm-up: nobody gets the handle, so let go of + // the template properly + err = errors.Join(err, tpl.Release(context.WithoutCancel(ctx))) + return nil, fmt.Errorf("mysql.EnsureTemplate %q: warm up prepared databases: %w", name, err) + } + } } return tpl, nil } diff --git a/mysql/template_benchmark_test.go b/mysql/template_benchmark_test.go index b3c701e..7e01ee7 100644 --- a/mysql/template_benchmark_test.go +++ b/mysql/template_benchmark_test.go @@ -121,7 +121,7 @@ func BenchmarkTemplateReady(b *testing.B) { schemaDir := writeBenchSchema(b) tpl := ensureBenchTemplate(b, "bench_mysql_tpl_ready", schemaDir, mysql.WithReadyDatabases(8), mysql.WithReadyWorkers(workers), mysql.WithReadyPrefix("bench_ready")) - time.Sleep(6 * time.Second) // let the pool fill once + // EnsureTemplate warms the pool before returning pooled := 0 b.ReportAllocs() for b.Loop() { diff --git a/mysql/template_test.go b/mysql/template_test.go index ae2ef37..51778c8 100644 --- a/mysql/template_test.go +++ b/mysql/template_test.go @@ -280,7 +280,10 @@ func TestTemplateReadyPool(t *testing.T) { } }) - waitFor(t, "the pool to fill", func() bool { return tpl.Ready() == 2 }) + // Warm-up is on by default: the pool is full when EnsureTemplate returns + if n := tpl.Ready(); n != 2 { + t.Fatalf("Ready() = %d right after EnsureTemplate, want 2 (warm-up)", n) + } // A test takes a prepared database: the name carries the pool's prefix, // the clone is complete, and the pool refills @@ -310,6 +313,62 @@ func TestTemplateReadyPool(t *testing.T) { t.Logf("5 rapid takes: %d served on demand, %d from the pool", onDemand, 5-onDemand) } +func TestTemplateReadyWarmupOff(t *testing.T) { + requireMySQL(t) + ctx := context.Background() + name := uniqueTemplateName(t, "tpl_nowarm") + + start := time.Now() + tpl, err := mysql.EnsureTemplate(ctx, name, mysql.SQLFiles(schemaDir), + mysql.WithReadyDatabases(2), mysql.WithReadyWarmup(false), mysql.WithReadyPrefix("nowarm")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { release(t, tpl) }) + t.Logf("EnsureTemplate without warm-up returned in %s with %d ready", time.Since(start), tpl.Ready()) + + // The pool fills alongside, and tests work meanwhile + db := mysql.Setup(t, tpl.Option()) + if !tableExists(t, db, "users") { + t.Fatal("clone lacks the schema") + } + waitFor(t, "the pool to fill in the background", func() bool { return tpl.Ready() == 2 }) +} + +func TestTemplateWarmupCancelled(t *testing.T) { + requireMySQL(t) + admin := adminDB(t) + name := uniqueTemplateName(t, "tpl_warmcancel") + + // A warm-up one worker cannot finish in time: 256 clones against a + // deadline a few clones long. Whether the deadline lands during the + // build or during the warm-up, the outcome must be the same. + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + tpl, err := mysql.EnsureTemplate(ctx, name, mysql.SQLFiles(schemaDir), + mysql.WithReadyDatabases(256), mysql.WithReadyWorkers(1), mysql.WithReadyPrefix("warmcancel")) + if err == nil { + release(t, tpl) + t.Fatal("256 clones in 500 ms: the warm-up should not have finished") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected a deadline error, got %v", err) + } + // Nothing was left behind: the template and its prepared databases were + // released on the way out + if databaseExists(t, admin, name) { + t.Errorf("template %s left behind after a cancelled warm-up", name) + } + var leftovers int + if err := admin.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME LIKE 'warmcancel\\_%'").Scan(&leftovers); err != nil { + t.Fatal(err) + } + if leftovers != 0 { + t.Errorf("%d prepared databases left behind after a cancelled warm-up", leftovers) + } +} + func TestTemplateNameValidation(t *testing.T) { ctx := context.Background() if _, err := mysql.EnsureTemplate(ctx, "", nil); err == nil {