From 141a653b3ccf7f5e5028d7f2aeb2c6569ba8f9f6 Mon Sep 17 00:00:00 2001 From: Marc Alvarez Date: Wed, 21 Oct 2026 23:42:53 -0600 Subject: [PATCH] Add a MySQL provider github.com/bashhack/testdb/mysql gives each test its own MySQL database: mysql.Setup returns a *sql.DB, mysql.New takes a custom initializer, and migrators, keep-on-failure and parallel use work as they do for PostgreSQL. Templates are not available yet (MySQL cannot clone a database); WithTemplate is refused with ErrTemplatesNotSupported, and the README does not list MySQL until they are. Cleanup is designed around MySQL's hazard: DROP DATABASE does not fail while sessions are connected, it waits on their metadata locks, by default for a year. The provider kills the database's sessions first and runs with a five-second lock_wait_timeout so a session it could not kill makes cleanup fail loudly rather than hang. A test leaks an open transaction and checks the drop still completes promptly. The admin DSN may be go-sql-driver's native form or a mysql:// URL; TEST_MYSQL_URL wins over the shared variables, and a PostgreSQL URL in TEST_DATABASE_URL gets an error that says to set it. Test DSNs are built from the admin config so every parameter carries over. mysql.SQLFiles and LoadSQLFile run multi-statement files. The pieces Setup and New are built from move into core as testdb.RunConfiguredMigrations and testdb.RegisterCleanup, so both providers behave identically around a test; a failed migration under KeepOnFailure now keeps the database too. testdb.InspectCommander lets a provider say how to open a kept database in its own client, so the log shows a mysql command line for MySQL and psql for PostgreSQL. The goose adapter infers the MySQL dialect from a native DSN and the golang-migrate adapter gains MySQL through its mysql driver. A mysql:8.4 service joins the compose file (durability off, as for Postgres) and CI; the Makefile waits for both. --- .github/workflows/ci.yml | 11 + Makefile | 24 +- docker-compose.test.yml | 21 ++ go.mod | 2 + go.sum | 4 + go.work.sum | 4 + helpers.go | 59 +++ keep_test.go | 30 +- migrate/golangmigrate/go.mod | 4 +- migrate/golangmigrate/go.sum | 8 +- migrate/golangmigrate/golangmigrate.go | 29 +- .../golangmigrate/golangmigrate_mysql_test.go | 83 ++++ migrate/golangmigrate/golangmigrate_test.go | 4 +- migrate/goose/go.mod | 2 + migrate/goose/go.sum | 4 + migrate/goose/goose.go | 12 +- migrate/goose/goose_mysql_test.go | 64 ++++ mysql/dsn_test.go | 134 +++++++ mysql/mysql.go | 356 ++++++++++++++++++ mysql/mysql_test.go | 315 ++++++++++++++++ mysql/sqlfiles.go | 91 +++++ postgres/postgres.go | 54 +-- .../migrations_goose/001_create_products.sql | 9 + .../000001_create_test_table.down.sql | 1 + .../000001_create_test_table.up.sql | 4 + testdata/mysql/schema/001_test_table.sql | 4 + testdata/mysql/schema/002_users.sql | 10 + testdata/mysql/schema/003_products.sql | 5 + testdb.go | 14 +- 29 files changed, 1301 insertions(+), 61 deletions(-) create mode 100644 helpers.go create mode 100644 migrate/golangmigrate/golangmigrate_mysql_test.go create mode 100644 migrate/goose/goose_mysql_test.go create mode 100644 mysql/dsn_test.go create mode 100644 mysql/mysql.go create mode 100644 mysql/mysql_test.go create mode 100644 mysql/sqlfiles.go create mode 100644 testdata/mysql/migrations_goose/001_create_products.sql create mode 100644 testdata/mysql/migrations_migrate/000001_create_test_table.down.sql create mode 100644 testdata/mysql/migrations_migrate/000001_create_test_table.up.sql create mode 100644 testdata/mysql/schema/001_test_table.sql create mode 100644 testdata/mysql/schema/002_users.sql create mode 100644 testdata/mysql/schema/003_products.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2de88a5..5037f2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,17 @@ jobs: --health-retries 5 ports: - 5432:5432 + mysql: + image: mysql:8.4 + env: + MYSQL_ROOT_PASSWORD: root + options: >- + --health-cmd "mysql -uroot -proot -e 'SELECT 1'" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + ports: + - 3306:3306 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 diff --git a/Makefile b/Makefile index a2cd49f..2acd63e 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,9 @@ POSTGRES_PORT := 5432 POSTGRES_USER := postgres POSTGRES_PASSWORD := postgres POSTGRES_DB := testdb +MYSQL_SERVICE := test-mysql +MYSQL_USER := root +MYSQL_PASSWORD := root .PHONY: check_pgcli check_pgcli: @@ -83,12 +86,30 @@ define wait_for_postgres echo "✅ Postgres is ready!" endef +# mysqladmin ping answers before the server accepts SQL, so wait for a query. +define wait_for_mysql + @echo "Waiting for MySQL to be ready..." + @attempts=0; \ + while ! docker compose -f $(1) exec -T $(2) mysql -u$(3) -p$(4) -e 'SELECT 1' >/dev/null 2>&1; do \ + attempts=$$((attempts+1)); \ + if [ $$attempts -gt 60 ]; then \ + echo "❌ MySQL failed to become ready after 60 seconds"; \ + echo "Try 'make $(5)' for a fresh start"; \ + exit 1; \ + fi; \ + echo "⏳ Waiting... ($$attempts/60)"; \ + sleep 1; \ + done; \ + echo "✅ MySQL is ready!" +endef + ## test/db/start: Start PostgreSQL test database .PHONY: test/db/start test/db/start: @echo 'Starting test database...' docker compose -f $(POSTGRES_COMPOSE_FILE) up -d $(call wait_for_postgres,$(POSTGRES_COMPOSE_FILE),$(POSTGRES_SERVICE),$(POSTGRES_USER),$(POSTGRES_DB),test/db/restart) + $(call wait_for_mysql,$(POSTGRES_COMPOSE_FILE),$(MYSQL_SERVICE),$(MYSQL_USER),$(MYSQL_PASSWORD),test/db/restart) ## test/db/stop: Stop PostgreSQL test database .PHONY: test/db/stop @@ -117,8 +138,9 @@ test/db/clean: test/db/restart: test/db/clean @echo "Starting fresh Postgres instance..." docker compose -f $(POSTGRES_COMPOSE_FILE) build --force-rm --no-cache $(POSTGRES_SERVICE) - docker compose -f $(POSTGRES_COMPOSE_FILE) up -d $(POSTGRES_SERVICE) + docker compose -f $(POSTGRES_COMPOSE_FILE) up -d $(call wait_for_postgres,$(POSTGRES_COMPOSE_FILE),$(POSTGRES_SERVICE),$(POSTGRES_USER),$(POSTGRES_DB),test/db/restart) + $(call wait_for_mysql,$(POSTGRES_COMPOSE_FILE),$(MYSQL_SERVICE),$(MYSQL_USER),$(MYSQL_PASSWORD),test/db/restart) ## test/db/psql: Connect to Postgres database .PHONY: test/db/psql diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 19f9bb1..a58d1ed 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -22,3 +22,24 @@ services: - "full_page_writes=off" - "-c" - "max_connections=300" + + test-mysql: + image: mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: root + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysql", "-uroot", "-proot", "-e", "SELECT 1"] + interval: 5s + timeout: 3s + retries: 12 + # Durability off, as for Postgres above: this server only ever holds + # throwaway test databases, and these settings make every DDL and drop + # markedly faster. + command: + - "--innodb-flush-log-at-trx-commit=0" + - "--sync-binlog=0" + - "--skip-log-bin" + - "--innodb-doublewrite=0" + - "--innodb-flush-method=O_DIRECT_NO_FSYNC" diff --git a/go.mod b/go.mod index c291090..e73103b 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/bashhack/testdb go 1.27.0 require ( + github.com/go-sql-driver/mysql v1.10.1 github.com/jackc/pgx/v5 v5.11.0 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.0 ) require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect diff --git a/go.sum b/go.sum index 7b6ad39..046141f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,10 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= diff --git a/go.work.sum b/go.work.sum index f1d4c11..43105c9 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,4 +1,8 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..23da018 --- /dev/null +++ b/helpers.go @@ -0,0 +1,59 @@ +package testdb + +import ( + "io" + "testing" +) + +// RunConfiguredMigrations and RegisterCleanup are what every database +// package's Setup and New are built from, so all providers behave the same +// way around a test: migrations run under the test's context, cleanup closes +// the entity before dropping, and KeepOnFailure means the same thing +// everywhere. + +// RunConfiguredMigrations runs the Migrator set with WithMigrator, if any. +// On failure it drops the database (or keeps it, under KeepOnFailure, since +// a half-applied migration is exactly what one wants to look at) and calls +// t.Fatalf, so it does not return. callerName is the function the test +// called, for the message. +func RunConfiguredMigrations(t testing.TB, db *TestDatabase, callerName string) { + t.Helper() + if db.Config().Migrator == nil { + return + } + err := db.RunMigrations(ContextOf(t)) + if err == nil { + return + } + if db.Config().KeepOnFailure { + db.Keep() + } + if closeErr := db.Close(); closeErr != nil { + t.Logf("Warning: failed to close database after migration error: %v", closeErr) + } + t.Fatalf("%s: migrations failed: %v", callerName, err) +} + +// RegisterCleanup arranges for the database to go away when the test ends: +// the entity is closed first if it implements io.Closer, then Close drops +// the database. With KeepOnFailure, a failed test's database is kept +// instead (see Keep). +func RegisterCleanup(t testing.TB, db *TestDatabase) { + t.Cleanup(func() { + if entity := db.Entity(); entity != nil { + if closer, ok := entity.(io.Closer); ok { + if err := closer.Close(); err != nil { + t.Logf("Warning: failed to close entity: %v", err) + } + } + } + + if db.Config().KeepOnFailure && t.Failed() { + db.Keep() + } + + if err := db.Close(); err != nil { + t.Errorf("testdb cleanup failed: %v", err) + } + }) +} diff --git a/keep_test.go b/keep_test.go index 8b10c32..b300363 100644 --- a/keep_test.go +++ b/keep_test.go @@ -30,6 +30,11 @@ func (p *countingProvider) Cleanup(ctx context.Context) error { return p.mockProvider.Cleanup(ctx) } +// InspectCommand stands in for a database client invocation. +func (p *countingProvider) InspectCommand(dsn string) string { + return "connect-with " + dsn +} + func TestCloseDropsByDefault(t *testing.T) { provider := &countingProvider{} db, err := New(t, provider, nil) @@ -74,8 +79,29 @@ func TestKeepLeavesDatabaseAndLogsHowToConnect(t *testing.T) { if kept == "" { t.Fatalf("expected a 'kept database' log line, got %v", spy.logs) } - if !strings.Contains(kept, db.Name()) || !strings.Contains(kept, "psql") || !strings.Contains(kept, db.DSN()) { - t.Errorf("kept-database log should name the database and give a psql command with its DSN, got: %s", kept) + if !strings.Contains(kept, db.Name()) || !strings.Contains(kept, "connect-with "+db.DSN()) { + t.Errorf("kept-database log should name the database and give the provider's connect command, got: %s", kept) + } +} + +func TestKeepLogsDSNWhenProviderHasNoInspectCommand(t *testing.T) { + spy := &verboseSpyTB{TB: t} + db, err := New(spy, &mockProvider{}, nil) // mockProvider has no InspectCommand + if err != nil { + t.Fatal(err) + } + db.Keep() + if err := db.Close(); err != nil { + t.Fatal(err) + } + var kept string + for _, msg := range spy.logs { + if strings.Contains(msg, "kept database") { + kept = msg + } + } + if !strings.Contains(kept, db.DSN()) { + t.Errorf("without an InspectCommander the log should show the DSN, got: %s", kept) } } diff --git a/migrate/golangmigrate/go.mod b/migrate/golangmigrate/go.mod index 78b7a50..2f7b041 100644 --- a/migrate/golangmigrate/go.mod +++ b/migrate/golangmigrate/go.mod @@ -4,11 +4,13 @@ go 1.27.0 require ( github.com/bashhack/testdb v0.0.0 + github.com/go-sql-driver/mysql v1.10.1 github.com/golang-migrate/migrate/v4 v4.20.1 - github.com/jackc/pgx/v5 v5.9.2 + github.com/jackc/pgx/v5 v5.11.0 ) require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/migrate/golangmigrate/go.sum b/migrate/golangmigrate/go.sum index 6cc236b..d1a5e7b 100644 --- a/migrate/golangmigrate/go.sum +++ b/migrate/golangmigrate/go.sum @@ -1,3 +1,5 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -27,6 +29,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/golang-migrate/migrate/v4 v4.20.1 h1:2N/ToVTKrKl58ynBpgeVJ4In7VcLCjWTZtm4eP1LxhU= github.com/golang-migrate/migrate/v4 v4.20.1/go.mod h1:DDPgKVb4ovSWc4FwSPfV2Uz1160f4XBiTHTrAJtljmM= github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa h1:s+4MhCQ6YrzisK6hFJUX53drDT4UsSW3DEhKn0ifuHw= @@ -35,8 +39,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= -github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= diff --git a/migrate/golangmigrate/golangmigrate.go b/migrate/golangmigrate/golangmigrate.go index f23e8ca..3f49028 100644 --- a/migrate/golangmigrate/golangmigrate.go +++ b/migrate/golangmigrate/golangmigrate.go @@ -17,14 +17,17 @@ import ( "os" "strings" + mysqldriver "github.com/go-sql-driver/mysql" "github.com/golang-migrate/migrate/v4" - // Registers the pgx5:// database driver that this package connects with. + // Registers the mysql:// database driver. + _ "github.com/golang-migrate/migrate/v4/database/mysql" + // Registers the pgx5:// database driver. _ "github.com/golang-migrate/migrate/v4/database/pgx/v5" "github.com/golang-migrate/migrate/v4/source/iofs" ) -// Migrator applies golang-migrate migrations to a PostgreSQL database. -// Create one with Dir or FS. +// Migrator applies golang-migrate migrations to a PostgreSQL or MySQL +// database. Create one with Dir or FS. type Migrator struct { fsys fs.FS path string @@ -101,18 +104,28 @@ func (m *Migrator) Migrate(ctx context.Context, dsn string) (err error) { return nil } -// databaseURL rewrites the DSN's scheme to pgx5, the one golang-migrate's -// pgx/v5 driver registers; the rest of the URL is the same. +// databaseURL turns a testdb DSN into the URL golang-migrate's drivers +// register for: a PostgreSQL URL gets the pgx5 scheme, and go-sql-driver's +// native MySQL DSN (which has no scheme) is prefixed with mysql://, which is +// the form golang-migrate's MySQL driver parses. func databaseURL(dsn string) (string, error) { + if !strings.Contains(dsn, "://") { + if _, err := mysqldriver.ParseDSN(dsn); err == nil { + return "mysql://" + dsn, nil + } + return "", errors.New("golangmigrate: the DSN must be a PostgreSQL URL (postgres://...) or a native MySQL DSN (user:pass@tcp(host:port)/db)") + } u, err := url.Parse(dsn) - if err != nil || u.Scheme == "" { - return "", errors.New("golangmigrate: the DSN must be a URL (postgres://...)") + if err != nil { + return "", fmt.Errorf("golangmigrate: parse DSN: %w", err) } switch strings.ToLower(u.Scheme) { case "postgres", "postgresql", "pgx", "pgx5": u.Scheme = "pgx5" + case "mysql": + return "", errors.New("golangmigrate: give MySQL DSNs in go-sql-driver's native form (user:pass@tcp(host:port)/db), as testdb's mysql package does") default: - return "", fmt.Errorf("golangmigrate: unsupported DSN scheme %q; only PostgreSQL is supported", u.Scheme) + return "", fmt.Errorf("golangmigrate: unsupported DSN scheme %q", u.Scheme) } return u.String(), nil } diff --git a/migrate/golangmigrate/golangmigrate_mysql_test.go b/migrate/golangmigrate/golangmigrate_mysql_test.go new file mode 100644 index 0000000..7ca0b91 --- /dev/null +++ b/migrate/golangmigrate/golangmigrate_mysql_test.go @@ -0,0 +1,83 @@ +package golangmigrate_test + +import ( + "context" + "database/sql" + "os" + "testing" + "time" + + "github.com/bashhack/testdb" + "github.com/bashhack/testdb/migrate/golangmigrate" + "github.com/bashhack/testdb/mysql" +) + +const mysqlMigrationsDir = "../../testdata/mysql/migrations_migrate" + +func requireMySQL(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + p := &mysql.Provider{} + if err := p.Initialize(ctx, testdb.Config{}); err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("MySQL is required in CI but not reachable: %v", err) + } + t.Skipf("MySQL not available: %v", err) + } + if err := p.Cleanup(ctx); err != nil { + t.Fatal(err) + } +} + +func mysqlTableExists(t *testing.T, db *sql.DB, table string) bool { + t.Helper() + var n int + err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", table).Scan(&n) + if err != nil { + t.Fatalf("query TABLES: %v", err) + } + return n == 1 +} + +func TestMySQLDir(t *testing.T) { + requireMySQL(t) + db := mysql.Setup(t, testdb.WithMigrator(golangmigrate.Dir(mysqlMigrationsDir))) + + if !mysqlTableExists(t, db, "test_table") { + t.Fatal("expected test_table from the golang-migrate migrations") + } + if !mysqlTableExists(t, db, "schema_migrations") { + t.Fatal("expected golang-migrate's schema_migrations table") + } +} + +func TestMySQLMigrateTwiceIsANoOp(t *testing.T) { + requireMySQL(t) + db := mysql.New(t, &mysql.SQLDBInitializer{}, testdb.WithMigrator(golangmigrate.Dir(mysqlMigrationsDir))) + + if err := golangmigrate.Dir(mysqlMigrationsDir).Migrate(context.Background(), db.DSN()); err != nil { + t.Fatalf("second Migrate on an up-to-date database: %v", err) + } +} + +func TestMySQLURLFormIsRejectedWithGuidance(t *testing.T) { + err := golangmigrate.Dir(mysqlMigrationsDir).Migrate(context.Background(), "mysql://root:root@localhost:3306/x") + if err == nil || !contains(err.Error(), "native form") { + t.Fatalf("expected guidance to use the native DSN form, got: %v", err) + } +} + +func contains(s, sub string) bool { + return len(sub) == 0 || (len(s) >= len(sub) && indexOf(s, sub) >= 0) +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} diff --git a/migrate/golangmigrate/golangmigrate_test.go b/migrate/golangmigrate/golangmigrate_test.go index c542f00..8de99eb 100644 --- a/migrate/golangmigrate/golangmigrate_test.go +++ b/migrate/golangmigrate/golangmigrate_test.go @@ -64,8 +64,8 @@ func TestMissingDirectory(t *testing.T) { func TestNonURLDSN(t *testing.T) { err := golangmigrate.Dir(migrationsDir).Migrate(context.Background(), "host=localhost dbname=x") - if err == nil || !strings.Contains(err.Error(), "must be a URL") { - t.Fatalf("expected a URL DSN error, got: %v", err) + if err == nil || !strings.Contains(err.Error(), "must be a PostgreSQL URL") { + t.Fatalf("expected a DSN form error, got: %v", err) } } diff --git a/migrate/goose/go.mod b/migrate/goose/go.mod index 8f43c7a..10ddab4 100644 --- a/migrate/goose/go.mod +++ b/migrate/goose/go.mod @@ -4,11 +4,13 @@ go 1.27.0 require ( github.com/bashhack/testdb v0.0.0 + github.com/go-sql-driver/mysql v1.10.1 github.com/jackc/pgx/v5 v5.11.0 github.com/pressly/goose/v3 v3.29.0 ) require ( + filippo.io/edwards25519 v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.3 // indirect diff --git a/migrate/goose/go.sum b/migrate/goose/go.sum index 91d45d4..a4f2052 100644 --- a/migrate/goose/go.sum +++ b/migrate/goose/go.sum @@ -1,6 +1,10 @@ +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= diff --git a/migrate/goose/goose.go b/migrate/goose/goose.go index a9d0533..97122c1 100644 --- a/migrate/goose/goose.go +++ b/migrate/goose/goose.go @@ -17,6 +17,8 @@ import ( "os" "strings" + // Registers the "mysql" database/sql driver, and parses native MySQL DSNs. + mysqldriver "github.com/go-sql-driver/mysql" // Registers the "pgx" database/sql driver that goose runs through. _ "github.com/jackc/pgx/v5/stdlib" "github.com/pressly/goose/v3" @@ -45,8 +47,8 @@ func WithDialect(dialect goose.Dialect) Option { // WithSQLDriver sets the database/sql driver name used to connect. The // default follows the dialect: "pgx" for PostgreSQL, "mysql" for MySQL and -// "sqlite" for SQLite. Only the pgx driver is registered by this package; -// import the others yourself. +// "sqlite" for SQLite. The pgx and mysql drivers are registered by this +// package; import any other yourself. func WithSQLDriver(name string) Option { return func(m *Migrator) { m.driver = name @@ -136,10 +138,14 @@ func (m *Migrator) migrations() (fs.FS, error) { return sub, nil } -// dialectFromDSN picks the goose dialect from the DSN's scheme. +// dialectFromDSN picks the goose dialect from the DSN's scheme, or from its +// shape when it is go-sql-driver's native MySQL form, which has no scheme. func dialectFromDSN(dsn string) (goose.Dialect, error) { scheme, _, ok := strings.Cut(dsn, "://") if !ok { + if _, err := mysqldriver.ParseDSN(dsn); err == nil { + return goose.DialectMySQL, nil + } return "", errors.New("goose: cannot infer the dialect from a DSN without a scheme; use goose.WithDialect") } switch strings.ToLower(scheme) { diff --git a/migrate/goose/goose_mysql_test.go b/migrate/goose/goose_mysql_test.go new file mode 100644 index 0000000..9a03005 --- /dev/null +++ b/migrate/goose/goose_mysql_test.go @@ -0,0 +1,64 @@ +package goose_test + +import ( + "context" + "database/sql" + "os" + "testing" + "time" + + "github.com/bashhack/testdb" + "github.com/bashhack/testdb/migrate/goose" + "github.com/bashhack/testdb/mysql" +) + +const mysqlMigrationsDir = "../../testdata/mysql/migrations_goose" + +func requireMySQL(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + p := &mysql.Provider{} + if err := p.Initialize(ctx, testdb.Config{}); err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("MySQL is required in CI but not reachable: %v", err) + } + t.Skipf("MySQL not available: %v", err) + } + if err := p.Cleanup(ctx); err != nil { + t.Fatal(err) + } +} + +func mysqlTableExists(t *testing.T, db *sql.DB, table string) bool { + t.Helper() + var n int + err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", table).Scan(&n) + if err != nil { + t.Fatalf("query TABLES: %v", err) + } + return n == 1 +} + +func TestMySQLDir(t *testing.T) { + requireMySQL(t) + // The dialect and driver are inferred from the native MySQL DSN + db := mysql.Setup(t, testdb.WithMigrator(goose.Dir(mysqlMigrationsDir))) + + if !mysqlTableExists(t, db, "products") { + t.Fatal("expected the products table from the goose migrations") + } + if !mysqlTableExists(t, db, "goose_db_version") { + t.Fatal("expected goose's version table") + } +} + +func TestMySQLMigrateTwiceIsANoOp(t *testing.T) { + requireMySQL(t) + db := mysql.New(t, &mysql.SQLDBInitializer{}, testdb.WithMigrator(goose.Dir(mysqlMigrationsDir))) + + if err := goose.Dir(mysqlMigrationsDir).Migrate(context.Background(), db.DSN()); err != nil { + t.Fatalf("second Migrate on an up-to-date database: %v", err) + } +} diff --git a/mysql/dsn_test.go b/mysql/dsn_test.go new file mode 100644 index 0000000..39b1184 --- /dev/null +++ b/mysql/dsn_test.go @@ -0,0 +1,134 @@ +package mysql + +import ( + "strings" + "testing" + + mysqldriver "github.com/go-sql-driver/mysql" +) + +func TestParseAdminDSN(t *testing.T) { + tests := map[string]struct { + raw string + user string + pass string + addr string + dbName string + params map[string]string + wantErr bool + }{ + "native form": { + raw: "root:secret@tcp(db.example:3307)/admin?parseTime=true", + user: "root", pass: "secret", addr: "db.example:3307", dbName: "admin", + }, + "mysql URL": { + // parseTime is a known parameter the driver stores in a field; + // unknown ones like custom_param land in Params. Both must survive. + raw: "mysql://root:secret@db.example:3307/admin?parseTime=true&custom_param=kept", + user: "root", pass: "secret", addr: "db.example:3307", dbName: "admin", + params: map[string]string{"custom_param": "kept"}, + }, + "mysql URL without port or database": { + raw: "mysql://root@localhost/", + user: "root", addr: "localhost:3306", + }, + "mysql URL without host": { + raw: "mysql:///admin", + wantErr: true, + }, + "garbage": { + raw: "host=localhost dbname=x", + wantErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + config, err := parseAdminDSN(tc.raw) + if tc.wantErr { + if err == nil { + t.Fatalf("expected an error, got config %+v", config) + } + return + } + if err != nil { + t.Fatalf("parseAdminDSN(%q): %v", tc.raw, err) + } + if config.User != tc.user || config.Passwd != tc.pass || config.Addr != tc.addr || config.DBName != tc.dbName { + t.Errorf("got user=%q pass=%q addr=%q db=%q, want user=%q pass=%q addr=%q db=%q", + config.User, config.Passwd, config.Addr, config.DBName, tc.user, tc.pass, tc.addr, tc.dbName) + } + if config.Net != "tcp" { + t.Errorf("Net = %q, want tcp", config.Net) + } + for k, v := range tc.params { + if config.Params[k] != v { + t.Errorf("param %s = %q, want %q", k, config.Params[k], v) + } + } + }) + } +} + +func TestBuildDSNCarriesAdminSettings(t *testing.T) { + p := &Provider{} + config, err := parseAdminDSN("mysql://root:secret@localhost:3306/?parseTime=true&timeout=3s&tls=skip-verify") + if err != nil { + t.Fatal(err) + } + p.adminConfig = config + + dsn, err := p.BuildDSN("test_123_abcd") + if err != nil { + t.Fatal(err) + } + got, err := mysqldriver.ParseDSN(dsn) + if err != nil { + t.Fatalf("BuildDSN produced an unparsable DSN %q: %v", dsn, err) + } + if got.DBName != "test_123_abcd" { + t.Errorf("DBName = %q", got.DBName) + } + if !got.ParseTime || got.Timeout.String() != "3s" || got.TLSConfig != "skip-verify" { + t.Errorf("admin settings were not carried over: parseTime=%v timeout=%s tls=%q", got.ParseTime, got.Timeout, got.TLSConfig) + } + if got.User != "root" || got.Passwd != "secret" || got.Addr != "localhost:3306" { + t.Errorf("credentials or address changed: %+v", got) + } + // The admin config itself is untouched + if p.adminConfig.DBName != "" { + t.Errorf("BuildDSN modified the admin config: DBName = %q", p.adminConfig.DBName) + } +} + +func TestBuildDSNBeforeInitialize(t *testing.T) { + if _, err := (&Provider{}).BuildDSN("x"); err == nil { + t.Fatal("expected an error before Initialize") + } +} + +func TestQuoteIdentifier(t *testing.T) { + if got := quoteIdentifier("plain"); got != "`plain`" { + t.Errorf("got %s", got) + } + if got := quoteIdentifier("with`tick"); got != "`with``tick`" { + t.Errorf("got %s", got) + } +} + +func TestInspectCommand(t *testing.T) { + p := &Provider{} + got := p.InspectCommand("root:p'w@tcp(db.example:3307)/test_1_a?parseTime=true") + for _, want := range []string{"mysql ", "--host='db.example'", "--port=3307", "--user='root'", `--password='p'\''w'`, "'test_1_a'"} { + if !strings.Contains(got, want) { + t.Errorf("InspectCommand = %q, missing %q", got, want) + } + } + if got := p.InspectCommand("not a dsn"); got != "not a dsn" { + t.Errorf("an unparsable DSN should be shown as is, got %q", got) + } + unix := p.InspectCommand("root@unix(/var/run/mysqld/mysqld.sock)/db") + if !strings.Contains(unix, "--socket='/var/run/mysqld/mysqld.sock'") { + t.Errorf("unix socket DSN should use --socket, got %q", unix) + } +} diff --git a/mysql/mysql.go b/mysql/mysql.go new file mode 100644 index 0000000..279f575 --- /dev/null +++ b/mysql/mysql.go @@ -0,0 +1,356 @@ +// Package mysql provides isolated MySQL databases for tests: each test gets +// its own database, created with CREATE DATABASE and dropped when the test +// ends, so tests can run in parallel against one server. +// +// import ( +// "github.com/bashhack/testdb" +// "github.com/bashhack/testdb/migrate/goose" +// "github.com/bashhack/testdb/mysql" +// ) +// +// func TestOrders(t *testing.T) { +// db := mysql.Setup(t, testdb.WithMigrator(goose.Dir("./migrations"))) +// // db is a *sql.DB on a database nobody else touches; cleanup is automatic +// } +// +// The admin connection string is discovered from TEST_MYSQL_URL, then +// TEST_DATABASE_URL, then DATABASE_URL, falling back to DefaultAdminDSN. It +// may be in go-sql-driver's native form (user:pass@tcp(host:port)/db?params) +// or a mysql://user:pass@host:port/db?params URL. The user needs CREATE and +// DROP DATABASE and the right to kill other sessions (CONNECTION_ADMIN). +// +// Templates are not available for MySQL yet; see testdb.TemplateProvider. +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/bashhack/testdb" + mysqldriver "github.com/go-sql-driver/mysql" +) + +const ( + // AdminDSNEnv is the environment variable this provider checks first for + // its admin connection string, before the shared TEST_DATABASE_URL and + // DATABASE_URL. Set it when a project tests against more than one + // database server. + AdminDSNEnv = "TEST_MYSQL_URL" + + // DefaultAdminDSN is used when no override or environment variable is set. + DefaultAdminDSN = "root:root@tcp(localhost:3306)/?parseTime=true" + + // lockWaitTimeout bounds how long DROP DATABASE waits for metadata locks + // held by sessions that could not be killed. MySQL's default is a year, + // which would turn a leaked transaction into a hung test run. + lockWaitTimeout = 5 * time.Second +) + +// MySQL error numbers this package reacts to. +const ( + errLockWaitTimeout = 1205 // ER_LOCK_WAIT_TIMEOUT + errUnknownThread = 1094 // ER_NO_SUCH_THREAD: the session ended on its own +) + +// Provider implements testdb.Provider for MySQL. It does not implement +// testdb.TemplateProvider: MySQL has no way to clone one database from +// another, so WithTemplate is refused with testdb.ErrTemplatesNotSupported. +type Provider struct { + db *sql.DB + conn *sql.Conn // one session, so SET SESSION settings stick + adminDSN string + adminConfig *mysqldriver.Config +} + +var ( + _ testdb.Provider = (*Provider)(nil) + _ testdb.InspectCommander = (*Provider)(nil) +) + +// Initialize resolves the admin DSN and opens the admin session. +func (p *Provider) Initialize(ctx context.Context, cfg testdb.Config) error { + raw := testdb.ResolveAdminDSN(cfg, DefaultAdminDSN, AdminDSNEnv) + config, err := parseAdminDSN(raw) + if err != nil { + if strings.HasPrefix(strings.ToLower(raw), "postgres") { + // The shared TEST_DATABASE_URL points at PostgreSQL; the usual + // case in a project that tests against both databases + return fmt.Errorf("parse admin DSN: it is a PostgreSQL URL; set %s to the MySQL server (%w)", AdminDSNEnv, err) + } + return fmt.Errorf("parse admin DSN: %w", err) + } + p.adminConfig = config + p.adminDSN = config.FormatDSN() + + db, err := sql.Open("mysql", p.adminDSN) + if err != nil { + return fmt.Errorf("open admin connection: %w", err) + } + conn, err := db.Conn(ctx) + if err != nil { + return errors.Join(fmt.Errorf("connect to admin database: %w", err), db.Close()) + } + // Bound how long a drop can wait on locks (see lockWaitTimeout) + seconds := int(lockWaitTimeout / time.Second) + if _, err := conn.ExecContext(ctx, "SET SESSION lock_wait_timeout = "+strconv.Itoa(seconds)); err != nil { + return errors.Join(fmt.Errorf("set lock_wait_timeout: %w", err), conn.Close(), db.Close()) + } + p.db, p.conn = db, conn + return nil +} + +// CreateDatabase creates an empty database with the server's default +// character set and collation. +func (p *Provider) CreateDatabase(ctx context.Context, name string) error { + if _, err := p.conn.ExecContext(ctx, "CREATE DATABASE "+quoteIdentifier(name)); err != nil { + return fmt.Errorf("create database: %w", err) + } + return nil +} + +// DropDatabase drops a database if it exists. Unlike PostgreSQL, MySQL does +// not refuse while other sessions are connected; it waits for the metadata +// locks they hold. TerminateConnections first, so the wait is short, and +// lockWaitTimeout so it is never long. +func (p *Provider) DropDatabase(ctx context.Context, name string) error { + _, err := p.conn.ExecContext(ctx, "DROP DATABASE IF EXISTS "+quoteIdentifier(name)) + if errNumber(err) == errLockWaitTimeout { + return fmt.Errorf("drop database: sessions still held locks on %s after %s: %w", name, lockWaitTimeout, err) + } + if err != nil { + return fmt.Errorf("drop database: %w", err) + } + return nil +} + +// TerminateConnections kills every other session whose current database is +// the given one. Sessions that use the database only through qualified +// names (other_db.table) are not found this way; the pools this package +// creates always connect to the database directly. +func (p *Provider) TerminateConnections(ctx context.Context, name string) error { + rows, err := p.conn.QueryContext(ctx, + "SELECT id FROM information_schema.PROCESSLIST WHERE db = ? AND id <> CONNECTION_ID()", name) + if err != nil { + return fmt.Errorf("list connections: %w", err) + } + var ids []uint64 + for rows.Next() { + var id uint64 + if err := rows.Scan(&id); err != nil { + return errors.Join(fmt.Errorf("list connections: %w", err), rows.Close()) + } + ids = append(ids, id) + } + if err := errors.Join(rows.Err(), rows.Close()); err != nil { + return fmt.Errorf("list connections: %w", err) + } + + for _, id := range ids { + // KILL takes no placeholders; id is a number we just read + _, err := p.conn.ExecContext(ctx, "KILL CONNECTION "+strconv.FormatUint(id, 10)) + if err != nil && errNumber(err) != errUnknownThread { + return fmt.Errorf("kill connection %d: %w", id, err) + } + } + return nil +} + +// BuildDSN returns a DSN for the given database that carries every setting +// of the admin DSN (credentials, address, parseTime, TLS, timeouts). +func (p *Provider) BuildDSN(dbName string) (string, error) { + if p.adminConfig == nil { + return "", errors.New("provider not initialized") + } + config := p.adminConfig.Clone() + config.DBName = dbName + return config.FormatDSN(), nil +} + +// ResolvedAdminDSN returns the admin DSN in use, in native form. +func (p *Provider) ResolvedAdminDSN() string { + return p.adminDSN +} + +// Cleanup closes the admin session. +func (p *Provider) Cleanup(ctx context.Context) error { + var errs []error + if p.conn != nil { + errs = append(errs, p.conn.Close()) + p.conn = nil + } + if p.db != nil { + errs = append(errs, p.db.Close()) + p.db = nil + } + return errors.Join(errs...) +} + +// InspectCommand returns a mysql command line that opens the database, for +// the kept-database log (see testdb.WithKeepOnFailure). +func (p *Provider) InspectCommand(dsn string) string { + config, err := mysqldriver.ParseDSN(dsn) + if err != nil { + return dsn + } + args := []string{"mysql"} + if config.Net == "unix" { + args = append(args, "--socket="+shellQuote(config.Addr)) + } else { + host, port, err := net.SplitHostPort(config.Addr) + if err != nil { + host = config.Addr + } + args = append(args, "--host="+shellQuote(host)) + if port != "" { + args = append(args, "--port="+port) + } + } + if config.User != "" { + args = append(args, "--user="+shellQuote(config.User)) + } + if config.Passwd != "" { + args = append(args, "--password="+shellQuote(config.Passwd)) + } + if config.DBName != "" { + args = append(args, shellQuote(config.DBName)) + } + return strings.Join(args, " ") +} + +// SQLDBInitializer is the default initializer: a *sql.DB on the test +// database with pool settings suited to tests. +type SQLDBInitializer struct { + // ConfigModifier can adjust the pool after it is opened and before it is + // pinged, for example to change SetMaxOpenConns. + ConfigModifier func(*sql.DB) +} + +// InitializeTestDatabase opens and pings a *sql.DB for the test database. +func (i *SQLDBInitializer) InitializeTestDatabase(ctx context.Context, dsn string) (any, error) { + db, err := sql.Open("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("open database: %w", err) + } + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(2) + db.SetConnMaxLifetime(time.Hour) + if i.ConfigModifier != nil { + i.ConfigModifier(db) + } + if err := db.PingContext(ctx); err != nil { + return nil, errors.Join(fmt.Errorf("ping database: %w", err), db.Close()) + } + return db, nil +} + +// Setup creates an isolated MySQL database, runs the configured migrator, +// and returns a *sql.DB on it. Cleanup is registered with t.Cleanup; do not +// close the returned *sql.DB yourself. Calls t.Fatal on any error. +// +// db := mysql.Setup(t, testdb.WithMigrator(goose.Dir("./migrations"))) +func Setup(t testing.TB, opts ...testdb.Option) *sql.DB { + t.Helper() + + db, err := testdb.New(t, &Provider{}, &SQLDBInitializer{}, opts...) + if err != nil { + t.Fatalf("mysql.Setup: %v", err) + } + testdb.RunConfiguredMigrations(t, db, "mysql.Setup") + testdb.RegisterCleanup(t, db) + + return db.Entity().(*sql.DB) +} + +// New creates an isolated MySQL database with a custom initializer (sqlx, +// GORM, your own wrapper) and returns the TestDatabase. Cleanup is +// registered with t.Cleanup. Calls t.Fatal on any error. +// +// db := mysql.New(t, &SqlxInitializer{}, testdb.WithMigrator(goose.Dir("./migrations"))) +// sqlxDB := db.Entity().(*sqlx.DB) +func New(t testing.TB, initializer testdb.DBInitializer, opts ...testdb.Option) *testdb.TestDatabase { + t.Helper() + + if initializer == nil { + t.Fatalf("mysql.New: initializer cannot be nil\n" + + " Use mysql.Setup() for a ready-to-use *sql.DB\n" + + " Use testdb.New() for the low-level API with manual initialization") + } + + db, err := testdb.New(t, &Provider{}, initializer, opts...) + if err != nil { + t.Fatalf("mysql.New: %v", err) + } + testdb.RunConfiguredMigrations(t, db, "mysql.New") + testdb.RegisterCleanup(t, db) + + return db +} + +// parseAdminDSN accepts go-sql-driver's native DSN form or a mysql:// URL. +func parseAdminDSN(raw string) (*mysqldriver.Config, error) { + if strings.HasPrefix(strings.ToLower(raw), "mysql://") { + native, err := urlToNative(raw) + if err != nil { + return nil, err + } + raw = native + } + return mysqldriver.ParseDSN(raw) +} + +// urlToNative rewrites mysql://user:pass@host:port/db?params as +// user:pass@tcp(host:port)/db?params. A missing port means 3306. +func urlToNative(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("parse URL: %w", err) + } + if u.Host == "" { + return "", errors.New("mysql:// URL has no host") + } + var b strings.Builder + if u.User != nil { + b.WriteString(u.User.Username()) + if password, ok := u.User.Password(); ok { + b.WriteString(":" + password) + } + b.WriteString("@") + } + host := u.Host + if u.Port() == "" { + host = net.JoinHostPort(u.Hostname(), "3306") + } + b.WriteString("tcp(" + host + ")/") + b.WriteString(strings.TrimPrefix(u.Path, "/")) + if u.RawQuery != "" { + b.WriteString("?" + u.RawQuery) + } + return b.String(), nil +} + +// quoteIdentifier quotes a name for use as a MySQL identifier. +func quoteIdentifier(name string) string { + return "`" + strings.ReplaceAll(name, "`", "``") + "`" +} + +// shellQuote single-quotes a value for a shell command line. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// errNumber returns the MySQL error number in err, or 0. +func errNumber(err error) uint16 { + var mysqlErr *mysqldriver.MySQLError + if errors.As(err, &mysqlErr) { + return mysqlErr.Number + } + return 0 +} diff --git a/mysql/mysql_test.go b/mysql/mysql_test.go new file mode 100644 index 0000000..505e138 --- /dev/null +++ b/mysql/mysql_test.go @@ -0,0 +1,315 @@ +package mysql_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/bashhack/testdb" + "github.com/bashhack/testdb/mysql" +) + +const schemaDir = "../testdata/mysql/schema" + +// requireMySQL skips the test when no MySQL server answers, so a checkout +// with only Postgres running stays green; in CI (where the service must be +// there) it fails instead, so a broken service cannot pass silently. +func requireMySQL(t *testing.T) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + p := &mysql.Provider{} + if err := p.Initialize(ctx, testdb.Config{}); err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("MySQL is required in CI but not reachable: %v", err) + } + t.Skipf("MySQL not available: %v", err) + } + if err := p.Cleanup(ctx); err != nil { + t.Fatal(err) + } +} + +// adminDB opens a plain connection with the admin credentials, for +// assertions about what exists on the server. +func adminDB(t *testing.T) *sql.DB { + t.Helper() + ctx := context.Background() + p := &mysql.Provider{} + if err := p.Initialize(ctx, testdb.Config{}); err != nil { + t.Fatal(err) + } + dsn := p.ResolvedAdminDSN() + if err := p.Cleanup(ctx); err != nil { + t.Fatal(err) + } + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + return db +} + +func databaseExists(t *testing.T, admin *sql.DB, name string) bool { + t.Helper() + var n int + err := admin.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = ?", name).Scan(&n) + if err != nil { + t.Fatalf("query SCHEMATA: %v", err) + } + return n == 1 +} + +func currentDatabase(t *testing.T, db *sql.DB) string { + t.Helper() + var name string + if err := db.QueryRowContext(context.Background(), "SELECT DATABASE()").Scan(&name); err != nil { + t.Fatalf("SELECT DATABASE(): %v", err) + } + return name +} + +func tableExists(t *testing.T, db *sql.DB, table string) bool { + t.Helper() + var n int + err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", table).Scan(&n) + if err != nil { + t.Fatalf("query TABLES: %v", err) + } + return n == 1 +} + +func TestSetupIsolation(t *testing.T) { + requireMySQL(t) + first := mysql.Setup(t) + second := mysql.Setup(t) + + if currentDatabase(t, first) == currentDatabase(t, second) { + t.Fatal("two Setup calls returned the same database") + } + if _, err := first.ExecContext(context.Background(), "CREATE TABLE only_here (id INT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if tableExists(t, second, "only_here") { + t.Fatal("a table created in one test database is visible in another") + } +} + +func TestParallel(t *testing.T) { + requireMySQL(t) + for i := range 4 { + t.Run(fmt.Sprintf("test%d", i), func(t *testing.T) { + t.Parallel() + db := mysql.Setup(t, testdb.WithMigrator(mysql.SQLFiles(schemaDir))) + if _, err := db.ExecContext(context.Background(), + "INSERT INTO users (email, name) VALUES (?, ?)", fmt.Sprintf("u%d@example.com", i), "n"); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM users").Scan(&n); err != nil { + t.Fatal(err) + } + // 2 from the fixture, plus ours; nobody else's + if n != 3 { + t.Fatalf("expected 3 users in an isolated database, got %d", n) + } + }) + } +} + +func TestCleanupDropsDatabase(t *testing.T) { + requireMySQL(t) + admin := adminDB(t) + + db, err := testdb.New(t, &mysql.Provider{}, &mysql.SQLDBInitializer{}) + if err != nil { + t.Fatal(err) + } + name := db.Name() + if !databaseExists(t, admin, name) { + t.Fatalf("database %s should exist after New", name) + } + if err := db.Entity().(*sql.DB).Close(); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if databaseExists(t, admin, name) { + t.Fatalf("database %s still exists after Close", name) + } +} + +// TestCleanupWithOpenTransactionDoesNotHang is the MySQL-specific hazard: +// DROP DATABASE waits on metadata locks held by open transactions, for up +// to lock_wait_timeout (a year by default). Cleanup must kill the session +// first and finish promptly. +func TestCleanupWithOpenTransactionDoesNotHang(t *testing.T) { + requireMySQL(t) + admin := adminDB(t) + ctx := context.Background() + + db, err := testdb.New(t, &mysql.Provider{}, &mysql.SQLDBInitializer{}, + testdb.WithMigrator(mysql.SQLFiles(schemaDir))) + if err != nil { + t.Fatal(err) + } + if err := db.RunMigrations(ctx); err != nil { + t.Fatal(err) + } + pool := db.Entity().(*sql.DB) + + // Another session takes a metadata lock on a table and never commits + holder, err := sql.Open("mysql", db.DSN()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = holder.Close() }) // its connection was killed; errors here are expected + holderConn, err := holder.Conn(ctx) + if err != nil { + t.Fatal(err) + } + tx, err := holderConn.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO users (email, name) VALUES ('locker@example.com', 'locker')"); err != nil { + t.Fatal(err) + } + + start := time.Now() + if err := pool.Close(); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatalf("Close with an open transaction in another session: %v", err) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Fatalf("cleanup took %s; it should not wait on the leaked transaction", elapsed) + } + if databaseExists(t, admin, db.Name()) { + t.Fatalf("database %s still exists", db.Name()) + } + // The leaked session is gone too + if err := tx.Commit(); err == nil { + t.Error("the leaked transaction should have been killed") + } +} + +// spyTB records what a test under observation does to its testing.TB. +type spyTB struct { + testing.TB + failed bool + cleanups []func() + logs []string +} + +func (s *spyTB) Failed() bool { return s.failed } +func (s *spyTB) Helper() {} +func (s *spyTB) Cleanup(f func()) { s.cleanups = append(s.cleanups, f) } +func (s *spyTB) Logf(f string, a ...any) { s.logs = append(s.logs, fmt.Sprintf(f, a...)) } +func (s *spyTB) runCleanups() { + for i := len(s.cleanups) - 1; i >= 0; i-- { + s.cleanups[i]() + } +} + +func TestKeepOnFailureKeepsDatabaseAndSaysHowToOpenIt(t *testing.T) { + requireMySQL(t) + admin := adminDB(t) + + spy := &spyTB{TB: t} + db := mysql.Setup(spy, testdb.WithKeepOnFailure(), testdb.WithDBPrefix("keep_mysql")) + name := currentDatabase(t, db) + t.Cleanup(func() { + if databaseExists(t, admin, name) { + if _, err := admin.ExecContext(context.Background(), "DROP DATABASE `"+name+"`"); err != nil { + t.Errorf("drop kept database: %v", err) + } + } + }) + + spy.failed = true + spy.runCleanups() + + if !databaseExists(t, admin, name) { + t.Fatalf("database %s was dropped although the test failed with KeepOnFailure", name) + } + var kept string + for _, msg := range spy.logs { + if strings.Contains(msg, "kept database") { + kept = msg + } + } + if !strings.Contains(kept, name) || !strings.Contains(kept, "mysql --host=") { + t.Errorf("expected a mysql command line naming %s, got: %q", name, kept) + } +} + +func TestWithTemplateIsRefused(t *testing.T) { + _, err := testdb.New(t, &mysql.Provider{}, nil, testdb.WithTemplate("anything")) + if !errors.Is(err, testdb.ErrTemplatesNotSupported) { + t.Fatalf("expected ErrTemplatesNotSupported, got %v", err) + } +} + +func TestSQLFilesAndLoadSQLFile(t *testing.T) { + requireMySQL(t) + + t.Run("SQLFiles runs every file, multi-statement included", func(t *testing.T) { + db := mysql.Setup(t, testdb.WithMigrator(mysql.SQLFiles(schemaDir))) + for _, table := range []string{"test_table", "users", "products"} { + if !tableExists(t, db, table) { + t.Errorf("expected %s table", table) + } + } + var n int + if err := db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM users").Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("expected the 2 fixture users from the second statement of 002_users.sql, got %d", n) + } + }) + + t.Run("LoadSQLFile runs just the one", func(t *testing.T) { + db := mysql.Setup(t, testdb.WithMigrator(mysql.LoadSQLFile(schemaDir+"/003_products.sql"))) + if !tableExists(t, db, "products") || tableExists(t, db, "users") { + t.Fatal("LoadSQLFile should create products and nothing else") + } + }) + + t.Run("empty directory is an error", func(t *testing.T) { + err := mysql.SQLFiles(t.TempDir())(context.Background(), "root@tcp(unused:3306)/x") + if err == nil || !strings.Contains(err.Error(), "no .sql files") { + t.Fatalf("expected a 'no .sql files' error, got %v", err) + } + }) +} + +func TestAdminDSNEnvPrecedence(t *testing.T) { + // An address nothing listens on, so Initialize fails fast after resolving + t.Setenv(mysql.AdminDSNEnv, "root:root@tcp(127.0.0.1:1)/?timeout=1s") + t.Setenv("TEST_DATABASE_URL", "root:root@tcp(shared.should.lose:3306)/") + + p := &mysql.Provider{} + if err := p.Initialize(context.Background(), testdb.Config{}); err == nil { + t.Cleanup(func() { _ = p.Cleanup(context.Background()) }) + t.Fatal("expected the connection to the unreachable address to fail") + } + if !strings.Contains(p.ResolvedAdminDSN(), "127.0.0.1:1") { + t.Errorf("resolved %q, want %s to win", p.ResolvedAdminDSN(), mysql.AdminDSNEnv) + } +} diff --git a/mysql/sqlfiles.go b/mysql/sqlfiles.go new file mode 100644 index 0000000..807a7d3 --- /dev/null +++ b/mysql/sqlfiles.go @@ -0,0 +1,91 @@ +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/bashhack/testdb" + mysqldriver "github.com/go-sql-driver/mysql" +) + +// LoadSQLFile returns a migrator that runs one SQL file against the +// database, for schema dumps and fixture data. The file may hold several +// statements. Client-only syntax such as the mysql CLI's DELIMITER command +// is not SQL and will not work. +// +// mysql.Setup(t, testdb.WithMigrator(mysql.LoadSQLFile("schema.sql"))) +func LoadSQLFile(path string) testdb.MigratorFunc { + return func(ctx context.Context, dsn string) error { + return execSQLFiles(ctx, dsn, []string{path}) + } +} + +// SQLFiles returns a migrator that runs every .sql file in dir, in lexical +// order, against the database: the simplest way to apply a schema without a +// migration tool. Name files so they sort the way they should run +// (001_users.sql, 002_orders.sql). The directory is read when the migrator +// runs, not when it is created. +// +// mysql.Setup(t, testdb.WithMigrator(mysql.SQLFiles("./schema"))) +func SQLFiles(dir string) testdb.MigratorFunc { + return func(ctx context.Context, dsn string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("read SQL directory: %w", err) + } + var paths []string + for _, entry := range entries { + if !entry.IsDir() && strings.EqualFold(filepath.Ext(entry.Name()), ".sql") { + paths = append(paths, filepath.Join(dir, entry.Name())) + } + } + if len(paths) == 0 { + return fmt.Errorf("no .sql files in %s", dir) + } + return execSQLFiles(ctx, dsn, paths) + } +} + +// execSQLFiles runs the files in order on one connection opened with +// multiStatements, so a file is one Exec however many statements it holds. +// Every file is read before connecting, so a missing file fails without +// touching the database. +func execSQLFiles(ctx context.Context, dsn string, paths []string) (err error) { + contents := make([]string, len(paths)) + for i, path := range paths { + sqlBytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read SQL file %s: %w", path, err) + } + contents[i] = string(sqlBytes) + } + + config, err := mysqldriver.ParseDSN(dsn) + if err != nil { + return fmt.Errorf("parse DSN: %w", err) + } + config.MultiStatements = true + db, err := sql.Open("mysql", config.FormatDSN()) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer func() { err = errors.Join(err, db.Close()) }() + + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("connect to database: %w", err) + } + defer func() { err = errors.Join(err, conn.Close()) }() + + for i, statements := range contents { + if _, err := conn.ExecContext(ctx, statements); err != nil { + return fmt.Errorf("execute SQL from %s: %w", paths[i], err) + } + } + return nil +} diff --git a/postgres/postgres.go b/postgres/postgres.go index f4d2972..67db32e 100644 --- a/postgres/postgres.go +++ b/postgres/postgres.go @@ -96,8 +96,8 @@ import ( "context" "errors" "fmt" - "io" "net/url" + "strconv" "strings" "testing" "time" @@ -327,42 +327,16 @@ func (p *PostgresProvider) Cleanup(ctx context.Context) error { return nil } -// runMigrationsIfConfigured runs the configured Migrator, if there is one. -// It calls t.Fatalf if migrations fail, so this function does not return on error. -func runMigrationsIfConfigured(t testing.TB, db *testdb.TestDatabase, callerName string) { - if db.Config().Migrator != nil { - if err := db.RunMigrations(testdb.ContextOf(t)); err != nil { - if closeErr := db.Close(); closeErr != nil { - t.Logf("Warning: failed to close database after migration error: %v", closeErr) - } - t.Fatalf("%s: migrations failed: %v", callerName, err) - } - } +// InspectCommand returns a psql command line that opens the database, for +// the kept-database log (see testdb.WithKeepOnFailure). +func (p *PostgresProvider) InspectCommand(dsn string) string { + return "psql " + strconv.Quote(dsn) } -// registerCleanup registers cleanup that closes the connection pool before dropping the database. -// With KeepOnFailure, a failed test's database is kept for inspection instead. -func registerCleanup(t testing.TB, db *testdb.TestDatabase) { - t.Cleanup(func() { - // Close the pool/connection if it implements io.Closer - if entity := db.Entity(); entity != nil { - if closer, ok := entity.(io.Closer); ok { - if err := closer.Close(); err != nil { - t.Logf("Warning: failed to close entity: %v", err) - } - } - } - - if db.Config().KeepOnFailure && t.Failed() { - db.Keep() - } - - // Drop database (TerminateConnections handles any remaining connections) - if err := db.Close(); err != nil { - t.Errorf("testdb cleanup failed: %v", err) - } - }) -} +var ( + _ testdb.TemplateProvider = (*PostgresProvider)(nil) + _ testdb.InspectCommander = (*PostgresProvider)(nil) +) // InitializeTestDatabase creates a pgxpool.Pool for the test database. func (pi *PoolInitializer) InitializeTestDatabase(ctx context.Context, dsn string) (any, error) { @@ -435,9 +409,8 @@ func Setup(t testing.TB, opts ...testdb.Option) *pgxpool.Pool { t.Fatalf("postgres.Setup: %v", err) } - runMigrationsIfConfigured(t, db, "postgres.Setup") - - registerCleanup(t, db) + testdb.RunConfiguredMigrations(t, db, "postgres.Setup") + testdb.RegisterCleanup(t, db) return db.Entity().(*pgxpool.Pool) } @@ -516,9 +489,8 @@ func New(t testing.TB, initializer testdb.DBInitializer, opts ...testdb.Option) t.Fatalf("postgres.New: %v", err) } - runMigrationsIfConfigured(t, db, "postgres.New") - - registerCleanup(t, db) + testdb.RunConfiguredMigrations(t, db, "postgres.New") + testdb.RegisterCleanup(t, db) return db } diff --git a/testdata/mysql/migrations_goose/001_create_products.sql b/testdata/mysql/migrations_goose/001_create_products.sql new file mode 100644 index 0000000..1d1351e --- /dev/null +++ b/testdata/mysql/migrations_goose/001_create_products.sql @@ -0,0 +1,9 @@ +-- +goose Up +CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + price DECIMAL(10, 2) NOT NULL +); + +-- +goose Down +DROP TABLE products; diff --git a/testdata/mysql/migrations_migrate/000001_create_test_table.down.sql b/testdata/mysql/migrations_migrate/000001_create_test_table.down.sql new file mode 100644 index 0000000..97080df --- /dev/null +++ b/testdata/mysql/migrations_migrate/000001_create_test_table.down.sql @@ -0,0 +1 @@ +DROP TABLE test_table; diff --git a/testdata/mysql/migrations_migrate/000001_create_test_table.up.sql b/testdata/mysql/migrations_migrate/000001_create_test_table.up.sql new file mode 100644 index 0000000..fc2e469 --- /dev/null +++ b/testdata/mysql/migrations_migrate/000001_create_test_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE test_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL +); diff --git a/testdata/mysql/schema/001_test_table.sql b/testdata/mysql/schema/001_test_table.sql new file mode 100644 index 0000000..fc2e469 --- /dev/null +++ b/testdata/mysql/schema/001_test_table.sql @@ -0,0 +1,4 @@ +CREATE TABLE test_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL +); diff --git a/testdata/mysql/schema/002_users.sql b/testdata/mysql/schema/002_users.sql new file mode 100644 index 0000000..b6eddc7 --- /dev/null +++ b/testdata/mysql/schema/002_users.sql @@ -0,0 +1,10 @@ +CREATE TABLE users ( + id INT AUTO_INCREMENT PRIMARY KEY, + email VARCHAR(255) NOT NULL UNIQUE, + name VARCHAR(255), + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO users (email, name) VALUES + ('admin@example.com', 'Admin User'), + ('user@example.com', 'Test User'); diff --git a/testdata/mysql/schema/003_products.sql b/testdata/mysql/schema/003_products.sql new file mode 100644 index 0000000..54435fa --- /dev/null +++ b/testdata/mysql/schema/003_products.sql @@ -0,0 +1,5 @@ +CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + price DECIMAL(10, 2) NOT NULL +); diff --git a/testdb.go b/testdb.go index 67692d9..fdf0cb2 100644 --- a/testdb.go +++ b/testdb.go @@ -152,6 +152,14 @@ type TemplateProvider interface { TemplateExists(ctx context.Context, name string) (bool, error) } +// InspectCommander is implemented by providers that can say how to open a +// database in that database's own command-line client (psql, mysql, ...). +// The kept-database log line (see Keep and WithKeepOnFailure) shows that +// command; without it, it shows the DSN. +type InspectCommander interface { + InspectCommand(dsn string) string +} + // DBInitializer defines the interface for custom database initialization in tests. // // # When You Need a Custom Initializer @@ -380,8 +388,12 @@ func New(t testing.TB, provider Provider, initializer DBInitializer, opts ...Opt td.cleanup = func() error { if td.keep { + how := testDSN + if ic, ok := provider.(InspectCommander); ok { + how = ic.InspectCommand(testDSN) + } // Always worth saying, since someone asked to look at it - t.Logf("testdb: kept database %s for inspection:\n psql %q", dbName, testDSN) + t.Logf("testdb: kept database %s for inspection:\n %s", dbName, how) } else { if err := provider.TerminateConnections(cleanupCtx, dbName); err != nil { return &Error{