[baseline] Make start and migrate fail on non-empty schema without migration histories (#835)

Make the `pgroll start` and `pgroll migrate` commands fail when run
against a non-empty schema without a migration history. The error
message suggests running `pgroll baseline` before re-running `start` or
`migrate`.

Running an initial migration against a non-empty schema is not
recommended becauses it can result in a migration that can't be rolled
back if the migration references pre-existing objects in the schema.

The recommended workflow is:

1. Create the initial baseline migration
```
$ pgroll baseline 00_initial_baseline migrations/
```

2. Run `migrate` as usual

```
$ pgroll migrate migrations/ --complete
```

This ensures that the full schema is brought into `pgroll` before
running the first migration.

Part of #364

Author: andrew-farries

cmd/migrate.go

@@ -59,13 +59,23 @@ func migrateCmd() *cobra.Command {
 				return fmt.Errorf("migrations directory %q is not a directory", migrationsDir)
 			}
 
+			// Check whether the schema needs an initial baseline migration
+			needsBaseline, err := m.State().HasExistingSchemaWithoutHistory(ctx, m.Schema())
+			if err != nil {
+				return fmt.Errorf("failed to check for existing schema: %w", err)
+			}
+			if needsBaseline {
+				fmt.Printf("Schema %q is non-empty but has no migration history. Run `pgroll baseline` first\n", m.Schema())
+				return nil
+			}
+
 			migs, err := m.UnappliedMigrations(ctx, os.DirFS(migrationsDir))
 			if err != nil {
 				return fmt.Errorf("failed to get migrations to apply: %w", err)
 			}
 
 			if len(migs) == 0 {
-				fmt.Println("database is up to date; no migrations to apply")
+				fmt.Println("Database is up to date; no migrations to apply")
 				return nil
 			}
 

cmd/start.go

@@ -31,21 +31,32 @@ func startCmd() *cobra.Command {
 		Args:      cobra.ExactArgs(1),
 		ValidArgs: []string{"file"},
 		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
 			fileName := args[0]
 
 			// Create a roll instance and check if pgroll is initialized
-			m, err := NewRollWithInitCheck(cmd.Context())
+			m, err := NewRollWithInitCheck(ctx)
 			if err != nil {
 				return err
 			}
 			defer m.Close()
 
+			// Check whether the schema needs an initial baseline migration
+			needsBaseline, err := m.State().HasExistingSchemaWithoutHistory(ctx, m.Schema())
+			if err != nil {
+				return fmt.Errorf("failed to check for existing schema: %w", err)
+			}
+			if needsBaseline {
+				fmt.Printf("Schema %q is non-empty but has no migration history. Run `pgroll baseline` first\n", m.Schema())
+				return nil
+			}
+
 			c := backfill.NewConfig(
 				backfill.WithBatchSize(batchSize),
 				backfill.WithBatchDelay(batchDelay),
 			)
 
-			return runMigrationFromFile(cmd.Context(), m, fileName, complete, c)
+			return runMigrationFromFile(ctx, m, fileName, complete, c)
 		},
 	}
 

docs/cli/migrate.mdx

@@ -14,3 +14,7 @@ $ pgroll migrate examples/
 will apply migrations from `41_add_enum_column` onwards to the target database.
 
 If the `--complete` flag is passed to `pgroll migrate` the final migration to be applied will be completed. Otherwise the final migration will be left active (started but not completed).
+
+## Existing Database Schema
+
+If you attempt to run `pgroll migrate` against a database that has existing tables but no migration history, the command will fail with an error message. In this case, you should first run `pgroll baseline` to establish a baseline migration that captures the current schema state before applying any new migrations.

docs/cli/start.mdx

@@ -29,3 +29,7 @@ This is equivalent to running `pgroll start` immediately followed by `pgroll com
   workflow is to run `pgroll start`, then gracefully shut down old applications
   before running `pgroll complete` as a separate step.
 </Warning>
+
+## Existing Database Schema
+
+If you attempt to run `pgroll start` against a database that has existing tables but no migration history, the command will fail with an error message. In this case, you should first run `pgroll baseline` to establish a baseline migration that captures the current schema state before starting any new migrations.

internal/testutils/util.go

@@ -145,6 +145,20 @@ func WithUninitializedState(t *testing.T, fn func(*state.State)) {
 	fn(st)
 }
 
+func WithUninitializedStateAndConnectionInfo(t *testing.T, fn func(*state.State, string, *sql.DB)) {
+	t.Helper()
+	ctx := context.Background()
+
+	db, connStr, _ := setupTestDatabase(t)
+
+	st, err := state.New(ctx, connStr, "pgroll")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fn(st, connStr, db)
+}
+
 func WithMigratorInSchemaAndConnectionToContainerWithOptions(t testing.TB, schema string, opts []roll.Option, fn func(mig *roll.Roll, db *sql.DB)) {
 	t.Helper()
 	ctx := context.Background()

pkg/roll/execute.go

@@ -17,6 +17,15 @@ import (
 
 // Start will apply the required changes to enable supporting the new schema version
 func (m *Roll) Start(ctx context.Context, migration *migrations.Migration, cfg *backfill.Config) error {
+	// Fail early if we have existing schema without migration history
+	hasExistingSchema, err := m.state.HasExistingSchemaWithoutHistory(ctx, m.schema)
+	if err != nil {
+		return fmt.Errorf("failed to check for existing schema: %w", err)
+	}
+	if hasExistingSchema {
+		return ErrExistingSchemaWithoutHistory
+	}
+
 	m.logger.LogMigrationStart(migration)
 
 	tablesToBackfill, err := m.StartDDLOperations(ctx, migration)

pkg/roll/execute_test.go

@@ -851,6 +851,35 @@ func TestConnectionsSetPostgresApplicationName(t *testing.T) {
 	}
 }
 
+func TestStartFailsWithExistingSchemaWithoutHistory(t *testing.T) {
+	t.Parallel()
+
+	testutils.WithUninitializedStateAndConnectionInfo(t, func(st *state.State, connStr string, db *sql.DB) {
+		ctx := context.Background()
+
+		// Create a table to before initializing `pgroll`
+		_, err := db.ExecContext(ctx, "CREATE TABLE existing_table (id int)")
+		require.NoError(t, err)
+
+		// Initialize `pgroll`
+		err = st.Init(ctx)
+		require.NoError(t, err)
+
+		// Create a Roll instance
+		m, err := roll.New(ctx, connStr, "public", st)
+		require.NoError(t, err)
+
+		// Attempt to start a migration
+		err = m.Start(ctx, &migrations.Migration{
+			Name:       "01_create_table",
+			Operations: migrations.Operations{createTableOp("new_table")},
+		}, backfill.NewConfig())
+
+		// Verify that the error is ErrExistingSchemaWithoutHistory
+		assert.ErrorIs(t, err, roll.ErrExistingSchemaWithoutHistory)
+	})
+}
+
 func addColumnOp(tableName string) *migrations.OpAddColumn {
 	return &migrations.OpAddColumn{
 		Table: tableName,

pkg/roll/roll.go

@@ -25,7 +25,10 @@ const (
 	applicationName = "pgroll"
 )
 
-var ErrMismatchedMigration = fmt.Errorf("remote migration does not match local migration")
+var (
+	ErrMismatchedMigration          = fmt.Errorf("remote migration does not match local migration")
+	ErrExistingSchemaWithoutHistory = fmt.Errorf("schema has existing tables but no migration history - baseline required")
+)
 
 type Roll struct {
 	pgConn db.DB

pkg/state/state.go

@@ -104,6 +104,43 @@ func (s *State) Schema() string {
 	return s.schema
 }
 
+// HasExistingSchemaWithoutHistory checks if there's an existing schema with
+// tables but no migration history. Returns true if the schema exists, has
+// tables, but has no pgroll migration history
+func (s *State) HasExistingSchemaWithoutHistory(ctx context.Context, schemaName string) (bool, error) {
+	// Check if pgroll is initialized
+	ok, err := s.IsInitialized(ctx)
+	if err != nil {
+		return false, err
+	}
+	if !ok {
+		return false, nil
+	}
+
+	// Check if there's any migration history for this schema
+	var migrationCount int
+	err = s.pgConn.QueryRowContext(ctx,
+		fmt.Sprintf("SELECT COUNT(*) FROM %s.migrations WHERE schema=$1", pq.QuoteIdentifier(s.schema)),
+		schemaName).Scan(&migrationCount)
+	if err != nil {
+		return false, fmt.Errorf("failed to check migration history: %w", err)
+	}
+
+	// If there's migration history, return false
+	if migrationCount > 0 {
+		return false, nil
+	}
+
+	// Check if the schema is empty or not, as determined by ReadSchema
+	schema, err := s.ReadSchema(ctx, schemaName)
+	if err != nil {
+		return false, fmt.Errorf("failed to read schema: %w", err)
+	}
+
+	// Return true if there are tables but no migration history
+	return len(schema.Tables) > 0, nil
+}
+
 // IsActiveMigrationPeriod returns true if there is an active migration
 func (s *State) IsActiveMigrationPeriod(ctx context.Context, schema string) (bool, error) {
 	var isActive bool