cgosqlite,sqlite: expose incremental API and add documentation

It is likely useful to report progress information and offer abort for
large databases, so expose this in the high level API in a safe way.

Author: raggi

cgosqlite/cgosqlite.go

@@ -182,30 +182,31 @@ type backup struct {
 	backup *C.sqlite3_backup
 }
 
-func (b *backup) Step(numPages int) (more bool, err error) {
+func (b *backup) Step(numPages int) (more bool, remaining, pageCount int, err error) {
 	res := C.sqlite3_backup_step(b.backup, C.int(numPages))
+
+	// It is not safe to call remaining and pagecount concurrently with step, so
+	// instead just return them each time.
+	remaining = int(C.sqlite3_backup_remaining(b.backup))
+	pageCount = int(C.sqlite3_backup_pagecount(b.backup))
+
 	if res == C.SQLITE_DONE {
-		return false, nil
+		return false, remaining, pageCount, nil
 	}
 	if res == C.SQLITE_OK {
-		return true, nil
+		return true, remaining, pageCount, nil
 	}
-	return false, errCode(res)
+	return false, remaining, pageCount, errCode(res)
 }
 
 func (b *backup) Finish() error {
 	res := C.sqlite3_backup_finish(b.backup)
+	if res == C.SQLITE_OK {
+		return nil
+	}
 	return errCode(res)
 }
 
-func (b *backup) Remaining() int {
-	return int(C.sqlite3_backup_remaining(b.backup))
-}
-
-func (b *backup) PageCount() int {
-	return int(C.sqlite3_backup_pagecount(b.backup))
-}
-
 func (db *DB) Prepare(query string, prepFlags sqliteh.PrepareFlags) (stmt sqliteh.Stmt, remainingQuery string, err error) {
 	csql := C.CString(query)
 	defer C.free(unsafe.Pointer(csql))

sqlite.go

@@ -855,23 +855,52 @@ func DB(sqlconn SQLConn, fn func(sqliteh.DB) error) error {
 	})
 }
 
-// Backup backups the specified database from srcConn to dstConn.
-func Backup(dstConn SQLConn, dstSchema string, srcConn SQLConn, srcSchema string) error {
-	return DB(dstConn, func(dst sqliteh.DB) error {
+type Backup struct {
+	backup sqliteh.Backup
+	src    sqliteh.DB
+	dst    sqliteh.DB
+}
+
+// NewBackup starts a new backup operation that will read from the SQLite
+// database srcConn, schema srcSchema, and write to the database dstConn, schema
+// dstSchema. SQLite backups can be efficient as long as `srcConn` is the only
+// writer to the database, if there are other writers the backup process will
+// restart on the next Step() after a write. Finish must be called on the
+// returned backup object in order to free resources consumed by the backup
+// operation, even if errors occur during steps of the backup process.
+func NewBackup(dstConn SQLConn, dstSchema string, srcConn SQLConn, srcSchema string) (*Backup, error) {
+	var b Backup
+	err := DB(dstConn, func(dst sqliteh.DB) error {
 		return DB(srcConn, func(src sqliteh.DB) error {
-			b, err := dst.BackupInit(dstSchema, src, srcSchema)
-			if err != nil {
-				return errWithMsg(dst, err, "Backup")
-			}
-			more := true
-			for more {
-				more, err = b.Step(1024)
-				if err != nil {
-					b.Finish()
-					return errWithMsg(dst, err, "Step")
-				}
-			}
-			return errWithMsg(dst, b.Finish(), "Finish")
+			var err error
+			b.src = src
+			b.dst = dst
+			b.backup, err = dst.BackupInit(dstSchema, src, srcSchema)
+			return errWithMsg(dst, err, "Backup")
 		})
 	})
+	return &b, err
+}
+
+// Step must be called repeatedly until it returns more=false. It may return
+// non-fatal busy or locked errors, which the applicaiton may wish to record for
+// tracking purposes. The SQLite docs suggest sleeping between calls to Step in
+// order to allow the application to make forward progress on other work.
+// numPages controls the desired amount of work performed per step. `remaining`
+// is the number of outstanding pages to be copied, and `pageCount` is the total
+// number of pages in the source database.
+func (b *Backup) Step(numPages int) (more bool, remaining, pageCount int, err error) {
+	more, remaining, pageCount, err = b.backup.Step(numPages)
+	err = errWithMsg(b.dst, err, "Step")
+	return
+}
+
+// Finish frees up the backup object and any resources it consumes. It must be
+// called even if errors occured calling Step. If Step reports a fatal error,
+// Finish will also return the same error. Finish can be called at any time to
+// abort a backup operation early.
+func (b *Backup) Finish() error {
+	err := b.backup.Finish()
+	err = errWithMsg(b.dst, err, "Finish")
+	return err
 }

sqlite_test.go

@@ -813,7 +813,30 @@ func TestBackup(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer dstConn.Close()
-	if err := Backup(dstConn, "main", srcConn, "main"); err != nil {
+
+	var backup = func(dstConn *sql.Conn, dstName string, srcConn *sql.Conn, srcName string) error {
+		t.Helper()
+		b, err := NewBackup(dstConn, dstName, srcConn, srcName)
+		if err != nil {
+			return err
+		}
+		var (
+			more      = true
+			remaining int
+			pageCount int
+		)
+		for more {
+			more, remaining, pageCount, err = b.Step(1024)
+			t.Logf("backup step: more=%v, remaining=%d, pageCount=%d", more, remaining, pageCount)
+			if err != nil {
+				t.Errorf("backup step: %v", err)
+				more = false
+			}
+		}
+		return b.Finish()
+	}
+
+	if err := backup(dstConn, "main", srcConn, "main"); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := dstConn.ExecContext(ctx, "ATTACH 'file:dst2?mode=memory' AS dst2;"); err != nil {
@@ -823,7 +846,7 @@ func TestBackup(t *testing.T) {
 	if err := dstConn.QueryRowContext(ctx, "SELECT count(*) FROM t1").Scan(&count); err != nil || count != 1 {
 		t.Fatalf("err=%v, count=%d", err, count)
 	}
-	if err := Backup(dstConn, "dst2", srcConn, "src2"); err != nil {
+	if err := backup(dstConn, "dst2", srcConn, "src2"); err != nil {
 		t.Fatal(err)
 	}
 	count = 0

sqliteh/sqliteh.go

@@ -172,11 +172,9 @@ type Stmt interface {
 // https://www.sqlite.org/c3ref/backup_finish.html
 type Backup interface {
 	// Step is called repeatedly to transfer data between the two DBs.
-	Step(numPages int) (more bool, err error)
+	Step(numPages int) (more bool, remaining, pageCount int, err error)
 	// Finish releases all resources associated with the Backup.
 	Finish() error
-	Remaining() int
-	PageCount() int
 }
 
 // ColumnType are constants for each of the SQLite datatypes.