Merge branch 'lukeshu/process-without-pv'

8 files changed, 148 insertions, 72 deletions

diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go b/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go
new file mode 100644
index 0000000..a41a16b
--- /dev/null
+++ b/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go
@@ -0,0 +1,49 @@
+// Copyright (C) 2023  Luke Shumaker <lukeshu@lukeshu.com>
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+package rebuildmappings
+
+import (
+	"fmt"
+
+	"git.lukeshu.com/btrfs-progs-ng/lib/binstruct"
+	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
+	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree"
+	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
+	"git.lukeshu.com/btrfs-progs-ng/lib/diskio"
+	"git.lukeshu.com/btrfs-progs-ng/lib/slices"
+)
+
+type phonyFile struct {
+	size btrfsvol.PhysicalAddr
+	sb   btrfstree.Superblock
+}
+
+var _ diskio.File[btrfsvol.PhysicalAddr] = (*phonyFile)(nil)
+
+func NewPhonyFile(size btrfsvol.PhysicalAddr, sb btrfstree.Superblock) *phonyFile {
+	return &phonyFile{
+		size: size,
+		sb:   sb,
+	}
+}
+
+func (f *phonyFile) Name() string                { return fmt.Sprintf("phony_file:device_id=%v", f.sb.DevItem.DevID) }
+func (f *phonyFile) Size() btrfsvol.PhysicalAddr { return f.size }
+func (*phonyFile) Close() error                  { return nil }
+
+func (f *phonyFile) ReadAt(p []byte, off btrfsvol.PhysicalAddr) (int, error) {
+	if len(p) == int(btrfs.SuperblockSize) && slices.Contains(off, btrfs.SuperblockAddrs) {
+		bs, err := binstruct.Marshal(f.sb)
+		if err != nil {
+			return 0, err
+		}
+		return copy(p, bs), nil
+	}
+	panic(fmt.Errorf("%T: should not happen: ReadAt should not be called for a phony file", f))
+}
+
+func (f *phonyFile) WriteAt([]byte, btrfsvol.PhysicalAddr) (int, error) {
+	panic(fmt.Errorf("%T: should not happen: WriteAt should not be called for a phony file", f))
+}
diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/scan.go b/cmd/btrfs-rec/inspect/rebuildmappings/scan.go
index 9454ca0..d689136 100644
--- a/cmd/btrfs-rec/inspect/rebuildmappings/scan.go
+++ b/cmd/btrfs-rec/inspect/rebuildmappings/scan.go
@@ -25,6 +25,11 @@ import (
 
 // Result types ////////////////////////////////////////////////////////////////
 
+type ScanResult struct {
+	Mappings []btrfsvol.Mapping
+	Devices  ScanDevicesResult
+}
+
 type ScanDevicesResult = map[btrfsvol.DeviceID]ScanOneDeviceResult
 
 type ScanOneDeviceResult struct {
diff --git a/cmd/btrfs-rec/inspect_listnodes.go b/cmd/btrfs-rec/inspect_listnodes.go
index d9b24ed..eb088e8 100644
--- a/cmd/btrfs-rec/inspect_listnodes.go
+++ b/cmd/btrfs-rec/inspect_listnodes.go
@@ -28,7 +28,7 @@ func init() {
 			"to take advantage of the sector-by-sector scan that's already " +
 			"performed by `btrfs-rec inspect rebuild-mappings scan`.",
 		Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)),
-		RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+		RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
 			nodeList, err := btrfsutil.ListNodes(ctx, fs)
diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go
index ae2e415..dcd4b0a 100644
--- a/cmd/btrfs-rec/inspect_rebuildmappings.go
+++ b/cmd/btrfs-rec/inspect_rebuildmappings.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"git.lukeshu.com/go/lowmemjson"
@@ -37,7 +38,7 @@ func init() {
 			"\tbtrfs-rec inspect rebuild-mappings scan > SCAN.json   # read\n" +
 			"\tbtrfs-rec inspect rebuild-mappings process SCAN.json  # CPU\n",
 		Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
-		RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+		RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
 			scanResults, err := rebuildmappings.ScanDevices(ctx, fs)
@@ -67,14 +68,19 @@ func init() {
 		Use:   "scan",
 		Short: "Read from the filesystem all data nescessary to rebuild the mappings",
 		Args:  cliutil.WrapPositionalArgs(cobra.NoArgs),
-		RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) {
+		RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) {
 			ctx := cmd.Context()
 
-			scanResults, err := rebuildmappings.ScanDevices(ctx, fs)
+			devResults, err := rebuildmappings.ScanDevices(ctx, fs)
 			if err != nil {
 				return err
 			}
 
+			scanResults := rebuildmappings.ScanResult{
+				Mappings: fs.LV.Mappings(),
+				Devices:  devResults,
+			}
+
 			dlog.Info(ctx, "Writing scan results to stdout...")
 			if err := writeJSONFile(os.Stdout, scanResults, lowmemjson.ReEncoderConfig{
 				Indent:                "\t",
@@ -89,6 +95,7 @@ func init() {
 		}),
 	})
 
+	var scanResults rebuildmappings.ScanResult
 	cmd.AddCommand(&cobra.Command{
 		Use:   "process",
 		Short: "Rebuild the mappings based on previously read data",
@@ -97,13 +104,37 @@ func init() {
 			ctx := cmd.Context()
 
 			dlog.Infof(ctx, "Reading %q...", args[0])
-			scanResults, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, args[0])
+			var err error
+			scanResults, err = readJSONFile[rebuildmappings.ScanResult](ctx, args[0])
 			if err != nil {
 				return err
 			}
 			dlog.Infof(ctx, "... done reading %q", args[0])
 
-			if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults); err != nil {
+			pvDevices := fs.LV.PhysicalVolumes()
+			for _, devID := range maps.SortedKeys(scanResults.Devices) {
+				if maps.HasKey(pvDevices, devID) {
+					continue
+				}
+				devFile := &btrfs.Device{
+					File: rebuildmappings.NewPhonyFile(
+						scanResults.Devices[devID].Size,
+						scanResults.Devices[devID].Superblock.Val),
+				}
+				if err := fs.AddDevice(ctx, devFile); err != nil {
+					return fmt.Errorf("device file: %q: %w", devFile.Name(), err)
+				}
+			}
+			for _, mapping := range scanResults.Mappings {
+				if err := fs.LV.AddMapping(mapping); err != nil {
+					return err
+				}
+			}
+			return nil
+		}, func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
+			ctx := cmd.Context()
+
+			if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults.Devices); err != nil {
 				return err
 			}
 
@@ -132,17 +163,17 @@ func init() {
 		RunE: run(func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
-			scanResults, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, args[0])
+			scanResults, err := readJSONFile[rebuildmappings.ScanResult](ctx, args[0])
 			if err != nil {
 				return err
 			}
 
 			var cnt int
-			for _, devResults := range scanResults {
+			for _, devResults := range scanResults.Devices {
 				cnt += len(devResults.FoundNodes)
 			}
 			set := make(containers.Set[btrfsvol.LogicalAddr], cnt)
-			for _, devResults := range scanResults {
+			for _, devResults := range scanResults.Devices {
 				for laddr := range devResults.FoundNodes {
 					set.Insert(laddr)
 				}
diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go
index 39ba9ef..7f89369 100644
--- a/cmd/btrfs-rec/main.go
+++ b/cmd/btrfs-rec/main.go
@@ -21,6 +21,7 @@ import (
 	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
 	"git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil"
 	"git.lukeshu.com/btrfs-progs-ng/lib/containers"
+	"git.lukeshu.com/btrfs-progs-ng/lib/diskio"
 	"git.lukeshu.com/btrfs-progs-ng/lib/profile"
 	"git.lukeshu.com/btrfs-progs-ng/lib/textui"
 )
@@ -158,29 +159,63 @@ func run(runE func(*cobra.Command, []string) error) func(*cobra.Command, []strin
 	}
 }
 
-func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
+func runWithRawFS(
+	overrideInitChunks func(*btrfs.FS, *cobra.Command, []string) error,
+	runE func(*btrfs.FS, *cobra.Command, []string) error,
+) func(*cobra.Command, []string) error {
 	return run(func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+
 		maybeSetErr := func(_err error) {
 			if _err != nil && err == nil {
 				err = _err
 			}
 		}
 
-		if len(globalFlags.pvs) == 0 {
+		if len(globalFlags.pvs) == 0 && overrideInitChunks == nil {
 			// We do this here instead of calling argparser.MarkPersistentFlagRequired("pv") so that
 			// it doesn't interfere with the `help` sub-command.
 			return cliutil.FlagErrorFunc(cmd, fmt.Errorf("must specify 1 or more physical volumes with --pv"))
 		}
-		fs, err := btrfsutil.Open(cmd.Context(), globalFlags.openFlag, globalFlags.pvs...)
-		if err != nil {
-			return err
-		}
+		fs := new(btrfs.FS)
 		defer func() {
 			maybeSetErr(fs.Close())
 		}()
+		for i, filename := range globalFlags.pvs {
+			dlog.Debugf(ctx, "Adding device file %d/%d %q...", i, len(globalFlags.pvs), filename)
+			osFile, err := os.OpenFile(filename, globalFlags.openFlag, 0)
+			if err != nil {
+				return fmt.Errorf("device file %q: %w", filename, err)
+			}
+			typedFile := &diskio.OSFile[btrfsvol.PhysicalAddr]{
+				File: osFile,
+			}
+			bufFile := diskio.NewBufferedFile[btrfsvol.PhysicalAddr](
+				ctx,
+				typedFile,
+				//nolint:gomnd // False positive: gomnd.ignored-functions=[textui.Tunable] doesn't support type params.
+				textui.Tunable[btrfsvol.PhysicalAddr](16*1024), // block size: 16KiB
+				textui.Tunable(1024),                           // number of blocks to buffer; total of 16MiB
+			)
+			devFile := &btrfs.Device{
+				File: bufFile,
+			}
+			if err := fs.AddDevice(ctx, devFile); err != nil {
+				return fmt.Errorf("device file %q: %w", filename, err)
+			}
+		}
+		if overrideInitChunks != nil {
+			if err := overrideInitChunks(fs, cmd, args); err != nil {
+				return err
+			}
+		} else {
+			if err := fs.InitChunks(ctx); err != nil {
+				dlog.Errorf(ctx, "error: InitChunks: %v", err)
+			}
+		}
 
 		if globalFlags.mappings != "" {
-			mappingsJSON, err := readJSONFile[[]btrfsvol.Mapping](cmd.Context(), globalFlags.mappings)
+			mappingsJSON, err := readJSONFile[[]btrfsvol.Mapping](ctx, globalFlags.mappings)
 			if err != nil {
 				return err
 			}
@@ -196,7 +231,7 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co
 }
 
 func runWithRawFSAndNodeList(runE func(*btrfs.FS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error {
-	return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+	return runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
 		ctx := cmd.Context()
 
 		var nodeList []btrfsvol.LogicalAddr
@@ -245,7 +280,7 @@ func _runWithReadableFS(wantNodeList bool, runE func(btrfs.ReadableFS, []btrfsvo
 		if wantNodeList || globalFlags.rebuild || globalFlags.treeRoots != "" {
 			return runWithRawFSAndNodeList(inner)(cmd, args)
 		}
-		return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
+		return runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
 			return inner(fs, nil, cmd, args)
 		})(cmd, args)
 	}
diff --git a/lib/btrfs/io1_pv.go b/lib/btrfs/io1_pv.go
index 3b13f73..46eb24d 100644
--- a/lib/btrfs/io1_pv.go
+++ b/lib/btrfs/io1_pv.go
@@ -30,19 +30,18 @@ var SuperblockAddrs = []btrfsvol.PhysicalAddr{
 	0x40_0000_0000, // 256GiB
 }
 
-var superblockSize = binstruct.StaticSize(btrfstree.Superblock{})
+var SuperblockSize = btrfsvol.PhysicalAddr(binstruct.StaticSize(btrfstree.Superblock{}))
 
 func (dev *Device) Superblocks() ([]*diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock], error) {
 	if dev.cacheSuperblocks != nil {
 		return dev.cacheSuperblocks, nil
 	}
-	superblockSize := btrfsvol.PhysicalAddr(superblockSize)
 
 	sz := dev.Size()
 
 	var ret []*diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock]
 	for i, addr := range SuperblockAddrs {
-		if addr+superblockSize <= sz {
+		if addr+SuperblockSize <= sz {
 			superblock := &diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock]{
 				File: dev,
 				Addr: addr,
diff --git a/lib/btrfs/io2_lv.go b/lib/btrfs/io2_lv.go
index e9de215..16c5987 100644
--- a/lib/btrfs/io2_lv.go
+++ b/lib/btrfs/io2_lv.go
@@ -43,7 +43,7 @@ func (fs *FS) AddDevice(ctx context.Context, dev *Device) error {
 	}
 	fs.cacheSuperblocks = nil
 	fs.cacheSuperblock = nil
-	if err := fs.initDev(ctx, *sb); err != nil {
+	if err := fs.initDev(*sb); err != nil {
 		dlog.Errorf(ctx, "error: AddDevice: %q: %v", dev.Name(), err)
 	}
 	return nil
@@ -142,14 +142,14 @@ func (fs *FS) ReInit(ctx context.Context) error {
 		if err != nil {
 			return fmt.Errorf("file %q: %w", dev.Name(), err)
 		}
-		if err := fs.initDev(ctx, *sb); err != nil {
+		if err := fs.initDev(*sb); err != nil {
 			return fmt.Errorf("file %q: %w", dev.Name(), err)
 		}
 	}
-	return nil
+	return fs.InitChunks(ctx)
 }
 
-func (fs *FS) initDev(ctx context.Context, sb btrfstree.Superblock) error {
+func (fs *FS) initDev(sb btrfstree.Superblock) error {
 	syschunks, err := sb.ParseSysChunkArray()
 	if err != nil {
 		return err
@@ -161,6 +161,10 @@ func (fs *FS) initDev(ctx context.Context, sb btrfstree.Superblock) error {
 			}
 		}
 	}
+	return nil
+}
+
+func (fs *FS) InitChunks(ctx context.Context) error {
 	chunkTree, err := fs.ForrestLookup(ctx, btrfsprim.CHUNK_TREE_OBJECTID)
 	if err != nil {
 		return err
diff --git a/lib/btrfsutil/open.go b/lib/btrfsutil/open.go
deleted file mode 100644
index abbd466..0000000
--- a/lib/btrfsutil/open.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright (C) 2022-2023  Luke Shumaker <lukeshu@lukeshu.com>
-//
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package btrfsutil
-
-import (
-	"context"
-	"fmt"
-	"os"
-
-	"github.com/datawire/dlib/dlog"
-
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
-	"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
-	"git.lukeshu.com/btrfs-progs-ng/lib/diskio"
-	"git.lukeshu.com/btrfs-progs-ng/lib/textui"
-)
-
-func Open(ctx context.Context, flag int, filenames ...string) (*btrfs.FS, error) {
-	fs := new(btrfs.FS)
-	for i, filename := range filenames {
-		dlog.Debugf(ctx, "Adding device file %d/%d %q...", i, len(filenames), filename)
-		osFile, err := os.OpenFile(filename, flag, 0)
-		if err != nil {
-			_ = fs.Close()
-			return nil, fmt.Errorf("device file %q: %w", filename, err)
-		}
-		typedFile := &diskio.OSFile[btrfsvol.PhysicalAddr]{
-			File: osFile,
-		}
-		bufFile := diskio.NewBufferedFile[btrfsvol.PhysicalAddr](
-			ctx,
-			typedFile,
-			//nolint:gomnd // False positive: gomnd.ignored-functions=[textui.Tunable] doesn't support type params.
-			textui.Tunable[btrfsvol.PhysicalAddr](16*1024), // block size: 16KiB
-			textui.Tunable(1024),                           // number of blocks to buffer; total of 16MiB
-		)
-		devFile := &btrfs.Device{
-			File: bufFile,
-		}
-		if err := fs.AddDevice(ctx, devFile); err != nil {
-			return nil, fmt.Errorf("device file %q: %w", filename, err)
-		}
-	}
-	return fs, nil
-}