From 1624c2ef8c5f95e428ad185278c3cba576f9dcb0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 22 Jul 2023 21:18:40 -0600 Subject: btrfs: Export SuperBlockSize --- lib/btrfs/io1_pv.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/btrfs/io1_pv.go b/lib/btrfs/io1_pv.go index 3b13f73..46eb24d 100644 --- a/lib/btrfs/io1_pv.go +++ b/lib/btrfs/io1_pv.go @@ -30,19 +30,18 @@ var SuperblockAddrs = []btrfsvol.PhysicalAddr{ 0x40_0000_0000, // 256GiB } -var superblockSize = binstruct.StaticSize(btrfstree.Superblock{}) +var SuperblockSize = btrfsvol.PhysicalAddr(binstruct.StaticSize(btrfstree.Superblock{})) func (dev *Device) Superblocks() ([]*diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock], error) { if dev.cacheSuperblocks != nil { return dev.cacheSuperblocks, nil } - superblockSize := btrfsvol.PhysicalAddr(superblockSize) sz := dev.Size() var ret []*diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock] for i, addr := range SuperblockAddrs { - if addr+superblockSize <= sz { + if addr+SuperblockSize <= sz { superblock := &diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Superblock]{ File: dev, Addr: addr, -- cgit v1.2.3 From 17a9ea959d2f77b1ba55d45ac3feaf918f42f7dd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 22 Jul 2023 21:18:40 -0600 Subject: rebuildmappings: Include existing mappings in the scan JSON --- cmd/btrfs-rec/inspect/rebuildmappings/scan.go | 5 +++++ cmd/btrfs-rec/inspect_rebuildmappings.go | 17 +++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/scan.go b/cmd/btrfs-rec/inspect/rebuildmappings/scan.go index 9454ca0..d689136 100644 --- a/cmd/btrfs-rec/inspect/rebuildmappings/scan.go +++ b/cmd/btrfs-rec/inspect/rebuildmappings/scan.go @@ -25,6 +25,11 @@ import ( // Result types //////////////////////////////////////////////////////////////// +type ScanResult struct { + Mappings []btrfsvol.Mapping + Devices ScanDevicesResult +} + type ScanDevicesResult = map[btrfsvol.DeviceID]ScanOneDeviceResult type ScanOneDeviceResult struct { diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index ae2e415..e883b93 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -70,11 +70,16 @@ func init() { RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { ctx := cmd.Context() - scanResults, err := rebuildmappings.ScanDevices(ctx, fs) + devResults, err := rebuildmappings.ScanDevices(ctx, fs) if err != nil { return err } + scanResults := rebuildmappings.ScanResult{ + Mappings: fs.LV.Mappings(), + Devices: devResults, + } + dlog.Info(ctx, "Writing scan results to stdout...") if err := writeJSONFile(os.Stdout, scanResults, lowmemjson.ReEncoderConfig{ Indent: "\t", @@ -97,13 +102,13 @@ func init() { ctx := cmd.Context() dlog.Infof(ctx, "Reading %q...", args[0]) - scanResults, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, args[0]) + scanResults, err := readJSONFile[rebuildmappings.ScanResult](ctx, args[0]) if err != nil { return err } dlog.Infof(ctx, "... done reading %q", args[0]) - if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults); err != nil { + if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults.Devices); err != nil { return err } @@ -132,17 +137,17 @@ func init() { RunE: run(func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - scanResults, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, args[0]) + scanResults, err := readJSONFile[rebuildmappings.ScanResult](ctx, args[0]) if err != nil { return err } var cnt int - for _, devResults := range scanResults { + for _, devResults := range scanResults.Devices { cnt += len(devResults.FoundNodes) } set := make(containers.Set[btrfsvol.LogicalAddr], cnt) - for _, devResults := range scanResults { + for _, devResults := range scanResults.Devices { for laddr := range devResults.FoundNodes { set.Insert(laddr) } -- cgit v1.2.3 From a2e7457d7fc339caac4e1f55d0eeadd732af60b1 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 25 Jul 2023 12:31:30 -0600 Subject: Don't try to read the chunk tree until all devices are added --- lib/btrfs/io2_lv.go | 12 ++++++++---- lib/btrfsutil/open.go | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/btrfs/io2_lv.go b/lib/btrfs/io2_lv.go index e9de215..16c5987 100644 --- a/lib/btrfs/io2_lv.go +++ b/lib/btrfs/io2_lv.go @@ -43,7 +43,7 @@ func (fs *FS) AddDevice(ctx context.Context, dev *Device) error { } fs.cacheSuperblocks = nil fs.cacheSuperblock = nil - if err := fs.initDev(ctx, *sb); err != nil { + if err := fs.initDev(*sb); err != nil { dlog.Errorf(ctx, "error: AddDevice: %q: %v", dev.Name(), err) } return nil @@ -142,14 +142,14 @@ func (fs *FS) ReInit(ctx context.Context) error { if err != nil { return fmt.Errorf("file %q: %w", dev.Name(), err) } - if err := fs.initDev(ctx, *sb); err != nil { + if err := fs.initDev(*sb); err != nil { return fmt.Errorf("file %q: %w", dev.Name(), err) } } - return nil + return fs.InitChunks(ctx) } -func (fs *FS) initDev(ctx context.Context, sb btrfstree.Superblock) error { +func (fs *FS) initDev(sb btrfstree.Superblock) error { syschunks, err := sb.ParseSysChunkArray() if err != nil { return err @@ -161,6 +161,10 @@ func (fs *FS) initDev(ctx context.Context, sb btrfstree.Superblock) error { } } } + return nil +} + +func (fs *FS) InitChunks(ctx context.Context) error { chunkTree, err := fs.ForrestLookup(ctx, btrfsprim.CHUNK_TREE_OBJECTID) if err != nil { return err diff --git a/lib/btrfsutil/open.go b/lib/btrfsutil/open.go index abbd466..91bb943 100644 --- a/lib/btrfsutil/open.go +++ b/lib/btrfsutil/open.go @@ -43,5 +43,8 @@ func Open(ctx context.Context, flag int, filenames ...string) (*btrfs.FS, error) return nil, fmt.Errorf("device file %q: %w", filename, err) } } + if err := fs.InitChunks(ctx); err != nil { + dlog.Errorf(ctx, "error: InitChunks: %v", err) + } return fs, nil } -- cgit v1.2.3 From 4afd0a5b438f0eda8af3be8018bc024daeb738c3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 25 Jul 2023 12:32:13 -0600 Subject: Inline btrfsutil.Open() into main.go --- cmd/btrfs-rec/main.go | 36 +++++++++++++++++++++++++++++++----- lib/btrfsutil/open.go | 50 -------------------------------------------------- 2 files changed, 31 insertions(+), 55 deletions(-) delete mode 100644 lib/btrfsutil/open.go diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go index 39ba9ef..258e35f 100644 --- a/cmd/btrfs-rec/main.go +++ b/cmd/btrfs-rec/main.go @@ -21,6 +21,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil" "git.lukeshu.com/btrfs-progs-ng/lib/containers" + "git.lukeshu.com/btrfs-progs-ng/lib/diskio" "git.lukeshu.com/btrfs-progs-ng/lib/profile" "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) @@ -160,6 +161,8 @@ func run(runE func(*cobra.Command, []string) error) func(*cobra.Command, []strin func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*cobra.Command, []string) error { return run(func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + maybeSetErr := func(_err error) { if _err != nil && err == nil { err = _err @@ -171,16 +174,39 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co // it doesn't interfere with the `help` sub-command. return cliutil.FlagErrorFunc(cmd, fmt.Errorf("must specify 1 or more physical volumes with --pv")) } - fs, err := btrfsutil.Open(cmd.Context(), globalFlags.openFlag, globalFlags.pvs...) - if err != nil { - return err - } + fs := new(btrfs.FS) defer func() { maybeSetErr(fs.Close()) }() + for i, filename := range globalFlags.pvs { + dlog.Debugf(ctx, "Adding device file %d/%d %q...", i, len(globalFlags.pvs), filename) + osFile, err := os.OpenFile(filename, globalFlags.openFlag, 0) + if err != nil { + return fmt.Errorf("device file %q: %w", filename, err) + } + typedFile := &diskio.OSFile[btrfsvol.PhysicalAddr]{ + File: osFile, + } + bufFile := diskio.NewBufferedFile[btrfsvol.PhysicalAddr]( + ctx, + typedFile, + //nolint:gomnd // False positive: gomnd.ignored-functions=[textui.Tunable] doesn't support type params. + textui.Tunable[btrfsvol.PhysicalAddr](16*1024), // block size: 16KiB + textui.Tunable(1024), // number of blocks to buffer; total of 16MiB + ) + devFile := &btrfs.Device{ + File: bufFile, + } + if err := fs.AddDevice(ctx, devFile); err != nil { + return fmt.Errorf("device file %q: %w", filename, err) + } + } + if err := fs.InitChunks(ctx); err != nil { + dlog.Errorf(ctx, "error: InitChunks: %v", err) + } if globalFlags.mappings != "" { - mappingsJSON, err := readJSONFile[[]btrfsvol.Mapping](cmd.Context(), globalFlags.mappings) + mappingsJSON, err := readJSONFile[[]btrfsvol.Mapping](ctx, globalFlags.mappings) if err != nil { return err } diff --git a/lib/btrfsutil/open.go b/lib/btrfsutil/open.go deleted file mode 100644 index 91bb943..0000000 --- a/lib/btrfsutil/open.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2022-2023 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package btrfsutil - -import ( - "context" - "fmt" - "os" - - "github.com/datawire/dlib/dlog" - - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/diskio" - "git.lukeshu.com/btrfs-progs-ng/lib/textui" -) - -func Open(ctx context.Context, flag int, filenames ...string) (*btrfs.FS, error) { - fs := new(btrfs.FS) - for i, filename := range filenames { - dlog.Debugf(ctx, "Adding device file %d/%d %q...", i, len(filenames), filename) - osFile, err := os.OpenFile(filename, flag, 0) - if err != nil { - _ = fs.Close() - return nil, fmt.Errorf("device file %q: %w", filename, err) - } - typedFile := &diskio.OSFile[btrfsvol.PhysicalAddr]{ - File: osFile, - } - bufFile := diskio.NewBufferedFile[btrfsvol.PhysicalAddr]( - ctx, - typedFile, - //nolint:gomnd // False positive: gomnd.ignored-functions=[textui.Tunable] doesn't support type params. - textui.Tunable[btrfsvol.PhysicalAddr](16*1024), // block size: 16KiB - textui.Tunable(1024), // number of blocks to buffer; total of 16MiB - ) - devFile := &btrfs.Device{ - File: bufFile, - } - if err := fs.AddDevice(ctx, devFile); err != nil { - return nil, fmt.Errorf("device file %q: %w", filename, err) - } - } - if err := fs.InitChunks(ctx); err != nil { - dlog.Errorf(ctx, "error: InitChunks: %v", err) - } - return fs, nil -} -- cgit v1.2.3 From b581d0ceb3cd0fee2e78c677bb0805cc68115661 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 22 Jul 2023 21:18:40 -0600 Subject: rebuildmappings process: Don't require --pv --- .../inspect/rebuildmappings/file_phony.go | 49 ++++++++++++++++++++++ cmd/btrfs-rec/inspect_listnodes.go | 2 +- cmd/btrfs-rec/inspect_rebuildmappings.go | 32 ++++++++++++-- cmd/btrfs-rec/main.go | 21 +++++++--- 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go diff --git a/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go b/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go new file mode 100644 index 0000000..a41a16b --- /dev/null +++ b/cmd/btrfs-rec/inspect/rebuildmappings/file_phony.go @@ -0,0 +1,49 @@ +// Copyright (C) 2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package rebuildmappings + +import ( + "fmt" + + "git.lukeshu.com/btrfs-progs-ng/lib/binstruct" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/slices" +) + +type phonyFile struct { + size btrfsvol.PhysicalAddr + sb btrfstree.Superblock +} + +var _ diskio.File[btrfsvol.PhysicalAddr] = (*phonyFile)(nil) + +func NewPhonyFile(size btrfsvol.PhysicalAddr, sb btrfstree.Superblock) *phonyFile { + return &phonyFile{ + size: size, + sb: sb, + } +} + +func (f *phonyFile) Name() string { return fmt.Sprintf("phony_file:device_id=%v", f.sb.DevItem.DevID) } +func (f *phonyFile) Size() btrfsvol.PhysicalAddr { return f.size } +func (*phonyFile) Close() error { return nil } + +func (f *phonyFile) ReadAt(p []byte, off btrfsvol.PhysicalAddr) (int, error) { + if len(p) == int(btrfs.SuperblockSize) && slices.Contains(off, btrfs.SuperblockAddrs) { + bs, err := binstruct.Marshal(f.sb) + if err != nil { + return 0, err + } + return copy(p, bs), nil + } + panic(fmt.Errorf("%T: should not happen: ReadAt should not be called for a phony file", f)) +} + +func (f *phonyFile) WriteAt([]byte, btrfsvol.PhysicalAddr) (int, error) { + panic(fmt.Errorf("%T: should not happen: WriteAt should not be called for a phony file", f)) +} diff --git a/cmd/btrfs-rec/inspect_listnodes.go b/cmd/btrfs-rec/inspect_listnodes.go index d9b24ed..eb088e8 100644 --- a/cmd/btrfs-rec/inspect_listnodes.go +++ b/cmd/btrfs-rec/inspect_listnodes.go @@ -28,7 +28,7 @@ func init() { "to take advantage of the sector-by-sector scan that's already " + "performed by `btrfs-rec inspect rebuild-mappings scan`.", Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() nodeList, err := btrfsutil.ListNodes(ctx, fs) diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index e883b93..dcd4b0a 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -5,6 +5,7 @@ package main import ( + "fmt" "os" "git.lukeshu.com/go/lowmemjson" @@ -37,7 +38,7 @@ func init() { "\tbtrfs-rec inspect rebuild-mappings scan > SCAN.json # read\n" + "\tbtrfs-rec inspect rebuild-mappings process SCAN.json # CPU\n", Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() scanResults, err := rebuildmappings.ScanDevices(ctx, fs) @@ -67,7 +68,7 @@ func init() { Use: "scan", Short: "Read from the filesystem all data nescessary to rebuild the mappings", Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { + RunE: runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { ctx := cmd.Context() devResults, err := rebuildmappings.ScanDevices(ctx, fs) @@ -94,6 +95,7 @@ func init() { }), }) + var scanResults rebuildmappings.ScanResult cmd.AddCommand(&cobra.Command{ Use: "process", Short: "Rebuild the mappings based on previously read data", @@ -102,12 +104,36 @@ func init() { ctx := cmd.Context() dlog.Infof(ctx, "Reading %q...", args[0]) - scanResults, err := readJSONFile[rebuildmappings.ScanResult](ctx, args[0]) + var err error + scanResults, err = readJSONFile[rebuildmappings.ScanResult](ctx, args[0]) if err != nil { return err } dlog.Infof(ctx, "... done reading %q", args[0]) + pvDevices := fs.LV.PhysicalVolumes() + for _, devID := range maps.SortedKeys(scanResults.Devices) { + if maps.HasKey(pvDevices, devID) { + continue + } + devFile := &btrfs.Device{ + File: rebuildmappings.NewPhonyFile( + scanResults.Devices[devID].Size, + scanResults.Devices[devID].Superblock.Val), + } + if err := fs.AddDevice(ctx, devFile); err != nil { + return fmt.Errorf("device file: %q: %w", devFile.Name(), err) + } + } + for _, mapping := range scanResults.Mappings { + if err := fs.LV.AddMapping(mapping); err != nil { + return err + } + } + return nil + }, func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults.Devices); err != nil { return err } diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go index 258e35f..7f89369 100644 --- a/cmd/btrfs-rec/main.go +++ b/cmd/btrfs-rec/main.go @@ -159,7 +159,10 @@ func run(runE func(*cobra.Command, []string) error) func(*cobra.Command, []strin } } -func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*cobra.Command, []string) error { +func runWithRawFS( + overrideInitChunks func(*btrfs.FS, *cobra.Command, []string) error, + runE func(*btrfs.FS, *cobra.Command, []string) error, +) func(*cobra.Command, []string) error { return run(func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -169,7 +172,7 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co } } - if len(globalFlags.pvs) == 0 { + if len(globalFlags.pvs) == 0 && overrideInitChunks == nil { // We do this here instead of calling argparser.MarkPersistentFlagRequired("pv") so that // it doesn't interfere with the `help` sub-command. return cliutil.FlagErrorFunc(cmd, fmt.Errorf("must specify 1 or more physical volumes with --pv")) @@ -201,8 +204,14 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co return fmt.Errorf("device file %q: %w", filename, err) } } - if err := fs.InitChunks(ctx); err != nil { - dlog.Errorf(ctx, "error: InitChunks: %v", err) + if overrideInitChunks != nil { + if err := overrideInitChunks(fs, cmd, args); err != nil { + return err + } + } else { + if err := fs.InitChunks(ctx); err != nil { + dlog.Errorf(ctx, "error: InitChunks: %v", err) + } } if globalFlags.mappings != "" { @@ -222,7 +231,7 @@ func runWithRawFS(runE func(*btrfs.FS, *cobra.Command, []string) error) func(*co } func runWithRawFSAndNodeList(runE func(*btrfs.FS, []btrfsvol.LogicalAddr, *cobra.Command, []string) error) func(*cobra.Command, []string) error { - return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + return runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() var nodeList []btrfsvol.LogicalAddr @@ -271,7 +280,7 @@ func _runWithReadableFS(wantNodeList bool, runE func(btrfs.ReadableFS, []btrfsvo if wantNodeList || globalFlags.rebuild || globalFlags.treeRoots != "" { return runWithRawFSAndNodeList(inner)(cmd, args) } - return runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + return runWithRawFS(nil, func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { return inner(fs, nil, cmd, args) })(cmd, args) } -- cgit v1.2.3