From 5f45fa7d378edae2fac73517384749fc73d24e89 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 28 Feb 2023 20:33:28 -0700 Subject: tree-wide: Don't pass around a full rebuildmappings.ScanDevicesResult when a simple list of nodes will do --- cmd/btrfs-rec/inspect_lstrees.go | 43 +++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'cmd/btrfs-rec/inspect_lstrees.go') diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index df2473c..0c82fe0 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -12,7 +12,6 @@ import ( "github.com/datawire/ocibuild/pkg/cliutil" "github.com/spf13/cobra" - "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildmappings" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" @@ -36,13 +35,9 @@ func init() { }, RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { ctx := cmd.Context() - var scanResults rebuildmappings.ScanDevicesResult - if scandevicesFilename != "" { - var err error - scanResults, err = readJSONFile[rebuildmappings.ScanDevicesResult](ctx, scandevicesFilename) - if err != nil { - return err - } + nodeList, err := readNodeList(ctx, scandevicesFilename) + if err != nil { + return err } var treeErrCnt int @@ -98,23 +93,21 @@ func init() { treeItemCnt = make(map[btrfsitem.Type]int) textui.Fprintf(os.Stdout, "lost+found\n") sb, _ := fs.Superblock() - for _, devResults := range scanResults { - for laddr := range devResults.FoundNodes { - if visitedNodes.Has(laddr) { - continue - } - visitedNodes.Insert(laddr) - node, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ - LAddr: containers.Optional[btrfsvol.LogicalAddr]{OK: true, Val: laddr}, - }) - if err != nil { - treeErrCnt++ - continue - } - for _, item := range node.Data.BodyLeaf { - typ := item.Key.ItemType - treeItemCnt[typ]++ - } + for _, laddr := range nodeList { + if visitedNodes.Has(laddr) { + continue + } + visitedNodes.Insert(laddr) + node, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ + LAddr: containers.Optional[btrfsvol.LogicalAddr]{OK: true, Val: laddr}, + }) + if err != nil { + treeErrCnt++ + continue + } + for _, item := range node.Data.BodyLeaf { + typ := item.Key.ItemType + treeItemCnt[typ]++ } } flush() -- cgit v1.2.3-54-g00ecf From 058f8c36f6bf9ae2600bf7da33d680ced41eae9a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 6 Mar 2023 10:42:51 -0700 Subject: cmd/btrfs-rec: Have each subcommand call runWithRawFS itself --- cmd/btrfs-rec/inspect_dumptrees.go | 14 +++---- cmd/btrfs-rec/inspect_lsfiles.go | 14 +++---- cmd/btrfs-rec/inspect_lstrees.go | 23 +++++------- cmd/btrfs-rec/inspect_mount.go | 19 +++++----- cmd/btrfs-rec/inspect_rebuildmappings.go | 30 +++++++-------- cmd/btrfs-rec/inspect_rebuildtrees.go | 12 +++--- cmd/btrfs-rec/inspect_scandevices.go | 12 +++--- cmd/btrfs-rec/inspect_spewitems.go | 14 +++---- cmd/btrfs-rec/main.go | 64 +++++++++++--------------------- 9 files changed, 83 insertions(+), 119 deletions(-) (limited to 'cmd/btrfs-rec/inspect_lstrees.go') diff --git a/cmd/btrfs-rec/inspect_dumptrees.go b/cmd/btrfs-rec/inspect_dumptrees.go index efdc380..fd152d3 100644 --- a/cmd/btrfs-rec/inspect_dumptrees.go +++ b/cmd/btrfs-rec/inspect_dumptrees.go @@ -16,18 +16,16 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "dump-trees", - Short: "A clone of `btrfs inspect-internal dump-tree`", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { + inspectors.AddCommand(&cobra.Command{ + Use: "dump-trees", + Short: "A clone of `btrfs inspect-internal dump-tree`", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { const version = "6.1.3" out := os.Stdout textui.Fprintf(out, "btrfs-progs v%v\n", version) dumptrees.DumpTrees(cmd.Context(), out, fs) return nil - }, + }), }) } diff --git a/cmd/btrfs-rec/inspect_lsfiles.go b/cmd/btrfs-rec/inspect_lsfiles.go index 4f985ff..a2b46ab 100644 --- a/cmd/btrfs-rec/inspect_lsfiles.go +++ b/cmd/btrfs-rec/inspect_lsfiles.go @@ -26,13 +26,11 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "ls-files", - Short: "A listing of all files in the filesystem", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { + inspectors.AddCommand(&cobra.Command{ + Use: "ls-files", + Short: "A listing of all files in the filesystem", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { out := bufio.NewWriter(os.Stdout) defer func() { if _err := out.Flush(); _err != nil && err == nil { @@ -53,7 +51,7 @@ func init() { }) return nil - }, + }), }) } diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index 0c82fe0..be72860 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -27,13 +27,11 @@ import ( func init() { var scandevicesFilename string - cmd := subcommand{ - Command: cobra.Command{ - Use: "ls-trees", - Short: "A brief view what types of items are in each tree", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { + cmd := &cobra.Command{ + Use: "ls-trees", + Short: "A brief view what types of items are in each tree", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { ctx := cmd.Context() nodeList, err := readNodeList(ctx, scandevicesFilename) if err != nil { @@ -114,11 +112,10 @@ func init() { } return nil - }, + }), } - cmd.Command.Flags().StringVar(&scandevicesFilename, "scandevices", "", "Output of 'scandevices' to use for a lost+found tree") - if err := cmd.Command.MarkFlagFilename("scandevices"); err != nil { - panic(err) - } - inspectors = append(inspectors, cmd) + cmd.Flags().StringVar(&scandevicesFilename, "scandevices", "", "Output of 'scandevices' to use for a lost+found tree") + noError(cmd.MarkFlagFilename("scandevices")) + + inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/inspect_mount.go b/cmd/btrfs-rec/inspect_mount.go index f3fda34..4582f9f 100644 --- a/cmd/btrfs-rec/inspect_mount.go +++ b/cmd/btrfs-rec/inspect_mount.go @@ -14,17 +14,16 @@ import ( func init() { var skipFileSums bool - cmd := subcommand{ - Command: cobra.Command{ - Use: "mount MOUNTPOINT", - Short: "Mount the filesystem read-only", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + cmd := &cobra.Command{ + Use: "mount MOUNTPOINT", + Short: "Mount the filesystem read-only", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { return mount.MountRO(cmd.Context(), fs, args[0], skipFileSums) - }, + }), } - cmd.Command.Flags().BoolVar(&skipFileSums, "skip-filesums", false, + cmd.Flags().BoolVar(&skipFileSums, "skip-filesums", false, "ignore checksum failures on file contents; allow such files to be read") - inspectors = append(inspectors, cmd) + + inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index 005fd5d..81660b0 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -17,21 +17,19 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "rebuild-mappings SCAN_RESULT.json", - Short: "Rebuild broken chunk/dev-extent/blockgroup trees", - Long: "" + - "The rebuilt information is printed as JSON on stdout, and can\n" + - "be loaded by the --mappings flag.\n" + - "\n" + - "This is very similar to `btrfs rescue chunk-recover`, but (1)\n" + - "does a better job, (2) is less buggy, and (3) doesn't actually\n" + - "write the info back to the filesystem; instead writing it\n" + - "out-of-band to stdout.", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + inspectors.AddCommand(&cobra.Command{ + Use: "rebuild-mappings SCAN_RESULT.json", + Short: "Rebuild broken chunk/dev-extent/blockgroup trees", + Long: "" + + "The rebuilt information is printed as JSON on stdout, and can\n" + + "be loaded by the --mappings flag.\n" + + "\n" + + "This is very similar to `btrfs rescue chunk-recover`, but (1)\n" + + "does a better job, (2) is less buggy, and (3) doesn't actually\n" + + "write the info back to the filesystem; instead writing it\n" + + "out-of-band to stdout.", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() dlog.Infof(ctx, "Reading %q...", args[0]) @@ -56,6 +54,6 @@ func init() { dlog.Info(ctx, "... done writing") return nil - }, + }), }) } diff --git a/cmd/btrfs-rec/inspect_rebuildtrees.go b/cmd/btrfs-rec/inspect_rebuildtrees.go index 5d782cf..8c6fc92 100644 --- a/cmd/btrfs-rec/inspect_rebuildtrees.go +++ b/cmd/btrfs-rec/inspect_rebuildtrees.go @@ -21,12 +21,10 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "rebuild-nodes NODESCAN.json", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + inspectors.AddCommand(&cobra.Command{ + Use: "rebuild-nodes NODESCAN.json", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() // This is wrapped in a func in order to *ensure* that `nodeList` goes out of scope once @@ -66,6 +64,6 @@ func init() { dlog.Info(ctx, "... done writing") return rebuildErr - }, + }), }) } diff --git a/cmd/btrfs-rec/inspect_scandevices.go b/cmd/btrfs-rec/inspect_scandevices.go index f93d99d..4172c7c 100644 --- a/cmd/btrfs-rec/inspect_scandevices.go +++ b/cmd/btrfs-rec/inspect_scandevices.go @@ -21,12 +21,10 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "scandevices", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { + inspectors.AddCommand(&cobra.Command{ + Use: "scandevices", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { ctx := cmd.Context() results, err := rebuildmappings.ScanDevices(ctx, fs) @@ -45,7 +43,7 @@ func init() { dlog.Info(ctx, "... done writing") return nil - }, + }), }) } diff --git a/cmd/btrfs-rec/inspect_spewitems.go b/cmd/btrfs-rec/inspect_spewitems.go index 4abb2b0..d8a65ae 100644 --- a/cmd/btrfs-rec/inspect_spewitems.go +++ b/cmd/btrfs-rec/inspect_spewitems.go @@ -19,13 +19,11 @@ import ( ) func init() { - inspectors = append(inspectors, subcommand{ - Command: cobra.Command{ - Use: "spew-items", - Short: "Spew all items as parsed", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - }, - RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { + inspectors.AddCommand(&cobra.Command{ + Use: "spew-items", + Short: "Spew all items as parsed", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { ctx := cmd.Context() spew := spew.NewDefaultConfig() @@ -51,6 +49,6 @@ func init() { }, }) return nil - }, + }), }) } diff --git a/cmd/btrfs-rec/main.go b/cmd/btrfs-rec/main.go index 372dee6..5da8f52 100644 --- a/cmd/btrfs-rec/main.go +++ b/cmd/btrfs-rec/main.go @@ -21,12 +21,27 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) -type subcommand struct { - cobra.Command - RunE func(*btrfs.FS, *cobra.Command, []string) error -} +var ( + inspectors = &cobra.Command{ + Use: "inspect {[flags]|SUBCOMMAND}", + Short: "Inspect (but don't modify) a broken btrfs filesystem", -var inspectors, repairers []subcommand + Args: cliutil.WrapPositionalArgs(cliutil.OnlySubcommands), + RunE: cliutil.RunSubcommands, + } + repairers = &cobra.Command{ + Use: "repair {[flags]|SUBCOMMAND}", + Short: "Repair a broken btrfs filesystem", + + Args: cliutil.WrapPositionalArgs(cliutil.OnlySubcommands), + RunE: cliutil.RunSubcommands, + + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { + globalFlags.openFlag = os.O_RDWR + return nil + }, + } +) var globalFlags struct { logLevel textui.LogLevelFlag @@ -82,43 +97,8 @@ func main() { // Sub-commands - argparserInspect := &cobra.Command{ - Use: "inspect {[flags]|SUBCOMMAND}", - Short: "Inspect (but don't modify) a broken btrfs filesystem", - - Args: cliutil.WrapPositionalArgs(cliutil.OnlySubcommands), - RunE: cliutil.RunSubcommands, - } - argparser.AddCommand(argparserInspect) - - argparserRepair := &cobra.Command{ - Use: "repair {[flags]|SUBCOMMAND}", - Short: "Repair a broken btrfs filesystem", - - Args: cliutil.WrapPositionalArgs(cliutil.OnlySubcommands), - RunE: cliutil.RunSubcommands, - - PersistentPreRunE: func(_ *cobra.Command, _ []string) error { - globalFlags.openFlag = os.O_RDWR - return nil - }, - } - argparser.AddCommand(argparserRepair) - - for _, cmdgrp := range []struct { - parent *cobra.Command - children []subcommand - }{ - {argparserInspect, inspectors}, - {argparserRepair, repairers}, - } { - for _, child := range cmdgrp.children { - cmd := child.Command - runE := child.RunE - cmd.RunE = runWithRawFS(runE) - cmdgrp.parent.AddCommand(&cmd) - } - } + argparser.AddCommand(inspectors) + argparser.AddCommand(repairers) // Run -- cgit v1.2.3-54-g00ecf From 4278c185aafb0fb197e52cdb9eb9bd9d36b61c4d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 7 Mar 2023 01:35:08 -0700 Subject: cmd/btrfs-rec: Smash scandevices and rebuild-mappings together --- cmd/btrfs-rec/inspect_lstrees.go | 2 +- cmd/btrfs-rec/inspect_rebuildmappings.go | 71 ++++++++++++++++++++++++++++++-- cmd/btrfs-rec/inspect_scandevices.go | 69 ------------------------------- cmd/btrfs-rec/util.go | 25 +++++++++++ scripts/main.sh | 8 ++-- 5 files changed, 97 insertions(+), 78 deletions(-) delete mode 100644 cmd/btrfs-rec/inspect_scandevices.go (limited to 'cmd/btrfs-rec/inspect_lstrees.go') diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index be72860..1ff7671 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -114,7 +114,7 @@ func init() { return nil }), } - cmd.Flags().StringVar(&scandevicesFilename, "scandevices", "", "Output of 'scandevices' to use for a lost+found tree") + cmd.Flags().StringVar(&scandevicesFilename, "scandevices", "", "Output of 'btrfs-recs inspect rebuild-mappings scan' to use for a lost+found tree") noError(cmd.MarkFlagFilename("scandevices")) inspectors.AddCommand(cmd) diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index 81660b0..55e3408 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -17,8 +17,8 @@ import ( ) func init() { - inspectors.AddCommand(&cobra.Command{ - Use: "rebuild-mappings SCAN_RESULT.json", + cmd := &cobra.Command{ + Use: "rebuild-mappings", Short: "Rebuild broken chunk/dev-extent/blockgroup trees", Long: "" + "The rebuilt information is printed as JSON on stdout, and can\n" + @@ -27,8 +27,69 @@ func init() { "This is very similar to `btrfs rescue chunk-recover`, but (1)\n" + "does a better job, (2) is less buggy, and (3) doesn't actually\n" + "write the info back to the filesystem; instead writing it\n" + - "out-of-band to stdout.", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + "out-of-band to stdout.\n" + + "\n" + + "The I/O and the CPU parts of this can be split up as:\n" + + "\n" + + "\tbtrfs-rec inspect rebuild-mappings scan > SCAN.json # read\n" + + "\tbtrfs-rec inspect rebuild-mappings process SCAN.json # CPU\n", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + scanResults, err := rebuildmappings.ScanDevices(ctx, fs) + if err != nil { + return err + } + + if err := rebuildmappings.RebuildMappings(ctx, fs, scanResults); err != nil { + return err + } + + dlog.Infof(ctx, "Writing reconstructed mappings to stdout...") + if err := writeJSONFile(os.Stdout, fs.LV.Mappings(), lowmemjson.ReEncoderConfig{ + Indent: "\t", + ForceTrailingNewlines: true, + CompactIfUnder: 120, //nolint:gomnd // This is what looks nice. + }); err != nil { + return err + } + dlog.Info(ctx, "... done writing") + + return nil + }), + } + + cmd.AddCommand(&cobra.Command{ + Use: "scan", + Short: "Read from the filesystem all data nescessary to rebuild the mappings", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { + ctx := cmd.Context() + + scanResults, err := rebuildmappings.ScanDevices(ctx, fs) + if err != nil { + return err + } + + dlog.Info(ctx, "Writing scan results to stdout...") + if err := writeJSONFile(os.Stdout, scanResults, lowmemjson.ReEncoderConfig{ + Indent: "\t", + ForceTrailingNewlines: true, + CompactIfUnder: 16, //nolint:gomnd // This is what looks nice. + }); err != nil { + return err + } + dlog.Info(ctx, "... done writing") + + return nil + }), + }) + + cmd.AddCommand(&cobra.Command{ + Use: "process", + Short: "Rebuild the mappings based on previously read data", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -56,4 +117,6 @@ func init() { return nil }), }) + + inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/inspect_scandevices.go b/cmd/btrfs-rec/inspect_scandevices.go deleted file mode 100644 index 4172c7c..0000000 --- a/cmd/btrfs-rec/inspect_scandevices.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) 2022-2023 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package main - -import ( - "context" - "os" - - "git.lukeshu.com/go/lowmemjson" - "github.com/datawire/dlib/dlog" - "github.com/datawire/ocibuild/pkg/cliutil" - "github.com/spf13/cobra" - - "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildmappings" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/containers" - "git.lukeshu.com/btrfs-progs-ng/lib/maps" -) - -func init() { - inspectors.AddCommand(&cobra.Command{ - Use: "scandevices", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), - RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) { - ctx := cmd.Context() - - results, err := rebuildmappings.ScanDevices(ctx, fs) - if err != nil { - return err - } - - dlog.Info(ctx, "Writing scan results to stdout...") - if err := writeJSONFile(os.Stdout, results, lowmemjson.ReEncoderConfig{ - Indent: "\t", - ForceTrailingNewlines: true, - CompactIfUnder: 16, //nolint:gomnd // This is what looks nice. - }); err != nil { - return err - } - dlog.Info(ctx, "... done writing") - - return nil - }), - }) -} - -func readNodeList(ctx context.Context, filename string) ([]btrfsvol.LogicalAddr, error) { - if filename == "" { - return nil, nil - } - results, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, filename) - if err != nil { - return nil, err - } - var cnt int - for _, devResults := range results { - cnt += len(devResults.FoundNodes) - } - set := make(containers.Set[btrfsvol.LogicalAddr], cnt) - for _, devResults := range results { - for laddr := range devResults.FoundNodes { - set.Insert(laddr) - } - } - return maps.SortedKeys(set), nil -} diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index 3d751a6..2e4bb84 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -13,6 +13,10 @@ import ( "git.lukeshu.com/go/lowmemjson" "github.com/datawire/dlib/dlog" + "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildmappings" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/containers" + "git.lukeshu.com/btrfs-progs-ng/lib/maps" "git.lukeshu.com/btrfs-progs-ng/lib/streamio" ) @@ -47,3 +51,24 @@ func writeJSONFile(w io.Writer, obj any, cfg lowmemjson.ReEncoderConfig) (err er }() return lowmemjson.NewEncoder(lowmemjson.NewReEncoder(buffer, cfg)).Encode(obj) } + +func readNodeList(ctx context.Context, filename string) ([]btrfsvol.LogicalAddr, error) { + if filename == "" { + return nil, nil + } + results, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, filename) + if err != nil { + return nil, err + } + var cnt int + for _, devResults := range results { + cnt += len(devResults.FoundNodes) + } + set := make(containers.Set[btrfsvol.LogicalAddr], cnt) + for _, devResults := range results { + for laddr := range devResults.FoundNodes { + set.Insert(laddr) + } + } + return maps.SortedKeys(set), nil +} diff --git a/scripts/main.sh b/scripts/main.sh index 4aa96ea..67fb330 100755 --- a/scripts/main.sh +++ b/scripts/main.sh @@ -32,9 +32,9 @@ export GOMEMLIMIT="$(awk '/^MemTotal:/{ print $2 "KiB" }' Date: Tue, 7 Mar 2023 12:10:42 -0700 Subject: Expose node-lists as a thing on the CLI --- cmd/btrfs-rec/inspect_listnodes.go | 51 ++++++++++++++++++++++++ cmd/btrfs-rec/inspect_lstrees.go | 23 ++++++++--- cmd/btrfs-rec/inspect_rebuildmappings.go | 44 +++++++++++++++++++++ cmd/btrfs-rec/inspect_rebuildtrees.go | 46 +++++++++++++++------- cmd/btrfs-rec/util.go | 25 ------------ lib/btrfsutil/listnodes.go | 66 ++++++++++++++++++++++++++++++++ scripts/main.sh | 6 ++- 7 files changed, 214 insertions(+), 47 deletions(-) create mode 100644 cmd/btrfs-rec/inspect_listnodes.go create mode 100644 lib/btrfsutil/listnodes.go (limited to 'cmd/btrfs-rec/inspect_lstrees.go') diff --git a/cmd/btrfs-rec/inspect_listnodes.go b/cmd/btrfs-rec/inspect_listnodes.go new file mode 100644 index 0000000..d9b24ed --- /dev/null +++ b/cmd/btrfs-rec/inspect_listnodes.go @@ -0,0 +1,51 @@ +// Copyright (C) 2022-2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package main + +import ( + "os" + + "git.lukeshu.com/go/lowmemjson" + "github.com/datawire/dlib/dlog" + "github.com/datawire/ocibuild/pkg/cliutil" + "github.com/spf13/cobra" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil" +) + +func init() { + inspectors.AddCommand(&cobra.Command{ + Use: "list-nodes", + Short: "Scan the filesystem for btree nodes", + Long: "" + + "This scans the filesystem sector-by-sector looking for nodes. " + + "If you are needing to rebuild the chunk/dev-extent/blockgroup " + + "trees with `btrfs-rec inspect rebuild-mappings` anyway, you may " + + "want to instead use `btrfs-rec inspect rebuild-mappings list-nodes` " + + "to take advantage of the sector-by-sector scan that's already " + + "performed by `btrfs-rec inspect rebuild-mappings scan`.", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + nodeList, err := btrfsutil.ListNodes(ctx, fs) + if err != nil { + return err + } + + dlog.Infof(ctx, "Writing nodes to stdout...") + if err := writeJSONFile(os.Stdout, nodeList, lowmemjson.ReEncoderConfig{ + Indent: "\t", + ForceTrailingNewlines: true, + }); err != nil { + return err + } + dlog.Info(ctx, "... done writing") + + return nil + }), + }) +} diff --git a/cmd/btrfs-rec/inspect_lstrees.go b/cmd/btrfs-rec/inspect_lstrees.go index 1ff7671..05c3a57 100644 --- a/cmd/btrfs-rec/inspect_lstrees.go +++ b/cmd/btrfs-rec/inspect_lstrees.go @@ -26,14 +26,24 @@ import ( ) func init() { - var scandevicesFilename string + var nodeListFilename string cmd := &cobra.Command{ Use: "ls-trees", Short: "A brief view what types of items are in each tree", - Args: cliutil.WrapPositionalArgs(cobra.NoArgs), + Long: "" + + "If no --node-list is given, then a slow sector-by-sector scan " + + "will be used to find all lost+found nodes.", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error { ctx := cmd.Context() - nodeList, err := readNodeList(ctx, scandevicesFilename) + + var nodeList []btrfsvol.LogicalAddr + var err error + if nodeListFilename != "" { + nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename) + } else { + nodeList, err = btrfsutil.ListNodes(ctx, fs) + } if err != nil { return err } @@ -86,7 +96,7 @@ func init() { }, }) - if scandevicesFilename != "" { + { treeErrCnt = 0 treeItemCnt = make(map[btrfsitem.Type]int) textui.Fprintf(os.Stdout, "lost+found\n") @@ -114,8 +124,9 @@ func init() { return nil }), } - cmd.Flags().StringVar(&scandevicesFilename, "scandevices", "", "Output of 'btrfs-recs inspect rebuild-mappings scan' to use for a lost+found tree") - noError(cmd.MarkFlagFilename("scandevices")) + cmd.Flags().StringVar(&nodeListFilename, "node-list", "", + "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for a lost+found tree") + noError(cmd.MarkFlagFilename("node-list")) inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index 9c6b1cc..b215e7a 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -14,6 +14,9 @@ import ( "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildmappings" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/containers" + "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) func init() { @@ -118,5 +121,46 @@ func init() { }), }) + cmd.AddCommand(&cobra.Command{ + Use: "list-nodes", + Short: "Produce a listing of btree nodes from previously read data", + Long: "" + + "This is a variant of `btrfs-rec inspect list-nodes` that takes " + + "advantage of using previously read data from " + + "`btrfs-rec inspect rebuild-nodes scan`.", + Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + scanResults, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, args[0]) + if err != nil { + return err + } + + var cnt int + for _, devResults := range scanResults { + cnt += len(devResults.FoundNodes) + } + set := make(containers.Set[btrfsvol.LogicalAddr], cnt) + for _, devResults := range scanResults { + for laddr := range devResults.FoundNodes { + set.Insert(laddr) + } + } + nodeList := maps.SortedKeys(set) + + dlog.Infof(ctx, "Writing nodes to stdout...") + if err := writeJSONFile(os.Stdout, nodeList, lowmemjson.ReEncoderConfig{ + Indent: "\t", + ForceTrailingNewlines: true, + }); err != nil { + return err + } + dlog.Info(ctx, "... done writing") + + return nil + }, + }) + inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/inspect_rebuildtrees.go b/cmd/btrfs-rec/inspect_rebuildtrees.go index f66f287..676533a 100644 --- a/cmd/btrfs-rec/inspect_rebuildtrees.go +++ b/cmd/btrfs-rec/inspect_rebuildtrees.go @@ -5,7 +5,6 @@ package main import ( - "context" "os" "runtime" "time" @@ -17,25 +16,39 @@ import ( "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildtrees" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil" "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) func init() { - inspectors.AddCommand(&cobra.Command{ - Use: "rebuild-trees NODESCAN.json", - Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(1)), + var nodeListFilename string + cmd := &cobra.Command{ + Use: "rebuild-trees", + Long: "" + + "Rebuild broken btrees based on missing items that are implied " + + "by present items. This requires functioning " + + "chunk/dev-extent/blockgroup trees, which can be rebuilt " + + "separately with `btrfs-rec inspect rebuild-mappings`.\n" + + "\n" + + "If no --node-list is given, then a slow sector-by-sector scan " + + "will be used to find all nodes.", + Args: cliutil.WrapPositionalArgs(cobra.NoArgs), RunE: runWithRawFS(func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() - // This is wrapped in a func in order to *ensure* that `nodeList` goes out of scope once - // `rebuilder` has been created. - rebuilder, err := func(ctx context.Context) (rebuildtrees.Rebuilder, error) { - nodeList, err := readNodeList(ctx, args[0]) - if err != nil { - return nil, err - } - return rebuildtrees.NewRebuilder(ctx, fs, nodeList) - }(ctx) + var nodeList []btrfsvol.LogicalAddr + var err error + if nodeListFilename != "" { + nodeList, err = readJSONFile[[]btrfsvol.LogicalAddr](ctx, nodeListFilename) + } else { + nodeList, err = btrfsutil.ListNodes(ctx, fs) + } + if err != nil { + return err + } + + rebuilder, err := rebuildtrees.NewRebuilder(ctx, fs, nodeList) if err != nil { return err } @@ -65,5 +78,10 @@ func init() { return rebuildErr }), - }) + } + cmd.Flags().StringVar(&nodeListFilename, "node-list", "", + "Output of 'btrfs-recs inspect [rebuild-mappings] list-nodes' to use for the node list") + noError(cmd.MarkFlagFilename("node-list")) + + inspectors.AddCommand(cmd) } diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index 2e4bb84..3d751a6 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -13,10 +13,6 @@ import ( "git.lukeshu.com/go/lowmemjson" "github.com/datawire/dlib/dlog" - "git.lukeshu.com/btrfs-progs-ng/cmd/btrfs-rec/inspect/rebuildmappings" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" - "git.lukeshu.com/btrfs-progs-ng/lib/containers" - "git.lukeshu.com/btrfs-progs-ng/lib/maps" "git.lukeshu.com/btrfs-progs-ng/lib/streamio" ) @@ -51,24 +47,3 @@ func writeJSONFile(w io.Writer, obj any, cfg lowmemjson.ReEncoderConfig) (err er }() return lowmemjson.NewEncoder(lowmemjson.NewReEncoder(buffer, cfg)).Encode(obj) } - -func readNodeList(ctx context.Context, filename string) ([]btrfsvol.LogicalAddr, error) { - if filename == "" { - return nil, nil - } - results, err := readJSONFile[rebuildmappings.ScanDevicesResult](ctx, filename) - if err != nil { - return nil, err - } - var cnt int - for _, devResults := range results { - cnt += len(devResults.FoundNodes) - } - set := make(containers.Set[btrfsvol.LogicalAddr], cnt) - for _, devResults := range results { - for laddr := range devResults.FoundNodes { - set.Insert(laddr) - } - } - return maps.SortedKeys(set), nil -} diff --git a/lib/btrfsutil/listnodes.go b/lib/btrfsutil/listnodes.go new file mode 100644 index 0000000..16300da --- /dev/null +++ b/lib/btrfsutil/listnodes.go @@ -0,0 +1,66 @@ +// Copyright (C) 2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package btrfsutil + +import ( + "context" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/containers" + "git.lukeshu.com/btrfs-progs-ng/lib/diskio" + "git.lukeshu.com/btrfs-progs-ng/lib/maps" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" +) + +type nodeScanner struct { + nodes containers.Set[btrfsvol.LogicalAddr] +} + +type nodeStats struct { + numNodes int +} + +func (s nodeStats) String() string { + return textui.Sprintf("found: %d nodes", s.numNodes) +} + +var _ DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] = (*nodeScanner)(nil) + +func newNodeScanner(ctx context.Context, sb btrfstree.Superblock, numBytes btrfsvol.PhysicalAddr, numSectors int) DeviceScanner[nodeStats, containers.Set[btrfsvol.LogicalAddr]] { + s := new(nodeScanner) + s.nodes = make(containers.Set[btrfsvol.LogicalAddr]) + return s +} + +func (s *nodeScanner) ScanStats() nodeStats { + return nodeStats{numNodes: len(s.nodes)} +} + +func (*nodeScanner) ScanSector(ctx context.Context, dev *btrfs.Device, paddr btrfsvol.PhysicalAddr) error { + return nil +} + +func (s *nodeScanner) ScanNode(ctx context.Context, nodeRef *diskio.Ref[btrfsvol.PhysicalAddr, btrfstree.Node]) error { + s.nodes.Insert(nodeRef.Data.Head.Addr) + return nil +} + +func (s *nodeScanner) ScanDone(ctx context.Context) (containers.Set[btrfsvol.LogicalAddr], error) { + return s.nodes, nil +} + +func ListNodes(ctx context.Context, fs *btrfs.FS) ([]btrfsvol.LogicalAddr, error) { + perDev, err := ScanDevices[nodeStats, containers.Set[btrfsvol.LogicalAddr]](ctx, fs, newNodeScanner) + if err != nil { + return nil, err + } + set := make(containers.Set[btrfsvol.LogicalAddr]) + for _, devSet := range perDev { + set.InsertFrom(devSet) + } + return maps.SortedKeys(set), nil +} diff --git a/scripts/main.sh b/scripts/main.sh index 2ea5213..dddb9bb 100755 --- a/scripts/main.sh +++ b/scripts/main.sh @@ -33,6 +33,8 @@ export GOMEMLIMIT="$(awk '/^MemTotal:/{ print $2 "KiB" }'