summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmd/btrfs-rec/inspect_dumpsums.go35
-rw-r--r--cmd/btrfs-rec/inspect_scandevices.go (renamed from cmd/btrfs-rec/inspect_scanfornodes.go)34
-rw-r--r--cmd/btrfs-rec/inspect_scanforextents.go55
-rw-r--r--lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go159
-rw-r--r--lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go21
-rwxr-xr-xscripts/main.sh44
6 files changed, 39 insertions, 309 deletions
diff --git a/cmd/btrfs-rec/inspect_dumpsums.go b/cmd/btrfs-rec/inspect_dumpsums.go
deleted file mode 100644
index 28ae7ef..0000000
--- a/cmd/btrfs-rec/inspect_dumpsums.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright (C) 2022 Luke Shumaker <lukeshu@lukeshu.com>
-//
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package main
-
-import (
- "os"
-
- "github.com/datawire/dlib/dlog"
- "github.com/datawire/ocibuild/pkg/cliutil"
- "github.com/spf13/cobra"
-
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings"
-)
-
-func init() {
- inspectors = append(inspectors, subcommand{
- Command: cobra.Command{
- Use: "dump-sums",
- Short: "Dump a buncha checksums as JSON",
- Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
- },
- RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
- ctx := cmd.Context()
- sums, err := rebuildmappings.SumEverything(ctx, fs)
- if err != nil {
- return err
- }
- dlog.Info(ctx, "Writing sums as gob to stdout...")
- return rebuildmappings.WriteAllSums(os.Stdout, sums)
- },
- })
-}
diff --git a/cmd/btrfs-rec/inspect_scanfornodes.go b/cmd/btrfs-rec/inspect_scandevices.go
index 13f7760..f5caadb 100644
--- a/cmd/btrfs-rec/inspect_scanfornodes.go
+++ b/cmd/btrfs-rec/inspect_scandevices.go
@@ -5,11 +5,12 @@
package main
import (
+ "bufio"
"context"
- "encoding/json"
"os"
"sync"
+ "git.lukeshu.com/go/lowmemjson"
"github.com/datawire/dlib/dgroup"
"github.com/datawire/dlib/dlog"
"github.com/datawire/ocibuild/pkg/cliutil"
@@ -23,23 +24,15 @@ import (
func init() {
inspectors = append(inspectors, subcommand{
Command: cobra.Command{
- Use: "scan-for-nodes",
- Short: "Scan devices for (potentially lost) nodes",
- Long: "" +
- "The found information is printed as JSON on stdout, and can\n" +
- "be read by `btrfs-rec inspect rebuild-mappings`.\n" +
- "\n" +
- "This information is mostly useful for rebuilding a broken\n" +
- "chunk/dev-extent/blockgroup trees, but can also have some\n" +
- "minimal utility in repairing other trees.\n" +
- "\n" +
- "This is very similar the initial scan done by\n" +
- "`btrfs rescue chunk-recover`. Like `btrfs rescue chunk-recover`,\n" +
- "this is likely probably slow because it reads the entirety of\n" +
- "each device.",
+ Use: "scandevices",
Args: cliutil.WrapPositionalArgs(cobra.NoArgs),
},
- RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) error {
+ RunE: func(fs *btrfs.FS, cmd *cobra.Command, _ []string) (err error) {
+ maybeSetErr := func(_err error) {
+ if err == nil && _err != nil {
+ err = _err
+ }
+ }
ctx := cmd.Context()
var resultsMu sync.Mutex
@@ -66,7 +59,14 @@ func init() {
}
dlog.Info(ctx, "Writing scan results to stdout...")
- return json.NewEncoder(os.Stdout).Encode(results)
+ buffer := bufio.NewWriter(os.Stdout)
+ defer func() {
+ maybeSetErr(buffer.Flush())
+ }()
+ return lowmemjson.Encode(&lowmemjson.ReEncoder{
+ Out: buffer,
+ Indent: "\t",
+ }, results)
},
})
}
diff --git a/cmd/btrfs-rec/inspect_scanforextents.go b/cmd/btrfs-rec/inspect_scanforextents.go
deleted file mode 100644
index 9b5ada4..0000000
--- a/cmd/btrfs-rec/inspect_scanforextents.go
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright (C) 2022 Luke Shumaker <lukeshu@lukeshu.com>
-//
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-package main
-
-import (
- "os"
- "runtime"
-
- "github.com/datawire/dlib/dlog"
- "github.com/datawire/ocibuild/pkg/cliutil"
- "github.com/spf13/cobra"
-
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildmappings"
-)
-
-func init() {
- inspectors = append(inspectors, subcommand{
- Command: cobra.Command{
- Use: "scan-for-extents NODESCAN.json DUMPSUMS.gob",
- Args: cliutil.WrapPositionalArgs(cobra.ExactArgs(2)),
- },
- RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error {
- ctx := cmd.Context()
-
- dlog.Infof(ctx, "Reading %q...", args[0])
- bgs, err := rebuildmappings.ReadNodeScanResults(fs, args[0])
- if err != nil {
- return err
- }
- runtime.GC()
- dlog.Infof(ctx, "... done reading %q", args[0])
-
- dlog.Infof(ctx, "Reading %q...", args[1])
- sums, err := rebuildmappings.ReadAllSums(args[1])
- if err != nil {
- return err
- }
- dlog.Infof(ctx, "... done reading %q", args[1])
-
- if err := rebuildmappings.ScanForExtents(ctx, fs, bgs, sums); err != nil {
- return err
- }
-
- dlog.Infof(ctx, "Writing reconstructed mappings to stdout...")
- if err := writeMappingsJSON(os.Stdout, fs); err != nil {
- return err
- }
-
- return nil
- },
- })
-}
diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
index 43d09fe..1a8855b 100644
--- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
+++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/allsums.go
@@ -6,23 +6,10 @@ package rebuildmappings
import (
"context"
- "encoding/gob"
- "io"
"math"
- "os"
- "runtime"
- "strings"
- "sync"
- "github.com/datawire/dlib/dgroup"
- "github.com/datawire/dlib/dlog"
-
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect"
- "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsutil"
)
type AllSums struct {
@@ -67,149 +54,3 @@ func (as AllSums) WalkLogical(ctx context.Context, fn func(btrfsvol.LogicalAddr,
}
return nil
}
-
-// Read/Write AllSums ////////////////////////////////////////////////
-
-func ReadAllSums(filename string) (AllSums, error) {
- fh, err := os.Open(filename)
- if err != nil {
- return AllSums{}, err
- }
- defer fh.Close()
- var val AllSums
- if err := gob.NewDecoder(fh).Decode(&val); err != nil {
- return AllSums{}, err
- }
- return val, nil
-}
-
-func WriteAllSums(w io.Writer, sums AllSums) error {
- return gob.NewEncoder(w).Encode(sums)
-}
-
-func SumEverything(ctx context.Context, fs *btrfs.FS) (AllSums, error) {
- var ret AllSums
-
- // ChecksumSize
- var alg btrfssum.CSumType
- var csumSize int
- if err := func() error {
- sb, err := fs.Superblock()
- if err != nil {
- return err
- }
- alg = sb.ChecksumType
- csumSize = alg.Size()
- return nil
- }(); err != nil {
- return ret, err
- }
-
- // Logical
- dlog.Info(ctx, "Walking CSUM_TREE...")
- func() {
- var curAddr btrfsvol.LogicalAddr
- var curSums strings.Builder
- btrfsutil.NewBrokenTrees(ctx, fs).TreeWalk(ctx, btrfs.CSUM_TREE_OBJECTID,
- func(err *btrfs.TreeError) {
- dlog.Error(ctx, err)
- },
- btrfs.TreeWalkHandler{
- Item: func(path btrfs.TreePath, item btrfs.Item) error {
- if item.Key.ItemType != btrfsitem.EXTENT_CSUM_KEY {
- return nil
- }
- body := item.Body.(btrfsitem.ExtentCSum)
-
- for i, sum := range body.Sums {
- laddr := btrfsvol.LogicalAddr(item.Key.Offset) + (btrfsvol.LogicalAddr(i) * btrfsitem.CSumBlockSize)
- if laddr != curAddr+(btrfsvol.LogicalAddr(curSums.Len()/csumSize)*btrfsitem.CSumBlockSize) {
- if curSums.Len() > 0 {
- ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{
- ChecksumSize: csumSize,
- Addr: curAddr,
- Sums: curSums.String(),
- })
- }
- curAddr = laddr
- curSums.Reset()
- }
- curSums.Write(sum[:csumSize])
- }
- return nil
- },
- },
- )
- if curSums.Len() > 0 {
- ret.Logical = append(ret.Logical, btrfsinspect.SumRun[btrfsvol.LogicalAddr]{
- ChecksumSize: csumSize,
- Addr: curAddr,
- Sums: curSums.String(),
- })
- }
- }()
- if err := ctx.Err(); err != nil {
- return ret, err
- }
- dlog.Info(ctx, "... done walking")
- runtime.GC()
- dlog.Info(ctx, "... GC'd")
-
- // Physical
- dlog.Info(ctx, "Summing devices...")
- if err := func() error {
- devs := fs.LV.PhysicalVolumes()
-
- var mu sync.Mutex
- ret.Physical = make(map[btrfsvol.DeviceID]btrfsinspect.SumRun[btrfsvol.PhysicalAddr], len(devs))
-
- grp := dgroup.NewGroup(ctx, dgroup.GroupConfig{})
- for devID, dev := range devs {
- devID, dev := devID, dev
- grp.Go(dev.Name(), func(ctx context.Context) error {
- devSize := dev.Size()
- numSums := int(devSize / btrfsitem.CSumBlockSize)
- sums := make([]byte, numSums*csumSize)
- lastPct := -1
- progress := func(curSum int) {
- pct := int(100 * float64(curSum) / float64(numSums))
- if pct != lastPct || curSum == numSums {
- dlog.Infof(ctx, "... dev[%q] summed %v%%",
- dev.Name(), pct)
- lastPct = pct
- }
- }
- for i := 0; i < numSums; i++ {
- if err := ctx.Err(); err != nil {
- return err
- }
- progress(i)
- sum, err := btrfsutil.ChecksumPhysical(dev, alg, btrfsvol.PhysicalAddr(i*btrfsitem.CSumBlockSize))
- if err != nil {
- return err
- }
- copy(sums[i*csumSize:], sum[:csumSize])
- }
- progress(numSums)
- sumsStr := string(sums)
- mu.Lock()
- ret.Physical[devID] = btrfsinspect.SumRun[btrfsvol.PhysicalAddr]{
- ChecksumSize: csumSize,
- Addr: 0,
- Sums: sumsStr,
- }
- mu.Unlock()
- return nil
- })
- }
- return grp.Wait()
- }(); err != nil {
- return ret, err
- }
- dlog.Info(ctx, "... done summing devices")
- runtime.GC()
- dlog.Info(ctx, "... GC'd")
-
- // Return
- return ret, nil
-}
diff --git a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
index 2621c44..a98bf3f 100644
--- a/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
+++ b/lib/btrfsprogs/btrfsinspect/rebuildmappings/blockgroups.go
@@ -5,9 +5,7 @@
package rebuildmappings
import (
- "encoding/json"
"fmt"
- "os"
"sort"
"git.lukeshu.com/btrfs-progs-ng/lib/btrfs"
@@ -22,25 +20,6 @@ type BlockGroup struct {
Flags btrfsvol.BlockGroupFlags
}
-func ReadNodeScanResults(fs *btrfs.FS, filename string) (map[btrfsvol.LogicalAddr]BlockGroup, error) {
- scanResultsBytes, err := os.ReadFile(filename)
- if err != nil {
- return nil, err
- }
-
- var scanResults btrfsinspect.ScanDevicesResult
- if err := json.Unmarshal(scanResultsBytes, &scanResults); err != nil {
- return nil, err
- }
-
- bgTree, err := ReduceScanResults(fs, scanResults)
- if err != nil {
- return nil, err
- }
-
- return bgTree, nil
-}
-
func ReduceScanResults(fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (map[btrfsvol.LogicalAddr]BlockGroup, error) {
// Reduce
bgSet := make(map[BlockGroup]struct{})
diff --git a/scripts/main.sh b/scripts/main.sh
index 11ce901..cc03e9f 100755
--- a/scripts/main.sh
+++ b/scripts/main.sh
@@ -17,27 +17,27 @@ go build ./cmd/btrfs-rec
mkdir -p "$b.gen"
{ set +x; } &>/dev/null
-gen $b.gen/0.scan-for-nodes.json \
+gen $b.gen/0.scandevices.json \
./btrfs-rec --pv=$b.img \
- inspect scan-for-nodes
-gen $b.gen/1.mappings.json \
- ./btrfs-rec --pv=$b.img \
- inspect rebuild-mappings $b.gen/0.scan-for-nodes.json
-gen $b.gen/2.csums.gob \
- ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
- inspect dump-sums
-# gen $b.gen/3.dbg.txt \
+ inspect scandevices
+# gen $b.gen/1.mappings.json \
+# ./btrfs-rec --pv=$b.img \
+# inspect rebuild-mappings $b.gen/0.scan-for-nodes.json
+# gen $b.gen/2.csums.gob \
+# ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
+# inspect dump-sums
+# # gen $b.gen/3.dbg.txt \
+# # ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
+# # inspect dbg $b.gen/2.csums.gob
+# gen $b.gen/3.mappings.json \
# ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
-# inspect dbg $b.gen/2.csums.gob
-gen $b.gen/3.mappings.json \
- ./btrfs-rec --pv=$b.img --mappings=$b.gen/1.mappings.json \
- inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob
-gen $b.gen/4.ls-files.txt \
- ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
- inspect ls-files
-gen $b.gen/4.ls-trees.txt \
- ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
- inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json
-gen $b.gen/4.nodes.json \
- ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
- inspect rebuild-nodes $b.gen/0.scan-for-nodes.json
+# inspect scan-for-extents $b.gen/0.scan-for-nodes.json $b.gen/2.csums.gob
+# gen $b.gen/4.ls-files.txt \
+# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+# inspect ls-files
+# gen $b.gen/4.ls-trees.txt \
+# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+# inspect ls-trees --nodescan=$b.gen/0.scan-for-nodes.json
+# gen $b.gen/4.nodes.json \
+# ./btrfs-rec --pv=$b.img --mappings=$b.gen/3.mappings.json \
+# inspect rebuild-nodes $b.gen/0.scan-for-nodes.json
generated by cgit v1.2.3-54-g00ecf (git 2.45.2) at 2024-11-09 02:13:22 +0000