From 77046dcf26687fd74df0f7966d7d469f6b6de717 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 27 Nov 2022 16:41:44 -0700 Subject: rebuildnodes: Refactor the node-graph stuff in to a separate sub-package --- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 135 ++--------------------- 1 file changed, 9 insertions(+), 126 deletions(-) (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index c32ae8e..40ace46 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -6,7 +6,6 @@ package rebuildnodes import ( "context" - "fmt" "github.com/datawire/dlib/dlog" @@ -16,78 +15,14 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) -type nodeData struct { - Level uint8 - Generation btrfsprim.Generation - Owner btrfsprim.ObjID - NumItems uint32 - MinItem btrfsprim.Key - MaxItem btrfsprim.Key -} - -type kpData struct { - FromTree btrfsprim.ObjID - FromNode btrfsvol.LogicalAddr - FromItem int - - ToNode btrfsvol.LogicalAddr - ToLevel uint8 - ToKey btrfsprim.Key - ToGeneration btrfsprim.Generation -} - -func (kp kpData) String() string { - return fmt.Sprintf(`{t:%v,n:%v}[%d]->{n:%v,l:%v,g:%v,k:(%d,%v,%d)}`, - kp.FromTree, kp.FromNode, kp.FromItem, - kp.ToNode, kp.ToLevel, kp.ToGeneration, - kp.ToKey.ObjectID, - kp.ToKey.ItemType, - kp.ToKey.Offset) -} - -type nodeGraph struct { - Nodes map[btrfsvol.LogicalAddr]nodeData - BadNodes map[btrfsvol.LogicalAddr]error - EdgesFrom map[btrfsvol.LogicalAddr][]*kpData - EdgesTo map[btrfsvol.LogicalAddr][]*kpData -} - -func (g nodeGraph) insertEdge(kp kpData) { - ptr := &kp - if kp.ToNode == 0 { - panic("kp.ToNode should not be zero") - } - g.EdgesFrom[kp.FromNode] = append(g.EdgesFrom[kp.FromNode], ptr) - g.EdgesTo[kp.ToNode] = append(g.EdgesTo[kp.ToNode], ptr) -} - -func (g nodeGraph) insertTreeRoot(sb btrfstree.Superblock, treeID btrfsprim.ObjID) { - treeInfo, err := btrfstree.LookupTreeRoot(nil, sb, treeID) - if err != nil { - // This shouldn't ever happen for treeIDs that are - // mentioned directly in the superblock; which are the - // only trees for which we should call - // .insertTreeRoot(). - panic(fmt.Errorf("LookupTreeRoot(%v): %w", treeID, err)) - } - if treeInfo.RootNode == 0 { - return - } - g.insertEdge(kpData{ - FromTree: treeID, - ToNode: treeInfo.RootNode, - ToLevel: treeInfo.Level, - ToGeneration: treeInfo.Generation, - }) -} - type scanResult struct { uuidMap - nodeGraph + nodeGraph *graph.Graph } func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (*scanResult, error) { @@ -101,7 +36,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca lastPct := -1 total := countNodes(scanResults) done := 0 - progress := func() { + progress := func(done, total int) { pct := int(100 * float64(done) / float64(total)) if pct != lastPct || done == total { dlog.Infof(ctx, "... %v%% (%v/%v)", @@ -118,31 +53,17 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca SeenTrees: make(containers.Set[btrfsprim.ObjID]), }, - nodeGraph: nodeGraph{ - Nodes: make(map[btrfsvol.LogicalAddr]nodeData), - BadNodes: make(map[btrfsvol.LogicalAddr]error), - EdgesFrom: make(map[btrfsvol.LogicalAddr][]*kpData), - EdgesTo: make(map[btrfsvol.LogicalAddr][]*kpData), - }, + nodeGraph: graph.New(*sb), } // These 4 trees are mentioned directly in the superblock, so // they are always seen. - // - // 1 ret.SeenTrees.Insert(btrfsprim.ROOT_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.ROOT_TREE_OBJECTID) - // 2 ret.SeenTrees.Insert(btrfsprim.CHUNK_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.CHUNK_TREE_OBJECTID) - // 3 ret.SeenTrees.Insert(btrfsprim.TREE_LOG_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.TREE_LOG_OBJECTID) - // 4 ret.SeenTrees.Insert(btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - ret.insertTreeRoot(*sb, btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - progress() + progress(done, total) for _, devResults := range scanResults { for laddr := range devResults.FoundNodes { nodeRef, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ @@ -168,13 +89,6 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } ret.SeenTrees.Insert(item.Key.ObjectID) - // graph building - ret.insertEdge(kpData{ - FromTree: item.Key.ObjectID, - ToNode: itemBody.ByteNr, - ToLevel: itemBody.Level, - ToGeneration: itemBody.Generation, - }) case btrfsitem.UUIDMap: uuid := btrfsitem.KeyToUUID(item.Key) if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, itemBody.ObjID, uuid); err != nil { @@ -188,28 +102,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca ret.SeenTrees.Insert(nodeRef.Data.Head.Owner) // graph building - ret.Nodes[laddr] = nodeData{ - Level: nodeRef.Data.Head.Level, - Generation: nodeRef.Data.Head.Generation, - Owner: nodeRef.Data.Head.Owner, - NumItems: nodeRef.Data.Head.NumItems, - MinItem: func() btrfsprim.Key { k, _ := nodeRef.Data.MinItem(); return k }(), - MaxItem: func() btrfsprim.Key { k, _ := nodeRef.Data.MaxItem(); return k }(), - } - for i, kp := range nodeRef.Data.BodyInternal { - ret.insertEdge(kpData{ - FromTree: nodeRef.Data.Head.Owner, - FromNode: laddr, - FromItem: i, - ToNode: kp.BlockPtr, - ToLevel: nodeRef.Data.Head.Level - 1, - ToKey: kp.Key, - ToGeneration: kp.Generation, - }) - } + ret.nodeGraph.InsertNode(nodeRef) done++ - progress() + progress(done, total) } } @@ -225,21 +121,8 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca dlog.Info(ctx, "... done reading node data") dlog.Infof(ctx, "Checking keypointers for dead-ends...") - total = len(ret.EdgesTo) - done = 0 - progress() - for laddr := range ret.EdgesTo { - if _, ok := ret.Nodes[laddr]; !ok { - _, err := btrfstree.ReadNode[btrfsvol.LogicalAddr](fs, *sb, laddr, btrfstree.NodeExpectations{ - LAddr: containers.Optional[btrfsvol.LogicalAddr]{OK: true, Val: laddr}, - }) - if err == nil { - return nil, fmt.Errorf("node@%v exists but was not in node scan results", laddr) - } - ret.BadNodes[laddr] = err - } - done++ - progress() + if err := ret.nodeGraph.FinalCheck(fs, *sb, progress); err != nil { + return nil, err } dlog.Info(ctx, "... done checking keypointers") -- cgit v1.2.3-54-g00ecf From 182bab3aa8b8e08d6a35d344c32d93a4293a36f4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 29 Nov 2022 14:42:30 -0700 Subject: rebuildnodes: Refactor uuidMap in to a separate sub-package --- lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 51 ++------------- lib/btrfsprogs/btrfsinspect/rebuildnodes/util.go | 10 --- .../btrfsinspect/rebuildnodes/uuidmap.go | 55 ---------------- .../btrfsinspect/rebuildnodes/uuidmap/scan.go | 73 ++++++++++++++++++++++ .../btrfsinspect/rebuildnodes/uuidmap/uuidmap.go | 55 ++++++++++++++++ 5 files changed, 134 insertions(+), 110 deletions(-) delete mode 100644 lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap.go create mode 100644 lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/scan.go create mode 100644 lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/uuidmap.go (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index 40ace46..9174035 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -10,18 +10,17 @@ import ( "github.com/datawire/dlib/dlog" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/graph" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" ) type scanResult struct { - uuidMap + uuidMap *uuidmap.UUIDMap nodeGraph *graph.Graph } @@ -46,23 +45,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca } ret := &scanResult{ - uuidMap: uuidMap{ - ObjID2UUID: make(map[btrfsprim.ObjID]btrfsprim.UUID), - UUID2ObjID: make(map[btrfsprim.UUID]btrfsprim.ObjID), - TreeParent: make(map[btrfsprim.ObjID]btrfsprim.UUID), - - SeenTrees: make(containers.Set[btrfsprim.ObjID]), - }, + uuidMap: uuidmap.New(), nodeGraph: graph.New(*sb), } - // These 4 trees are mentioned directly in the superblock, so - // they are always seen. - ret.SeenTrees.Insert(btrfsprim.ROOT_TREE_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.CHUNK_TREE_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.TREE_LOG_OBJECTID) - ret.SeenTrees.Insert(btrfsprim.BLOCK_GROUP_TREE_OBJECTID) - progress(done, total) for _, devResults := range scanResults { for laddr := range devResults.FoundNodes { @@ -73,35 +59,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } - // UUID map rebuilding - for _, item := range nodeRef.Data.BodyLeaf { - switch itemBody := item.Body.(type) { - case btrfsitem.Root: - if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, item.Key.ObjectID, itemBody.UUID); err != nil { - return nil, err - } - if itemBody.UUID != (btrfsprim.UUID{}) { - if err := maybeSet("UUID2ObjID", ret.UUID2ObjID, itemBody.UUID, item.Key.ObjectID); err != nil { - return nil, err - } - } - if err := maybeSet("ParentUUID", ret.TreeParent, item.Key.ObjectID, itemBody.ParentUUID); err != nil { - return nil, err - } - ret.SeenTrees.Insert(item.Key.ObjectID) - case btrfsitem.UUIDMap: - uuid := btrfsitem.KeyToUUID(item.Key) - if err := maybeSet("ObjID2UUID", ret.ObjID2UUID, itemBody.ObjID, uuid); err != nil { - return nil, err - } - if err := maybeSet("UUID2ObjID", ret.UUID2ObjID, uuid, itemBody.ObjID); err != nil { - return nil, err - } - } + if err := ret.uuidMap.InsertNode(nodeRef); err != nil { + return nil, err } - ret.SeenTrees.Insert(nodeRef.Data.Head.Owner) - // graph building ret.nodeGraph.InsertNode(nodeRef) done++ @@ -113,7 +74,7 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca panic("should not happen") } - missing := ret.missingRootItems() + missing := ret.uuidMap.MissingRootItems() if len(missing) > 0 { dlog.Errorf(ctx, "... could not find root items for %d trees: %v", len(missing), maps.SortedKeys(missing)) } diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/util.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/util.go index bdaa0c4..8c43dad 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/util.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/util.go @@ -5,21 +5,11 @@ package rebuildnodes import ( - "fmt" - "golang.org/x/exp/constraints" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" ) -func maybeSet[K, V comparable](name string, m map[K]V, k K, v V) error { - if other, conflict := m[k]; conflict && other != v { - return fmt.Errorf("conflict: %s %v can't have both %v and %v", name, k, other, v) - } - m[k] = v - return nil -} - func countNodes(nodeScanResults btrfsinspect.ScanDevicesResult) int { var cnt int for _, devResults := range nodeScanResults { diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap.go deleted file mode 100644 index 4d93916..0000000 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2022 Luke Shumaker -// -// SPDX-License-Identifier: GPL-2.0-or-later - -package rebuildnodes - -import ( - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" - "git.lukeshu.com/btrfs-progs-ng/lib/containers" -) - -type uuidMap struct { - ObjID2UUID map[btrfsprim.ObjID]btrfsprim.UUID - UUID2ObjID map[btrfsprim.UUID]btrfsprim.ObjID - TreeParent map[btrfsprim.ObjID]btrfsprim.UUID - - SeenTrees containers.Set[btrfsprim.ObjID] -} - -func (m uuidMap) missingRootItems() containers.Set[btrfsprim.ObjID] { - missing := make(containers.Set[btrfsprim.ObjID]) - for treeID := range m.SeenTrees { - if _, ok := m.ObjID2UUID[treeID]; !ok && treeID != btrfsprim.ROOT_TREE_OBJECTID { - missing.Insert(treeID) - continue - } - if _, ok := m.TreeParent[treeID]; !ok && treeID >= btrfsprim.FIRST_FREE_OBJECTID && treeID <= btrfsprim.LAST_FREE_OBJECTID { - missing.Insert(treeID) - } - } - return missing -} - -// ParentTree implements btrfstree.NodeFile. -func (m uuidMap) ParentTree(tree btrfsprim.ObjID) (btrfsprim.ObjID, bool) { - if tree < btrfsprim.FIRST_FREE_OBJECTID || tree > btrfsprim.LAST_FREE_OBJECTID { - // no parent - return 0, true - } - parentUUID, ok := m.TreeParent[tree] - if !ok { - // could not look up parent info - return 0, false - } - if parentUUID == (btrfsprim.UUID{}) { - // no parent - return 0, true - } - parentObjID, ok := m.UUID2ObjID[parentUUID] - if !ok { - // could not look up parent info - return 0, false - } - return parentObjID, true -} diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/scan.go new file mode 100644 index 0000000..98ed097 --- /dev/null +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/scan.go @@ -0,0 +1,73 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package uuidmap + +import ( + "fmt" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/containers" + "git.lukeshu.com/btrfs-progs-ng/lib/diskio" +) + +func maybeSet[K, V comparable](name string, m map[K]V, k K, v V) error { + if other, conflict := m[k]; conflict && other != v { + return fmt.Errorf("conflict: %s %v can't have both %v and %v", name, k, other, v) + } + m[k] = v + return nil +} + +func New() *UUIDMap { + ret := &UUIDMap{ + ObjID2UUID: make(map[btrfsprim.ObjID]btrfsprim.UUID), + UUID2ObjID: make(map[btrfsprim.UUID]btrfsprim.ObjID), + TreeParent: make(map[btrfsprim.ObjID]btrfsprim.UUID), + + SeenTrees: make(containers.Set[btrfsprim.ObjID]), + } + + // These 4 trees are mentioned directly in the superblock, so + // they are always seen. + ret.SeenTrees.Insert(btrfsprim.ROOT_TREE_OBJECTID) + ret.SeenTrees.Insert(btrfsprim.CHUNK_TREE_OBJECTID) + ret.SeenTrees.Insert(btrfsprim.TREE_LOG_OBJECTID) + ret.SeenTrees.Insert(btrfsprim.BLOCK_GROUP_TREE_OBJECTID) + + return ret +} + +func (m *UUIDMap) InsertNode(nodeRef *diskio.Ref[btrfsvol.LogicalAddr, btrfstree.Node]) error { + for _, item := range nodeRef.Data.BodyLeaf { + switch itemBody := item.Body.(type) { + case btrfsitem.Root: + if err := maybeSet("ObjID2UUID", m.ObjID2UUID, item.Key.ObjectID, itemBody.UUID); err != nil { + return err + } + if itemBody.UUID != (btrfsprim.UUID{}) { + if err := maybeSet("UUID2ObjID", m.UUID2ObjID, itemBody.UUID, item.Key.ObjectID); err != nil { + return err + } + } + if err := maybeSet("ParentUUID", m.TreeParent, item.Key.ObjectID, itemBody.ParentUUID); err != nil { + return err + } + m.SeenTrees.Insert(item.Key.ObjectID) + case btrfsitem.UUIDMap: + uuid := btrfsitem.KeyToUUID(item.Key) + if err := maybeSet("ObjID2UUID", m.ObjID2UUID, itemBody.ObjID, uuid); err != nil { + return err + } + if err := maybeSet("UUID2ObjID", m.UUID2ObjID, uuid, itemBody.ObjID); err != nil { + return err + } + } + } + m.SeenTrees.Insert(nodeRef.Data.Head.Owner) + return nil +} diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/uuidmap.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/uuidmap.go new file mode 100644 index 0000000..32a34d1 --- /dev/null +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap/uuidmap.go @@ -0,0 +1,55 @@ +// Copyright (C) 2022 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package uuidmap + +import ( + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" + "git.lukeshu.com/btrfs-progs-ng/lib/containers" +) + +type UUIDMap struct { + ObjID2UUID map[btrfsprim.ObjID]btrfsprim.UUID + UUID2ObjID map[btrfsprim.UUID]btrfsprim.ObjID + TreeParent map[btrfsprim.ObjID]btrfsprim.UUID + + SeenTrees containers.Set[btrfsprim.ObjID] +} + +func (m UUIDMap) MissingRootItems() containers.Set[btrfsprim.ObjID] { + missing := make(containers.Set[btrfsprim.ObjID]) + for treeID := range m.SeenTrees { + if _, ok := m.ObjID2UUID[treeID]; !ok && treeID != btrfsprim.ROOT_TREE_OBJECTID { + missing.Insert(treeID) + continue + } + if _, ok := m.TreeParent[treeID]; !ok && treeID >= btrfsprim.FIRST_FREE_OBJECTID && treeID <= btrfsprim.LAST_FREE_OBJECTID { + missing.Insert(treeID) + } + } + return missing +} + +// ParentTree implements btrfstree.NodeFile. +func (m UUIDMap) ParentTree(tree btrfsprim.ObjID) (btrfsprim.ObjID, bool) { + if tree < btrfsprim.FIRST_FREE_OBJECTID || tree > btrfsprim.LAST_FREE_OBJECTID { + // no parent + return 0, true + } + parentUUID, ok := m.TreeParent[tree] + if !ok { + // could not look up parent info + return 0, false + } + if parentUUID == (btrfsprim.UUID{}) { + // no parent + return 0, true + } + parentObjID, ok := m.UUID2ObjID[parentUUID] + if !ok { + // could not look up parent info + return 0, false + } + return parentObjID, true +} -- cgit v1.2.3-54-g00ecf From 94a86a763157327ac969c98e19d7770db477a6a3 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 14 Dec 2022 16:19:07 -0700 Subject: Migrate existing progress logs to textui.NewProgress --- .../btrfsinspect/rebuildnodes/orphans.go | 60 ++++++++++++++++------ lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go | 26 +++++++--- lib/btrfsprogs/btrfsinspect/scandevices.go | 51 +++++++++++++----- 3 files changed, 100 insertions(+), 37 deletions(-) (limited to 'lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go') diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/orphans.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/orphans.go index 55cf95b..51313a9 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/orphans.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/orphans.go @@ -6,6 +6,8 @@ package rebuildnodes import ( "context" + "fmt" + "time" "github.com/datawire/dlib/dlog" @@ -16,6 +18,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/diskio" "git.lukeshu.com/btrfs-progs-ng/lib/maps" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) func listRoots(graph graph.Graph, leaf btrfsvol.LogicalAddr) containers.Set[btrfsvol.LogicalAddr] { @@ -54,6 +57,31 @@ func (a keyAndTree) Cmp(b keyAndTree) int { return containers.NativeCmp(a.TreeID, b.TreeID) } +type crawlStats struct { + DoneOrphans int + TotalOrphans int + + VisitedNodes int + FoundLeafs int +} + +func (s crawlStats) String() string { + return fmt.Sprintf("... crawling orphans %v%% (%v/%v); visited %d nodes, found %d leaf nodes", + int(100*float64(s.DoneOrphans)/float64(s.TotalOrphans)), + s.DoneOrphans, s.TotalOrphans, s.VisitedNodes, s.FoundLeafs) +} + +type readStats struct { + DoneLeafNodes int + TotalLeafNodes int +} + +func (s readStats) String() string { + return fmt.Sprintf("... reading leafs %v%% (%v/%v)", + int(100*float64(s.DoneLeafNodes)/float64(s.TotalLeafNodes)), + s.DoneLeafNodes, s.TotalLeafNodes) +} + func indexOrphans(ctx context.Context, fs diskio.File[btrfsvol.LogicalAddr], sb btrfstree.Superblock, graph graph.Graph) ( orphans containers.Set[btrfsvol.LogicalAddr], leaf2orphans map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr], @@ -71,18 +99,15 @@ func indexOrphans(ctx context.Context, fs diskio.File[btrfsvol.LogicalAddr], sb leaf2orphans = make(map[btrfsvol.LogicalAddr]containers.Set[btrfsvol.LogicalAddr]) visited := make(containers.Set[btrfsvol.LogicalAddr]) - lastPct, lastVisited, lastLeafs := -1, 0, 0 - total := len(orphans) done := 0 + crawlProgressWriter := textui.NewProgress[crawlStats](ctx, dlog.LogLevelInfo, 1*time.Second) progress := func() { - pct := int(100 * float64(done) / float64(total)) - if pct != lastPct || (len(visited) != lastVisited && len(visited)%500 == 0) || len(leaf2orphans) != lastLeafs || done == total { - dlog.Infof(ctx, "... crawling orphans %v%% (%v/%v); visited %d nodes, found %d leaf nodes", - pct, done, total, len(visited), len(leaf2orphans)) - lastPct = pct - lastVisited = len(visited) - lastLeafs = len(leaf2orphans) - } + crawlProgressWriter.Set(crawlStats{ + DoneOrphans: done, + TotalOrphans: len(orphans), + VisitedNodes: len(visited), + FoundLeafs: len(leaf2orphans), + }) } progress() for _, orphan := range maps.SortedKeys(orphans) { @@ -102,17 +127,16 @@ func indexOrphans(ctx context.Context, fs diskio.File[btrfsvol.LogicalAddr], sb done++ progress() } + crawlProgressWriter.Done() key2leaf = new(containers.SortedMap[keyAndTree, btrfsvol.LogicalAddr]) - total = len(leaf2orphans) done = 0 + readProgressWriter := textui.NewProgress[readStats](ctx, dlog.LogLevelInfo, 1*time.Second) progress = func() { - pct := int(100 * float64(done) / float64(total)) - if pct != lastPct || done == total { - dlog.Infof(ctx, "... reading leafs %v%% (%v/%v)", - pct, done, total) - lastPct = pct - } + readProgressWriter.Set(readStats{ + DoneLeafNodes: done, + TotalLeafNodes: len(leaf2orphans), + }) } progress() for _, laddr := range maps.SortedKeys(leaf2orphans) { @@ -136,5 +160,7 @@ func indexOrphans(ctx context.Context, fs diskio.File[btrfsvol.LogicalAddr], sb done++ progress() } + readProgressWriter.Done() + return orphans, leaf2orphans, key2leaf, nil } diff --git a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go index 9174035..3575534 100644 --- a/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go +++ b/lib/btrfsprogs/btrfsinspect/rebuildnodes/scan.go @@ -6,6 +6,8 @@ package rebuildnodes import ( "context" + "fmt" + "time" "github.com/datawire/dlib/dlog" @@ -17,6 +19,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes/uuidmap" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/maps" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type scanResult struct { @@ -24,6 +27,16 @@ type scanResult struct { nodeGraph *graph.Graph } +type scanStats struct { + N, D int +} + +func (s scanStats) String() string { + return fmt.Sprintf("... %v%% (%v/%v)", + int(100*float64(s.N)/float64(s.D)), + s.N, s.D) +} + func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.ScanDevicesResult) (*scanResult, error) { dlog.Infof(ctx, "Reading node data from FS...") @@ -32,16 +45,11 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca return nil, err } - lastPct := -1 total := countNodes(scanResults) done := 0 + progressWriter := textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) progress := func(done, total int) { - pct := int(100 * float64(done) / float64(total)) - if pct != lastPct || done == total { - dlog.Infof(ctx, "... %v%% (%v/%v)", - pct, done, total) - lastPct = pct - } + progressWriter.Set(scanStats{N: done, D: total}) } ret := &scanResult{ @@ -69,10 +77,10 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca progress(done, total) } } - if done != total { panic("should not happen") } + progressWriter.Done() missing := ret.uuidMap.MissingRootItems() if len(missing) > 0 { @@ -81,10 +89,12 @@ func ScanDevices(ctx context.Context, fs *btrfs.FS, scanResults btrfsinspect.Sca dlog.Info(ctx, "... done reading node data") + progressWriter = textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) dlog.Infof(ctx, "Checking keypointers for dead-ends...") if err := ret.nodeGraph.FinalCheck(fs, *sb, progress); err != nil { return nil, err } + progressWriter.Done() dlog.Info(ctx, "... done checking keypointers") return ret, nil diff --git a/lib/btrfsprogs/btrfsinspect/scandevices.go b/lib/btrfsprogs/btrfsinspect/scandevices.go index a382e17..4834826 100644 --- a/lib/btrfsprogs/btrfsinspect/scandevices.go +++ b/lib/btrfsprogs/btrfsinspect/scandevices.go @@ -10,6 +10,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/datawire/dlib/dgroup" "github.com/datawire/dlib/dlog" @@ -21,6 +22,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfssum" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) type ScanDevicesResult map[btrfsvol.DeviceID]ScanOneDeviceResult @@ -78,6 +80,31 @@ type SysExtentCSum struct { Sums btrfsitem.ExtentCSum } +type scanStats struct { + DevName string + DevSize btrfsvol.PhysicalAddr + + Pos btrfsvol.PhysicalAddr + + NumFoundNodes int + NumFoundChunks int + NumFoundBlockGroups int + NumFoundDevExtents int + NumFoundExtentCSums int +} + +func (s scanStats) String() string { + return fmt.Sprintf("... dev[%q] scanned %v%% (%d/%d) (found: %v nodes, %v chunks, %v block groups, %v dev extents, %v sum items)", + s.DevName, + int(100*float64(s.Pos)/float64(s.DevSize)), + s.Pos, s.DevSize, + s.NumFoundNodes, + s.NumFoundChunks, + s.NumFoundBlockGroups, + s.NumFoundDevExtents, + s.NumFoundExtentCSums) +} + // ScanOneDevice mostly mimics btrfs-progs // cmds/rescue-chunk-recover.c:scan_one_device(). func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfstree.Superblock) (ScanOneDeviceResult, error) { @@ -103,19 +130,18 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfstree.Superblo var sums strings.Builder sums.Grow(numSums * csumSize) - lastProgress := -1 + progressWriter := textui.NewProgress[scanStats](ctx, dlog.LogLevelInfo, 1*time.Second) progress := func(pos btrfsvol.PhysicalAddr) { - pct := int(100 * float64(pos) / float64(devSize)) - if pct != lastProgress || pos == devSize { - dlog.Infof(ctx, "... dev[%q] scanned %v%% (found: %v nodes, %v chunks, %v block groups, %v dev extents, %v sum items)", - dev.Name(), pct, - len(result.FoundNodes), - len(result.FoundChunks), - len(result.FoundBlockGroups), - len(result.FoundDevExtents), - len(result.FoundExtentCSums)) - lastProgress = pct - } + progressWriter.Set(scanStats{ + DevName: dev.Name(), + DevSize: devSize, + Pos: pos, + NumFoundNodes: len(result.FoundNodes), + NumFoundChunks: len(result.FoundChunks), + NumFoundBlockGroups: len(result.FoundBlockGroups), + NumFoundDevExtents: len(result.FoundDevExtents), + NumFoundExtentCSums: len(result.FoundExtentCSums), + }) } var minNextNode btrfsvol.PhysicalAddr @@ -212,6 +238,7 @@ func ScanOneDevice(ctx context.Context, dev *btrfs.Device, sb btrfstree.Superblo } } progress(devSize) + progressWriter.Done() result.Checksums = btrfssum.SumRun[btrfsvol.PhysicalAddr]{ ChecksumSize: csumSize, -- cgit v1.2.3-54-g00ecf