From 05456e5ffe328c975529645f8e1588ef89f49da0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 5 Apr 2023 11:18:11 -0600 Subject: btrfsutil: RebuiltForrest: Simplify by taking a btrfs.ReadableFS --- lib/btrfsutil/rebuilt_forrest.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'lib/btrfsutil/rebuilt_forrest.go') diff --git a/lib/btrfsutil/rebuilt_forrest.go b/lib/btrfsutil/rebuilt_forrest.go index 6231563..99213f1 100644 --- a/lib/btrfsutil/rebuilt_forrest.go +++ b/lib/btrfsutil/rebuilt_forrest.go @@ -10,9 +10,9 @@ import ( "github.com/datawire/dlib/dlog" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfstree" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/slices" @@ -134,8 +134,7 @@ func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfs // NewRebuiltForrest(). type RebuiltForrest struct { // static - file btrfstree.NodeSource - sb btrfstree.Superblock + inner btrfs.ReadableFS graph Graph cb RebuiltForrestCallbacks @@ -149,10 +148,9 @@ type RebuiltForrest struct { // NewRebuiltForrest returns a new RebuiltForrest instance. The // RebuiltForrestCallbacks may be nil. -func NewRebuiltForrest(file btrfstree.NodeSource, sb btrfstree.Superblock, graph Graph, cb RebuiltForrestCallbacks) *RebuiltForrest { +func NewRebuiltForrest(fs btrfs.ReadableFS, graph Graph, cb RebuiltForrestCallbacks) *RebuiltForrest { ret := &RebuiltForrest{ - file: file, - sb: sb, + inner: fs, graph: graph, cb: cb, @@ -213,13 +211,17 @@ func (ts *RebuiltForrest) addTree(ctx context.Context, treeID btrfsprim.ObjID, s var root btrfsvol.LogicalAddr switch treeID { case btrfsprim.ROOT_TREE_OBJECTID: - root = ts.sb.RootTree + sb, _ := ts.inner.Superblock() + root = sb.RootTree case btrfsprim.CHUNK_TREE_OBJECTID: - root = ts.sb.ChunkTree + sb, _ := ts.inner.Superblock() + root = sb.ChunkTree case btrfsprim.TREE_LOG_OBJECTID: - root = ts.sb.LogTree + sb, _ := ts.inner.Superblock() + root = sb.LogTree case btrfsprim.BLOCK_GROUP_TREE_OBJECTID: - root = ts.sb.BlockGroupRoot + sb, _ := ts.inner.Superblock() + root = sb.BlockGroupRoot default: if !ts.addTree(ctx, btrfsprim.ROOT_TREE_OBJECTID, stack) { dlog.Error(ctx, "failed to add tree: add ROOT_TREE") -- cgit v1.2.3-54-g00ecf From c88331692ea676acf67eb489a600688cf0c62fc6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 7 Apr 2023 13:43:52 -0600 Subject: btrfsutil: Split rebuilt_callbacks.go off from rebuilt_forrest.go --- lib/btrfsutil/rebuilt_callbacks.go | 85 ++++++++++++++++++++++++++++++++++++++ lib/btrfsutil/rebuilt_forrest.go | 73 -------------------------------- 2 files changed, 85 insertions(+), 73 deletions(-) create mode 100644 lib/btrfsutil/rebuilt_callbacks.go (limited to 'lib/btrfsutil/rebuilt_forrest.go') diff --git a/lib/btrfsutil/rebuilt_callbacks.go b/lib/btrfsutil/rebuilt_callbacks.go new file mode 100644 index 0000000..fef16d4 --- /dev/null +++ b/lib/btrfsutil/rebuilt_callbacks.go @@ -0,0 +1,85 @@ +// Copyright (C) 2022-2023 Luke Shumaker +// +// SPDX-License-Identifier: GPL-2.0-or-later + +package btrfsutil + +import ( + "context" + "fmt" + + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" +) + +type RebuiltForrestCallbacks interface { + AddedItem(ctx context.Context, tree btrfsprim.ObjID, key btrfsprim.Key) + AddedRoot(ctx context.Context, tree btrfsprim.ObjID, root btrfsvol.LogicalAddr) + LookupRoot(ctx context.Context, tree btrfsprim.ObjID) (offset btrfsprim.Generation, item btrfsitem.Root, ok bool) + LookupUUID(ctx context.Context, uuid btrfsprim.UUID) (id btrfsprim.ObjID, ok bool) +} + +type noopRebuiltForrestCallbacks struct { + forrest *RebuiltForrest +} + +func (noopRebuiltForrestCallbacks) AddedItem(context.Context, btrfsprim.ObjID, btrfsprim.Key) {} +func (noopRebuiltForrestCallbacks) AddedRoot(context.Context, btrfsprim.ObjID, btrfsvol.LogicalAddr) { +} + +func (cb noopRebuiltForrestCallbacks) LookupRoot(ctx context.Context, tree btrfsprim.ObjID) (offset btrfsprim.Generation, _item btrfsitem.Root, ok bool) { + rootTree := cb.forrest.RebuiltTree(ctx, btrfsprim.ROOT_TREE_OBJECTID) + if rootTree == nil { + return 0, btrfsitem.Root{}, false + } + tgt := btrfsprim.Key{ + ObjectID: tree, + ItemType: btrfsprim.ROOT_ITEM_KEY, + } + itemKey, itemPtr, ok := rootTree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int { + key.Offset = 0 + return tgt.Compare(key) + }) + rootTree.RebuiltReleaseItems() + if !ok { + return 0, btrfsitem.Root{}, false + } + itemBody := cb.forrest.readItem(ctx, itemPtr) + defer itemBody.Free() + switch itemBody := itemBody.(type) { + case *btrfsitem.Root: + return btrfsprim.Generation(itemKey.Offset), *itemBody, true + case *btrfsitem.Error: + return 0, btrfsitem.Root{}, false + default: + // This is a panic because the item decoder should not emit ROOT_ITEM items as anything but + // btrfsitem.Root or btrfsitem.Error without this code also being updated. + panic(fmt.Errorf("should not happen: ROOT_ITEM item has unexpected type: %T", itemBody)) + } +} + +func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfsprim.UUID) (id btrfsprim.ObjID, ok bool) { + uuidTree := cb.forrest.RebuiltTree(ctx, btrfsprim.UUID_TREE_OBJECTID) + if uuidTree == nil { + return 0, false + } + tgt := btrfsitem.UUIDToKey(uuid) + itemPtr, ok := uuidTree.RebuiltAcquireItems(ctx).Load(tgt) + uuidTree.RebuiltReleaseItems() + if !ok { + return 0, false + } + itemBody := cb.forrest.readItem(ctx, itemPtr) + defer itemBody.Free() + switch itemBody := itemBody.(type) { + case *btrfsitem.UUIDMap: + return itemBody.ObjID, true + case *btrfsitem.Error: + return 0, false + default: + // This is a panic because the item decoder should not emit UUID_SUBVOL items as anything but + // btrfsitem.UUIDMap or btrfsitem.Error without this code also being updated. + panic(fmt.Errorf("should not happen: UUID_SUBVOL item has unexpected type: %T", itemBody)) + } +} diff --git a/lib/btrfsutil/rebuilt_forrest.go b/lib/btrfsutil/rebuilt_forrest.go index 99213f1..29a5210 100644 --- a/lib/btrfsutil/rebuilt_forrest.go +++ b/lib/btrfsutil/rebuilt_forrest.go @@ -6,89 +6,16 @@ package btrfsutil import ( "context" - "fmt" "github.com/datawire/dlib/dlog" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" - "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" "git.lukeshu.com/btrfs-progs-ng/lib/containers" "git.lukeshu.com/btrfs-progs-ng/lib/slices" ) -type RebuiltForrestCallbacks interface { - AddedItem(ctx context.Context, tree btrfsprim.ObjID, key btrfsprim.Key) - AddedRoot(ctx context.Context, tree btrfsprim.ObjID, root btrfsvol.LogicalAddr) - LookupRoot(ctx context.Context, tree btrfsprim.ObjID) (offset btrfsprim.Generation, item btrfsitem.Root, ok bool) - LookupUUID(ctx context.Context, uuid btrfsprim.UUID) (id btrfsprim.ObjID, ok bool) -} - -type noopRebuiltForrestCallbacks struct { - forrest *RebuiltForrest -} - -func (noopRebuiltForrestCallbacks) AddedItem(context.Context, btrfsprim.ObjID, btrfsprim.Key) {} -func (noopRebuiltForrestCallbacks) AddedRoot(context.Context, btrfsprim.ObjID, btrfsvol.LogicalAddr) { -} - -func (cb noopRebuiltForrestCallbacks) LookupRoot(ctx context.Context, tree btrfsprim.ObjID) (offset btrfsprim.Generation, _item btrfsitem.Root, ok bool) { - rootTree := cb.forrest.RebuiltTree(ctx, btrfsprim.ROOT_TREE_OBJECTID) - if rootTree == nil { - return 0, btrfsitem.Root{}, false - } - tgt := btrfsprim.Key{ - ObjectID: tree, - ItemType: btrfsprim.ROOT_ITEM_KEY, - } - itemKey, itemPtr, ok := rootTree.RebuiltAcquireItems(ctx).Search(func(key btrfsprim.Key, _ ItemPtr) int { - key.Offset = 0 - return tgt.Compare(key) - }) - rootTree.RebuiltReleaseItems() - if !ok { - return 0, btrfsitem.Root{}, false - } - itemBody := cb.forrest.readItem(ctx, itemPtr) - defer itemBody.Free() - switch itemBody := itemBody.(type) { - case *btrfsitem.Root: - return btrfsprim.Generation(itemKey.Offset), *itemBody, true - case *btrfsitem.Error: - return 0, btrfsitem.Root{}, false - default: - // This is a panic because the item decoder should not emit ROOT_ITEM items as anything but - // btrfsitem.Root or btrfsitem.Error without this code also being updated. - panic(fmt.Errorf("should not happen: ROOT_ITEM item has unexpected type: %T", itemBody)) - } -} - -func (cb noopRebuiltForrestCallbacks) LookupUUID(ctx context.Context, uuid btrfsprim.UUID) (id btrfsprim.ObjID, ok bool) { - uuidTree := cb.forrest.RebuiltTree(ctx, btrfsprim.UUID_TREE_OBJECTID) - if uuidTree == nil { - return 0, false - } - tgt := btrfsitem.UUIDToKey(uuid) - itemPtr, ok := uuidTree.RebuiltAcquireItems(ctx).Load(tgt) - uuidTree.RebuiltReleaseItems() - if !ok { - return 0, false - } - itemBody := cb.forrest.readItem(ctx, itemPtr) - defer itemBody.Free() - switch itemBody := itemBody.(type) { - case *btrfsitem.UUIDMap: - return itemBody.ObjID, true - case *btrfsitem.Error: - return 0, false - default: - // This is a panic because the item decoder should not emit UUID_SUBVOL items as anything but - // btrfsitem.UUIDMap or btrfsitem.Error without this code also being updated. - panic(fmt.Errorf("should not happen: UUID_SUBVOL item has unexpected type: %T", itemBody)) - } -} - // RebuiltForrest is an abstraction for rebuilding and accessing // potentially broken btrees. // -- cgit v1.2.3-54-g00ecf From 01d7bcfc588c5551e8ac387f62d2fc4468ba1f4c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 7 Apr 2023 13:52:35 -0600 Subject: btrfsutil: RebuiltForrest: Allow skipping the .AddedItem loop --- .../inspect/rebuildtrees/rebuild_treecb.go | 5 ++- lib/btrfsutil/rebuilt_callbacks.go | 7 +++- lib/btrfsutil/rebuilt_forrest.go | 7 +++- lib/btrfsutil/rebuilt_tree.go | 48 +++++++++++++--------- 4 files changed, 42 insertions(+), 25 deletions(-) (limited to 'lib/btrfsutil/rebuilt_forrest.go') diff --git a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_treecb.go b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_treecb.go index 92b5ee5..0c52556 100644 --- a/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_treecb.go +++ b/cmd/btrfs-rec/inspect/rebuildtrees/rebuild_treecb.go @@ -11,13 +11,16 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsitem" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsprim" "git.lukeshu.com/btrfs-progs-ng/lib/btrfs/btrfsvol" + "git.lukeshu.com/btrfs-progs-ng/lib/btrfsutil" ) type forrestCallbacks struct { *rebuilder } -// AddedItem implements btrfsutil.RebuiltForrestCallbacks. +var _ btrfsutil.RebuiltForrestExtendedCallbacks = forrestCallbacks{} + +// AddedItem implements btrfsutil.RebuiltForrestExtendedCallbacks. func (o forrestCallbacks) AddedItem(_ context.Context, tree btrfsprim.ObjID, key btrfsprim.Key) { o.addedItemQueue.Insert(keyAndTree{ TreeID: tree, diff --git a/lib/btrfsutil/rebuilt_callbacks.go b/lib/btrfsutil/rebuilt_callbacks.go index fef16d4..3a7e6e6 100644 --- a/lib/btrfsutil/rebuilt_callbacks.go +++ b/lib/btrfsutil/rebuilt_callbacks.go @@ -14,17 +14,20 @@ import ( ) type RebuiltForrestCallbacks interface { - AddedItem(ctx context.Context, tree btrfsprim.ObjID, key btrfsprim.Key) AddedRoot(ctx context.Context, tree btrfsprim.ObjID, root btrfsvol.LogicalAddr) LookupRoot(ctx context.Context, tree btrfsprim.ObjID) (offset btrfsprim.Generation, item btrfsitem.Root, ok bool) LookupUUID(ctx context.Context, uuid btrfsprim.UUID) (id btrfsprim.ObjID, ok bool) } +type RebuiltForrestExtendedCallbacks interface { + RebuiltForrestCallbacks + AddedItem(ctx context.Context, tree btrfsprim.ObjID, key btrfsprim.Key) +} + type noopRebuiltForrestCallbacks struct { forrest *RebuiltForrest } -func (noopRebuiltForrestCallbacks) AddedItem(context.Context, btrfsprim.ObjID, btrfsprim.Key) {} func (noopRebuiltForrestCallbacks) AddedRoot(context.Context, btrfsprim.ObjID, btrfsvol.LogicalAddr) { } diff --git a/lib/btrfsutil/rebuilt_forrest.go b/lib/btrfsutil/rebuilt_forrest.go index 29a5210..4249396 100644 --- a/lib/btrfsutil/rebuilt_forrest.go +++ b/lib/btrfsutil/rebuilt_forrest.go @@ -73,8 +73,11 @@ type RebuiltForrest struct { rebuiltSharedCache } -// NewRebuiltForrest returns a new RebuiltForrest instance. The -// RebuiltForrestCallbacks may be nil. +// NewRebuiltForrest returns a new RebuiltForrest instance. +// +// The `cb` RebuiltForrestCallbacks may be nil. If `cb` also +// implements RebuiltForrestExtendedCallbacks, then a series of +// .AddedItem() calls will be made before each call to .AddedRoot(). func NewRebuiltForrest(fs btrfs.ReadableFS, graph Graph, cb RebuiltForrestCallbacks) *RebuiltForrest { ret := &RebuiltForrest{ inner: fs, diff --git a/lib/btrfsutil/rebuilt_tree.go b/lib/btrfsutil/rebuilt_tree.go index 31a31be..e4d5ddd 100644 --- a/lib/btrfsutil/rebuilt_tree.go +++ b/lib/btrfsutil/rebuilt_tree.go @@ -320,37 +320,45 @@ func (tree *RebuiltTree) RebuiltAddRoot(ctx context.Context, rootNode btrfsvol.L ctx = dlog.WithField(ctx, "btrfs.util.rebuilt-tree.add-root", fmt.Sprintf("tree=%v rootNode=%v", tree.ID, rootNode)) dlog.Info(ctx, "adding root...") - var stats rebuiltRootStats - leafToRoots := tree.acquireLeafToRoots(ctx) - stats.Leafs.D = len(leafToRoots) - progressWriter := textui.NewProgress[rebuiltRootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) - for i, leaf := range maps.SortedKeys(leafToRoots) { - stats.Leafs.N = i - progressWriter.Set(stats) + shouldFlush := tree.ID == btrfsprim.ROOT_TREE_OBJECTID || tree.ID == btrfsprim.UUID_TREE_OBJECTID + + if extCB, ok := tree.forrest.cb.(RebuiltForrestExtendedCallbacks); ok { + var stats rebuiltRootStats + leafToRoots := tree.acquireLeafToRoots(ctx) + stats.Leafs.D = len(leafToRoots) + progressWriter := textui.NewProgress[rebuiltRootStats](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)) + for i, leaf := range maps.SortedKeys(leafToRoots) { + stats.Leafs.N = i + progressWriter.Set(stats) - if tree.Roots.HasAny(leafToRoots[leaf]) || !leafToRoots[leaf].Has(rootNode) { - continue - } + if tree.Roots.HasAny(leafToRoots[leaf]) || !leafToRoots[leaf].Has(rootNode) { + continue + } + + stats.AddedLeafs++ + progressWriter.Set(stats) - stats.AddedLeafs++ + for _, itemKey := range tree.forrest.graph.Nodes[leaf].Items { + extCB.AddedItem(ctx, tree.ID, itemKey) + stats.AddedItems++ + progressWriter.Set(stats) + } + } + stats.Leafs.N = len(leafToRoots) + tree.releaseLeafToRoots() progressWriter.Set(stats) + progressWriter.Done() - for _, itemKey := range tree.forrest.graph.Nodes[leaf].Items { - tree.forrest.cb.AddedItem(ctx, tree.ID, itemKey) - stats.AddedItems++ - progressWriter.Set(stats) + if stats.AddedItems == 0 { + shouldFlush = false } } - stats.Leafs.N = len(leafToRoots) - tree.releaseLeafToRoots() - progressWriter.Set(stats) - progressWriter.Done() tree.Roots.Insert(rootNode) tree.forrest.incItems.Delete(tree.ID) // force re-gen tree.forrest.excItems.Delete(tree.ID) // force re-gen - if (tree.ID == btrfsprim.ROOT_TREE_OBJECTID || tree.ID == btrfsprim.UUID_TREE_OBJECTID) && stats.AddedItems > 0 { + if shouldFlush { tree.forrest.flushNegativeCache(ctx) } tree.forrest.cb.AddedRoot(ctx, tree.ID, rootNode) -- cgit v1.2.3-54-g00ecf