From 8efc82d0b1a167830970135c78d173667080b116 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 28 Dec 2022 18:49:09 -0700 Subject: rebuildnodes: Support graceful shutdown --- cmd/btrfs-rec/inspect_rebuildnodes.go | 18 ++++++++++++++---- cmd/btrfs-rec/util.go | 17 +++++++++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) (limited to 'cmd') diff --git a/cmd/btrfs-rec/inspect_rebuildnodes.go b/cmd/btrfs-rec/inspect_rebuildnodes.go index e61e6d2..9c86c3a 100644 --- a/cmd/btrfs-rec/inspect_rebuildnodes.go +++ b/cmd/btrfs-rec/inspect_rebuildnodes.go @@ -33,21 +33,31 @@ func init() { } dlog.Infof(ctx, "... done reading %q", args[0]) - rebuiltNodes, err := rebuildnodes.RebuildNodes(ctx, fs, nodeScanResults) + rebuilder, err := rebuildnodes.NewRebuilder(ctx, fs, nodeScanResults) if err != nil { return err } - dlog.Info(ctx, "Writing re-built nodes to stdout...") - if err := writeJSONFile(os.Stdout, rebuiltNodes, lowmemjson.ReEncoder{ + dlog.Info(ctx, "Rebuilding node tree...") + rebuildErr := rebuilder.Rebuild(ctx) + dst := os.Stdout + if rebuildErr != nil { + dst = os.Stderr + dlog.Errorf(ctx, "rebuild error: %v", rebuildErr) + } + dlog.Infof(ctx, "Writing re-built nodes to %s...", dst.Name()) + if err := writeJSONFile(dst, rebuilder.ListRoots(), lowmemjson.ReEncoder{ Indent: "\t", ForceTrailingNewlines: true, }); err != nil { + if rebuildErr != nil { + return rebuildErr + } return err } dlog.Info(ctx, "... done writing") - return nil + return rebuildErr }, }) } diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index ffc03cc..9a0d60c 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -18,6 +18,7 @@ import ( ) type runeScanner struct { + ctx context.Context //nolint:containedctx // For detecting shutdown from methods progress textui.Portion[int64] progressWriter *textui.Progress[textui.Portion[int64]] unreadCnt uint64 @@ -31,6 +32,7 @@ func newRuneScanner(ctx context.Context, fh *os.File) (*runeScanner, error) { return nil, err } ret := &runeScanner{ + ctx: ctx, progress: textui.Portion[int64]{ D: fi.Size(), }, @@ -42,6 +44,9 @@ func newRuneScanner(ctx context.Context, fh *os.File) (*runeScanner, error) { } func (rs *runeScanner) ReadRune() (r rune, size int, err error) { + if err := rs.ctx.Err(); err != nil { + return 0, 0, err + } r, size, err = rs.reader.ReadRune() if rs.unreadCnt > 0 { rs.unreadCnt-- @@ -53,8 +58,14 @@ func (rs *runeScanner) ReadRune() (r rune, size int, err error) { } func (rs *runeScanner) UnreadRune() error { + if err := rs.ctx.Err(); err != nil { + return err + } + if err := rs.reader.UnreadRune(); err != nil { + return err + } rs.unreadCnt++ - return rs.reader.UnreadRune() + return nil } func (rs *runeScanner) Close() error { @@ -69,6 +80,9 @@ func readJSONFile[T any](ctx context.Context, filename string) (T, error) { return zero, err } buf, err := newRuneScanner(dlog.WithField(ctx, "btrfs.read-json-file", filename), fh) + defer func() { + _ = buf.Close() + }() if err != nil { var zero T return zero, err @@ -78,7 +92,6 @@ func readJSONFile[T any](ctx context.Context, filename string) (T, error) { var zero T return zero, err } - _ = buf.Close() return ret, nil } -- cgit v1.2.3-54-g00ecf From 6ce1332d3cac5b74d2f049861f04cc2fa282d747 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 29 Dec 2022 23:53:55 -0700 Subject: cmd/btrfs-rec inspect rebuild-nodes: Optimize memory use --- cmd/btrfs-rec/inspect_rebuildnodes.go | 25 ++++++++++++++++++------- lib/textui/log_memstats.go | 4 ++-- 2 files changed, 20 insertions(+), 9 deletions(-) (limited to 'cmd') diff --git a/cmd/btrfs-rec/inspect_rebuildnodes.go b/cmd/btrfs-rec/inspect_rebuildnodes.go index 9c86c3a..d813f36 100644 --- a/cmd/btrfs-rec/inspect_rebuildnodes.go +++ b/cmd/btrfs-rec/inspect_rebuildnodes.go @@ -5,7 +5,10 @@ package main import ( + "context" "os" + "runtime" + "time" "git.lukeshu.com/go/lowmemjson" "github.com/datawire/dlib/dlog" @@ -15,6 +18,7 @@ import ( "git.lukeshu.com/btrfs-progs-ng/lib/btrfs" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect" "git.lukeshu.com/btrfs-progs-ng/lib/btrfsprogs/btrfsinspect/rebuildnodes" + "git.lukeshu.com/btrfs-progs-ng/lib/textui" ) func init() { @@ -26,18 +30,25 @@ func init() { RunE: func(fs *btrfs.FS, cmd *cobra.Command, args []string) error { ctx := cmd.Context() - dlog.Infof(ctx, "Reading %q...", args[0]) - nodeScanResults, err := readJSONFile[btrfsinspect.ScanDevicesResult](ctx, args[0]) - if err != nil { - return err - } - dlog.Infof(ctx, "... done reading %q", args[0]) + // This is wrapped in a func in order to *ensure* that `nodeScanResults` goes out of scope once + // `rebuilder` has been created. + rebuilder, err := func(ctx context.Context) (rebuildnodes.Rebuilder, error) { + dlog.Infof(ctx, "Reading %q...", args[0]) + nodeScanResults, err := readJSONFile[btrfsinspect.ScanDevicesResult](ctx, args[0]) + if err != nil { + return nil, err + } + dlog.Infof(ctx, "... done reading %q", args[0]) - rebuilder, err := rebuildnodes.NewRebuilder(ctx, fs, nodeScanResults) + return rebuildnodes.NewRebuilder(ctx, fs, nodeScanResults) + }(ctx) if err != nil { return err } + runtime.GC() + time.Sleep(textui.LiveMemUseUpdateInterval) // let the logs reflect that GC right away + dlog.Info(ctx, "Rebuilding node tree...") rebuildErr := rebuilder.Rebuild(ctx) dst := os.Stdout diff --git a/lib/textui/log_memstats.go b/lib/textui/log_memstats.go index 6e3c3a1..7ef35da 100644 --- a/lib/textui/log_memstats.go +++ b/lib/textui/log_memstats.go @@ -19,14 +19,14 @@ type LiveMemUse struct { var _ fmt.Stringer = (*LiveMemUse)(nil) -var liveMemUseUpdateInterval = Tunable(1 * time.Second) +var LiveMemUseUpdateInterval = Tunable(1 * time.Second) func (o *LiveMemUse) String() string { o.mu.Lock() // runtime.ReadMemStats() calls stopTheWorld(), so we want to // rate-limit how often we call it. - if now := time.Now(); now.Sub(o.last) > liveMemUseUpdateInterval { + if now := time.Now(); now.Sub(o.last) > LiveMemUseUpdateInterval { runtime.ReadMemStats(&o.stats) o.last = now } -- cgit v1.2.3-54-g00ecf