From d6d19b3a8f66f27f78fd2c6fa02b32fb2a88e9ad Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 25 Jan 2023 17:41:28 -0700 Subject: Move btrfs-rec's logging RuneScanner to a new `streamio` package --- cmd/btrfs-rec/util.go | 61 ++------------------------------------------------- 1 file changed, 2 insertions(+), 59 deletions(-) (limited to 'cmd/btrfs-rec/util.go') diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index 9a0d60c..b2497b2 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -9,77 +9,20 @@ import ( "context" "io" "os" - "time" "git.lukeshu.com/go/lowmemjson" "github.com/datawire/dlib/dlog" - "git.lukeshu.com/btrfs-progs-ng/lib/textui" + "git.lukeshu.com/btrfs-progs-ng/lib/streamio" ) -type runeScanner struct { - ctx context.Context //nolint:containedctx // For detecting shutdown from methods - progress textui.Portion[int64] - progressWriter *textui.Progress[textui.Portion[int64]] - unreadCnt uint64 - reader *bufio.Reader - closer io.Closer -} - -func newRuneScanner(ctx context.Context, fh *os.File) (*runeScanner, error) { - fi, err := fh.Stat() - if err != nil { - return nil, err - } - ret := &runeScanner{ - ctx: ctx, - progress: textui.Portion[int64]{ - D: fi.Size(), - }, - progressWriter: textui.NewProgress[textui.Portion[int64]](ctx, dlog.LogLevelInfo, textui.Tunable(1*time.Second)), - reader: bufio.NewReader(fh), - closer: fh, - } - return ret, nil -} - -func (rs *runeScanner) ReadRune() (r rune, size int, err error) { - if err := rs.ctx.Err(); err != nil { - return 0, 0, err - } - r, size, err = rs.reader.ReadRune() - if rs.unreadCnt > 0 { - rs.unreadCnt-- - } else { - rs.progress.N += int64(size) - rs.progressWriter.Set(rs.progress) - } - return -} - -func (rs *runeScanner) UnreadRune() error { - if err := rs.ctx.Err(); err != nil { - return err - } - if err := rs.reader.UnreadRune(); err != nil { - return err - } - rs.unreadCnt++ - return nil -} - -func (rs *runeScanner) Close() error { - rs.progressWriter.Done() - return rs.closer.Close() -} - func readJSONFile[T any](ctx context.Context, filename string) (T, error) { fh, err := os.Open(filename) if err != nil { var zero T return zero, err } - buf, err := newRuneScanner(dlog.WithField(ctx, "btrfs.read-json-file", filename), fh) + buf, err := streamio.NewRuneScanner(dlog.WithField(ctx, "btrfs.read-json-file", filename), fh) defer func() { _ = buf.Close() }() -- cgit v1.2.3-54-g00ecf From f4f062d7d4ed730411e04ecd36ee36387e50739c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 26 Jan 2023 01:08:22 -0700 Subject: Upgrade to lowmemjson v0.2.0 --- cmd/btrfs-rec/util.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- lib/btrfs/btrfssum/shortsum.go | 10 +++++----- lib/containers/set.go | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) (limited to 'cmd/btrfs-rec/util.go') diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index b2497b2..3c72cae 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -31,7 +31,7 @@ func readJSONFile[T any](ctx context.Context, filename string) (T, error) { return zero, err } var ret T - if err := lowmemjson.DecodeThenEOF(buf, &ret); err != nil { + if err := lowmemjson.NewDecoder(buf).DecodeThenEOF(&ret); err != nil { var zero T return zero, err } @@ -46,5 +46,5 @@ func writeJSONFile(w io.Writer, obj any, cfg lowmemjson.ReEncoder) (err error) { } }() cfg.Out = buffer - return lowmemjson.Encode(&cfg, obj) + return lowmemjson.NewEncoder(&cfg).Encode(obj) } diff --git a/go.mod b/go.mod index 0d06d37..bd3f702 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ module git.lukeshu.com/btrfs-progs-ng go 1.19 require ( - git.lukeshu.com/go/lowmemjson v0.1.0 + git.lukeshu.com/go/lowmemjson v0.2.0 github.com/datawire/dlib v1.3.0 github.com/datawire/ocibuild v0.0.3-0.20220423003204-fc6a4e9f90dc github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 62c6e41..c6e06c5 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.lukeshu.com/go/lowmemjson v0.1.0 h1:goVddyNYqpcotUNenrzz7sVjrtCf942h8qzGoqNuxMg= -git.lukeshu.com/go/lowmemjson v0.1.0/go.mod h1:7StdaFpmZNKYJmQ67fGbzcIcnrGjmD54f/2WbeHLaBw= +git.lukeshu.com/go/lowmemjson v0.2.0 h1:ybTArT2jmTJ1QVFGycnGX20zFDKBaqQp4S+dI3vOkTI= +git.lukeshu.com/go/lowmemjson v0.2.0/go.mod h1:7StdaFpmZNKYJmQ67fGbzcIcnrGjmD54f/2WbeHLaBw= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/datawire/dlib v1.3.0 h1:KkmyXU1kwm3oPBk1ypR70YbcOlEXWzEbx5RE0iRXTGk= github.com/datawire/dlib v1.3.0/go.mod h1:NiGDmetmbkBvtznpWSx6C0vA0s0LK9aHna3LJDqjruk= diff --git a/lib/btrfs/btrfssum/shortsum.go b/lib/btrfs/btrfssum/shortsum.go index c3d6f8b..e3441ae 100644 --- a/lib/btrfs/btrfssum/shortsum.go +++ b/lib/btrfs/btrfssum/shortsum.go @@ -248,7 +248,7 @@ func (sg SumRunWithGaps[Addr]) EncodeJSON(w io.Writer) error { } fallthrough default: - if err := lowmemjson.Encode(w, run); err != nil { + if err := lowmemjson.NewEncoder(w).Encode(run); err != nil { return err } cur = run.Addr.Add(run.Size()) @@ -274,18 +274,18 @@ func (sg *SumRunWithGaps[Addr]) DecodeJSON(r io.RuneScanner) error { var name string return lowmemjson.DecodeObject(r, func(r io.RuneScanner) error { - return lowmemjson.Decode(r, &name) + return lowmemjson.NewDecoder(r).Decode(&name) }, func(r io.RuneScanner) error { switch name { case "Addr": - return lowmemjson.Decode(r, &sg.Addr) + return lowmemjson.NewDecoder(r).Decode(&sg.Addr) case "Size": - return lowmemjson.Decode(r, &sg.Size) + return lowmemjson.NewDecoder(r).Decode(&sg.Size) case "Runs": return lowmemjson.DecodeArray(r, func(r io.RuneScanner) error { var run SumRun[Addr] - if err := lowmemjson.Decode(r, &run); err != nil { + if err := lowmemjson.NewDecoder(r).Decode(&run); err != nil { return err } if run.ChecksumSize > 0 { diff --git a/lib/containers/set.go b/lib/containers/set.go index 4fc8aad..b2af494 100644 --- a/lib/containers/set.go +++ b/lib/containers/set.go @@ -71,7 +71,7 @@ func (o Set[T]) EncodeJSON(w io.Writer) error { return less(keys[i], keys[j]) }) - return lowmemjson.Encode(w, keys) + return lowmemjson.NewEncoder(w).Encode(keys) } func (o *Set[T]) DecodeJSON(r io.RuneScanner) error { @@ -87,7 +87,7 @@ func (o *Set[T]) DecodeJSON(r io.RuneScanner) error { *o = Set[T]{} return lowmemjson.DecodeArray(r, func(r io.RuneScanner) error { var val T - if err := lowmemjson.Decode(r, &val); err != nil { + if err := lowmemjson.NewDecoder(r).Decode(&val); err != nil { return err } (*o)[val] = struct{}{} -- cgit v1.2.3-54-g00ecf From fffb8ca73af4fd9e1df353ccf779e89fe097d8ce Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 30 Jan 2023 22:06:46 -0700 Subject: Upgrade to lowmemjson v0.3.0 --- cmd/btrfs-rec/inspect_rebuildmappings.go | 2 +- cmd/btrfs-rec/inspect_rebuildnodes.go | 2 +- cmd/btrfs-rec/inspect_scandevices.go | 2 +- cmd/btrfs-rec/util.go | 5 ++--- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 8 insertions(+), 9 deletions(-) (limited to 'cmd/btrfs-rec/util.go') diff --git a/cmd/btrfs-rec/inspect_rebuildmappings.go b/cmd/btrfs-rec/inspect_rebuildmappings.go index e1f273c..51f0327 100644 --- a/cmd/btrfs-rec/inspect_rebuildmappings.go +++ b/cmd/btrfs-rec/inspect_rebuildmappings.go @@ -47,7 +47,7 @@ func init() { } dlog.Infof(ctx, "Writing reconstructed mappings to stdout...") - if err := writeJSONFile(os.Stdout, fs.LV.Mappings(), lowmemjson.ReEncoder{ + if err := writeJSONFile(os.Stdout, fs.LV.Mappings(), lowmemjson.ReEncoderConfig{ Indent: "\t", ForceTrailingNewlines: true, CompactIfUnder: 120, //nolint:gomnd // This is what looks nice. diff --git a/cmd/btrfs-rec/inspect_rebuildnodes.go b/cmd/btrfs-rec/inspect_rebuildnodes.go index d813f36..e8f9ada 100644 --- a/cmd/btrfs-rec/inspect_rebuildnodes.go +++ b/cmd/btrfs-rec/inspect_rebuildnodes.go @@ -57,7 +57,7 @@ func init() { dlog.Errorf(ctx, "rebuild error: %v", rebuildErr) } dlog.Infof(ctx, "Writing re-built nodes to %s...", dst.Name()) - if err := writeJSONFile(dst, rebuilder.ListRoots(), lowmemjson.ReEncoder{ + if err := writeJSONFile(dst, rebuilder.ListRoots(), lowmemjson.ReEncoderConfig{ Indent: "\t", ForceTrailingNewlines: true, }); err != nil { diff --git a/cmd/btrfs-rec/inspect_scandevices.go b/cmd/btrfs-rec/inspect_scandevices.go index ade9d74..3de35d0 100644 --- a/cmd/btrfs-rec/inspect_scandevices.go +++ b/cmd/btrfs-rec/inspect_scandevices.go @@ -31,7 +31,7 @@ func init() { } dlog.Info(ctx, "Writing scan results to stdout...") - if err := writeJSONFile(os.Stdout, results, lowmemjson.ReEncoder{ + if err := writeJSONFile(os.Stdout, results, lowmemjson.ReEncoderConfig{ Indent: "\t", ForceTrailingNewlines: true, CompactIfUnder: 16, //nolint:gomnd // This is what looks nice. diff --git a/cmd/btrfs-rec/util.go b/cmd/btrfs-rec/util.go index 3c72cae..3d751a6 100644 --- a/cmd/btrfs-rec/util.go +++ b/cmd/btrfs-rec/util.go @@ -38,13 +38,12 @@ func readJSONFile[T any](ctx context.Context, filename string) (T, error) { return ret, nil } -func writeJSONFile(w io.Writer, obj any, cfg lowmemjson.ReEncoder) (err error) { +func writeJSONFile(w io.Writer, obj any, cfg lowmemjson.ReEncoderConfig) (err error) { buffer := bufio.NewWriter(w) defer func() { if _err := buffer.Flush(); err == nil && _err != nil { err = _err } }() - cfg.Out = buffer - return lowmemjson.NewEncoder(&cfg).Encode(obj) + return lowmemjson.NewEncoder(lowmemjson.NewReEncoder(buffer, cfg)).Encode(obj) } diff --git a/go.mod b/go.mod index d14ff26..52aa205 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ module git.lukeshu.com/btrfs-progs-ng go 1.19 require ( - git.lukeshu.com/go/lowmemjson v0.2.0 + git.lukeshu.com/go/lowmemjson v0.3.0 git.lukeshu.com/go/typedsync v0.0.0-20230126205501-1e8afc0ceb1e github.com/datawire/dlib v1.3.0 github.com/datawire/ocibuild v0.0.3-0.20220423003204-fc6a4e9f90dc diff --git a/go.sum b/go.sum index ee0b036..9df9c6b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.lukeshu.com/go/lowmemjson v0.2.0 h1:ybTArT2jmTJ1QVFGycnGX20zFDKBaqQp4S+dI3vOkTI= -git.lukeshu.com/go/lowmemjson v0.2.0/go.mod h1:7StdaFpmZNKYJmQ67fGbzcIcnrGjmD54f/2WbeHLaBw= +git.lukeshu.com/go/lowmemjson v0.3.0 h1:JBoTVAYij6pSMWnUSwxpv+e+3Fz8VRgZVNuYvY+0pvc= +git.lukeshu.com/go/lowmemjson v0.3.0/go.mod h1:7StdaFpmZNKYJmQ67fGbzcIcnrGjmD54f/2WbeHLaBw= git.lukeshu.com/go/typedsync v0.0.0-20230126205501-1e8afc0ceb1e h1:ZAzzElMx7aMgJXC9QXOxIPyoZrWxX00eP2sR4UHYP+4= git.lukeshu.com/go/typedsync v0.0.0-20230126205501-1e8afc0ceb1e/go.mod h1:EAn7NcfoGeGMv3DWxKQnifcT/rYPAIEqp9Rsz//oYqY= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -- cgit v1.2.3-54-g00ecf