diff options
| -rw-r--r-- | rrdformat/format.go | 210 | ||||
| -rw-r--r-- | rrdformat/rrdbinary/types.go | 11 | ||||
| -rw-r--r-- | rrdformat/rrdbinary/unmarshal.go | 99 | ||||
| -rw-r--r-- | rrdformat/sniff.go | 125 |
4 files changed, 237 insertions, 208 deletions
diff --git a/rrdformat/format.go b/rrdformat/format.go index 4e68b93..71bd966 100644 --- a/rrdformat/format.go +++ b/rrdformat/format.go @@ -10,15 +10,43 @@ import ( "git.lukeshu.com/go/librrd/rrdformat/rrdbinary" ) +// File format versions: +// +// For the most part, the layout of the file format hasn't +// changed--the version number just indicates which enum members you +// need to support. The only parser-breaking change is that the +// 0002->0003 transition added a few bytes to the timestamp, in order +// to go from second-precision to microsecond-precision. +// +// - "0005" (added in rrdtool 1.5.0, 2015-04-16) +// * added DST_DCOUNTER +// * added DST_DDERIVE +// * otherwise identical to "0004" +// +// - "0004" (added in rrdtool 1.3.0, 2008-06-10) +// * added MHWPREDICT function +// * otherwise identical to "0003" +// +// - "0003" (added in the unreleased rrdtool 1.1.x in 2003, +// present in rrdtool 1.2.0, 2005-04-25) +// * last-updated timestamp gained a `long` microsecond component +// * otherwise identical to "0002" +// +// - "0002" (never used by a stable release of rrdtool, but was used +// by development versions of 1.1.x 2001-2003) +// * added DST_CDEF +// * added CF_HWPREDICT +// * added CF_SEASONAL +// * added CF_DEVPREDICT +// * added CF_DEVSEASONAL +// * added CF_FAILURES +// +// - "0001" (the original in rrdtool 1.0.0, 1999-07-15) + const XMLNS = "https://oss.oetiker.ch/rrdtool/rrdtool-dump.xml" type RRDValue = rrdbinary.Float -// rrdtool: -// rrd_format.h: stat_head_t -- static header of the database -// -// javascriptRRD: -// rrdFile.js: RRDHeader type Header struct { Cookie rrdbinary.String `rrdbinary:"size=3" xml:"-"` Version rrdbinary.String `rrdbinary:"size=4" xml:"version"` @@ -41,6 +69,11 @@ type RRADef struct { PDPCnt rrdBinary.Uint } +type Timestamp struct { + Sec rrdbinary.Time + Usec rrdbinary.Int // signed, but always >= 0 +} + type PDPPrep struct { LastDS rrdbinary.String `rrdbinary:"size=30"` Scratch [10]rrdbinary.Unival @@ -54,166 +87,29 @@ type RRAPtr struct { CurRow rrdbinary.Uint } -// rrdtool: -// rrd_format.h: rrd_t One single struct to hold all the others. -type RRD struct { +type RRDv0005 = RRDv0004 +type RRDv0004 = RRDv0003 + +type RRDv0003 struct { Header Header DSDefs []DSDef RRADefs []RRADef - LastUpdated rrdbinary.Timesstamp + LastUpdated Timestamp PDPPrep TODO CPDPrep TODO RRAPtr TODO Values []RRDValue } -type Header struct { - // identification section - Cookie []byte // 4 bytes - Version []byte // 4 bytes (eh, we let the \0-terminator decided how long) - // padding - FloatCookie []byte // 8 bytes - - Bytes - - // structure definition - DSCnt uint64 // how many different DS provide input to the RRD - RRACnt uint64 // how many RRAs will be maintained in the RRD - PDPStep uint64 // PDP interval in seconds - Parameters [10]Unival - - // sniffed metadata - ByteOrder binary.ByteOrder - FloatWidth int - FloatAlign int - IntWidth int - IntAlign int - UnivalWidth int - UnivalAlign int -} +type RRDv0002 = RRDv0001 -func (h *Header) UnmarshalBinary(data []byte) error { - // magic number cookie - if !bytes.HasPrefix(data, []byte("RRD\x00")) { - return newBinError("not an RRD file: wrong magic number", data, 0, 4) - } - h.Cookie = data[0:4] - - // version string - null := bytes.IndexByte(data[4:], 0) - if null < 0 { - return newBinError("no null-terminator on version string", data, 4, 5) - } - null += 4 - h.Version = data[4:null] - switch string(h.Version) { - case "0003": - case "0004": - case "0005": - default: - } - - // float cookie - // - // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic - // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that: - // - a 'double' is 8 bytes wide - // - the value will be exactly equal, and we don't need to worry about weird rounding. - h.FloatWidth = 8 - magicFloat := float64(8.642135e130) - floatAddrPacked := null + 1 - floatAddr32 := ((floatAddrPacked + 3) / 4) * 4 - floatAddr64 := ((floatAddrPacked + 7) / 8) * 8 - var restOffset int - switch { - case len(data) < floatAddr32+h.FloatWidth: - return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) - case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat: - h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] - h.ByteOrder = binary.LittleEndian - h.FloatAlign = 4 - restOffset = floatAddr32 + h.FloatWidth - case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat: - h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] - h.ByteOrder = binary.BigEndian - h.FloatAlign = 4 - restOffset = floatAddr32 + h.FloatWidth - case len(data) < floatAddr64+h.FloatWidth: - return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) - case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat: - h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] - h.ByteOrder = binary.LittleEndian - h.FloatAlign = 8 - restOffset = floatAddr64 + h.FloatWidth - case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat: - h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] - h.ByteOrder = binary.BigEndian - h.FloatAlign = 8 - restOffset = floatAddr64 + h.FloatWidth - default: - return newBinError("failed to sniff byte-order and float-alignment", - data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) - } - - switch h.FloatAlign { - case 4: - // Assume that if floats are only 32-bit aligned, then everything is 32-bit - h.IntWidth = 4 - h.IntAlign = 4 - case 8: - // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints. - - // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.) - // - // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be - // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is. - // - // By assuming - // 1. ds_cnt <= math.MaxUint32 - // 2l. rra_cnt > 0 (relevant if little-endian) - // 2b. ds_cnt > 0 (relevant if big-endian) - // we can inspect the 4 bytes (marked "big" or "little" below) that are either - // - the most significant bits of 64-bit ds_cnt, or - // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian) - // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt. - // - // | | | | | | | big | little | - // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31| - // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->| - // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>| - // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>| - if len(data) < restOffset+8 { - return newBinError("unexpected end of file", data, restOffset, 8) - } - offset := map[binary.ByteOrder]int{ - binary.BigEndian: restOffset, // 24 in the above diagram - binary.LittleEndian: restOffset + 4, // 28 in the above diagram - }[h.ByteOrder] - if h.ByteOrder.Uint32(data[offset:]) == 0 { - h.IntWidth = 8 - h.IntAlign = 8 - } else { - h.IntWidth = 4 - h.IntAlign = 4 - } - } - - return nil -} - -func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil { - return err - } - if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil { - return err - } - return nil +type RRDv0001 struct { + Header Header + DSDefs []DSDef + RRADefs []RRADef + LastUpdated rrdbinary.Timestamp + PDPPrep TODO + CPDPrep TODO + RRAPtr TODO + Values []RRDValue } - -//var _ encoding.BinaryMarshaler = &Header{} -var _ encoding.BinaryUnmarshaler = &Header{} - -var _ xml.Marshaler = &Header{} - -//var _ xml.Unmarshaler = &Header{} diff --git a/rrdformat/rrdbinary/types.go b/rrdformat/rrdbinary/types.go index 892bb25..36e89fa 100644 --- a/rrdformat/rrdbinary/types.go +++ b/rrdformat/rrdbinary/types.go @@ -21,11 +21,12 @@ type Architecture struct { TimeAlign int } -type String string // \0-terminatd -type Float float64 // 8 bytes -type Uint uint64 // 4 or 8 bytes -type Unival uint64 // 8 bytes -type Timestamp time.time // 8, 12, or 16 bytes +type String string // \0-terminated +type Float float64 // 8 bytes +type Uint uint64 // 4 or 8 bytes +type _Int int64 // 4 or 8 bytes +type Unival uint64 // 8 bytes +type Time int64 // 4 or 8 bytes, only has second-precision func (u Unival) AsUint64() uint64 { return uint64(u) } func (u Unival) AsFloat64() float64 { return math.Float64frombits(uint64(u)) } diff --git a/rrdformat/rrdbinary/unmarshal.go b/rrdformat/rrdbinary/unmarshal.go index 34fc537..e0f8988 100644 --- a/rrdformat/rrdbinary/unmarshal.go +++ b/rrdformat/rrdbinary/unmarshal.go @@ -44,10 +44,12 @@ func (d *unmarshaler) unmarshal(v reflect.Value, tag string) error { return d.unmarshalFloat(v, tag) case reflect.TypeOf(Uint(0)): return d.unmarshalUint(v, tag) + case reflect.TypeOf(_Int(0)): + return d.unmarshalInt(v, tag) case reflect.TypeOf(Unival(0)): return d.unmarshalUnival(v, tag) - case reflect.TypeOf(Timestamp{}): - return d.unmarshalTimestamp(v, tag) + case reflect.TypeOf(Time(0)): + return d.unmarshalTime(v, tag) default: switch v.Type().Kind() { case reflect.Struct: @@ -210,6 +212,42 @@ func (d *unmarshaler) unmarshalUint(v reflect.Value, tag string) error { return nil } +func (d *unmarshaler) unmarshalInt(v reflect.Value, tag string) error { + panicUnless(v.Type() == reflect.TypeOf(_Int(0))) + panicUnless(v.CanSet()) + + if d.arch.IntWidth != 4 && d.arch.IntWidth != 8 { + return archErrorf("rrdbinary does not support IntWidth=%d; only supports 4 or 8", d.arch.IntWidth) + } + if tag != "" { + return typeErrorf("invalid rrdbinary struct tag for int: %q", tag) + } + + data := d.data[d.pos:] + + padding := 0 + if d.pos%d.arch.IntAlign != 0 { + padding = d.arch.IntAlign - (d.pos % d.arch.IntAlign) + } + if len(data) < padding { + return d.binErrorf(padding+d.arch.IntWidth, "unexpected end-of-file in %d-byte padding-before-int", padding) + } + data = data[padding:] + + if len(data) < d.arch.IntWidth { + return d.binErrorf(d.arch.IntWidth, "unexpected end-of-file in %d-byte int", d.arch.IntWidth) + } + + switch d.arch.IntWidth { + case 4: + v.SetInt(int64(int32(d.arch.ByteOrder.Uint32(data)))) + case 8: + v.SetInt(int64(d.arch.ByteOrder.Uint64(data))) + } + d.pos += padding + d.arch.IntWidth + return nil +} + func (d *unmarshaler) unmarshalUnival(v reflect.Value, tag string) error { panicUnless(v.Type() == reflect.TypeOf(Unival(0))) panicUnless(v.CanSet()) @@ -241,69 +279,38 @@ func (d *unmarshaler) unmarshalUnival(v reflect.Value, tag string) error { return nil } -func (d *unmarshaler) unmarshalTimestamp(v reflect.Value, tag string) error { - panicUnless(v.Type() == reflect.TypeOf(Timestamp{})) +func (d *unmarshaler) unmarshalTime(v reflect.Value, tag string) error { + panicUnless(v.Type() == reflect.TypeOf(Time(0))) panicUnless(v.CanSet()) + if d.arch.TimeWidth != 4 && d.arch.TimeWidth != 8 { + return archErrorf("rrdbinary does not support TimeWidth=%d; only supports 4 or 8", d.arch.TimeWidth) + } if tag != "" { - return typeErrorf("invalid rrdbinary struct tag for timestamp: %q", tag) + return typeErrorf("invalid rrdbinary struct tag for time: %q", tag) } data := d.data[d.pos:] - var sec, usec int64 - // seconds -- time_t - if d.arch.TimeWidth != 4 && d.arch.TimeWidth != 8 { - return archErrorf("rrdbinary does not support TimeWidth=%d; only supports 4 or 8", d.arch.TimeWidth) - } - // padding padding := 0 if d.pos%d.arch.TimeAlign != 0 { padding = d.arch.TimeAlign - (d.pos % d.arch.TimeAlign) } if len(data) < padding { - return d.binErrorf(padding+d.arch.TimeWidth, "unexpected end-of-file in %d-byte padding-before-time_t", padding) + return d.binErrorf(padding+d.arch.TimeWidth, "unexpected end-of-file in %d-byte padding-before-time", padding) } data = data[padding:] - // value + if len(data) < d.arch.TimeWidth { - return d.binErrorf(d.arch.TimeWidth, "unexpected end-of-file in %d-byte time_t", d.arch.TimeWidth) + return d.binErrorf(d.arch.TimeWidth, "unexpected end-of-file in %d-byte time", d.arch.TimeWidth) } - switch d.arch.TimeWidth { - case 4: - sec = int64(d.arch.ByteOrder.Int32(data)) - case 8: - sec = d.arch.ByteOrder.Int64(data) - } - data = data[d.arch.TimeWidth:] - // nanoseconds -- long - if d.arch.IntWidth != 4 && d.arch.IntWidth != 8 { - return archErrorf("rrdbinary does not support IntWidth=%d; only supports 4 or 8", d.arch.IntWidth) - } - // padding - padding = 0 - if d.pos%d.arch.IntAlign != 0 { - padding = d.arch.IntAlign - (d.pos % d.arch.IntAlign) - } - if len(data) < padding { - return d.binErrorf(padding+d.arch.IntWidth, "unexpected end-of-file in %d-byte padding-before-int", padding) - } - data = data[padding:] - // value - if len(data) < d.arch.IntWidth { - return d.binErrorf(d.arch.IntWidth, "unexpected end-of-file in %d-byte int", d.arch.IntWidth) - } - switch d.arch.IntWidth { + switch d.arch.TimeWidth { case 4: - usec = int64(d.arch.ByteOrder.Int32(data)) + v.SetInt(int64(int32(d.arch.ByteOrder.Uint32(data)))) case 8: - usec = d.arch.ByteOrder.Int64(data) + v.SetInt(int64(d.arch.ByteOrder.Uint64(data))) } - data = data[d.arch.IntWidth:] - - // put it all together - v.Set(reflect.ValueOf(time.Unix(sec, usec*1000))) - d.pos = len(pos.data) - len(data) + d.pos += padding + d.arch.TimeWidth return nil } diff --git a/rrdformat/sniff.go b/rrdformat/sniff.go new file mode 100644 index 0000000..f8f3397 --- /dev/null +++ b/rrdformat/sniff.go @@ -0,0 +1,125 @@ +func (h *Header) UnmarshalBinary(data []byte) error { + // magic number cookie + if !bytes.HasPrefix(data, []byte("RRD\x00")) { + return newBinError("not an RRD file: wrong magic number", data, 0, 4) + } + h.Cookie = data[0:4] + + // version string + null := bytes.IndexByte(data[4:], 0) + if null < 0 { + return newBinError("no null-terminator on version string", data, 4, 5) + } + null += 4 + h.Version = data[4:null] + switch string(h.Version) { + case "0003": + case "0004": + case "0005": + default: + } + + // float cookie + // + // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic + // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that: + // - a 'double' is 8 bytes wide + // - the value will be exactly equal, and we don't need to worry about weird rounding. + h.FloatWidth = 8 + magicFloat := float64(8.642135e130) + floatAddrPacked := null + 1 + floatAddr32 := ((floatAddrPacked + 3) / 4) * 4 + floatAddr64 := ((floatAddrPacked + 7) / 8) * 8 + var restOffset int + switch { + case len(data) < floatAddr32+h.FloatWidth: + return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat: + h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] + h.ByteOrder = binary.LittleEndian + h.FloatAlign = 4 + restOffset = floatAddr32 + h.FloatWidth + case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat: + h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] + h.ByteOrder = binary.BigEndian + h.FloatAlign = 4 + restOffset = floatAddr32 + h.FloatWidth + case len(data) < floatAddr64+h.FloatWidth: + return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat: + h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] + h.ByteOrder = binary.LittleEndian + h.FloatAlign = 8 + restOffset = floatAddr64 + h.FloatWidth + case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat: + h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] + h.ByteOrder = binary.BigEndian + h.FloatAlign = 8 + restOffset = floatAddr64 + h.FloatWidth + default: + return newBinError("failed to sniff byte-order and float-alignment", + data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + } + + switch h.FloatAlign { + case 4: + // Assume that if floats are only 32-bit aligned, then everything is 32-bit + h.IntWidth = 4 + h.IntAlign = 4 + case 8: + // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints. + + // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.) + // + // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be + // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is. + // + // By assuming + // 1. ds_cnt <= math.MaxUint32 + // 2l. rra_cnt > 0 (relevant if little-endian) + // 2b. ds_cnt > 0 (relevant if big-endian) + // we can inspect the 4 bytes (marked "big" or "little" below) that are either + // - the most significant bits of 64-bit ds_cnt, or + // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian) + // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt. + // + // | | | | | | | big | little | + // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31| + // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->| + // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>| + // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>| + if len(data) < restOffset+8 { + return newBinError("unexpected end of file", data, restOffset, 8) + } + offset := map[binary.ByteOrder]int{ + binary.BigEndian: restOffset, // 24 in the above diagram + binary.LittleEndian: restOffset + 4, // 28 in the above diagram + }[h.ByteOrder] + if h.ByteOrder.Uint32(data[offset:]) == 0 { + h.IntWidth = 8 + h.IntAlign = 8 + } else { + h.IntWidth = 4 + h.IntAlign = 4 + } + } + + return nil +} + +func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil { + return err + } + if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil { + return err + } + return nil +} + +//var _ encoding.BinaryMarshaler = &Header{} +var _ encoding.BinaryUnmarshaler = &Header{} + +var _ xml.Marshaler = &Header{} + +//var _ xml.Unmarshaler = &Header{} |