summaryrefslogtreecommitdiff
path: root/rrdformat
diff options
context:
space:
mode:
Diffstat (limited to 'rrdformat')
-rw-r--r--rrdformat/format.go210
-rw-r--r--rrdformat/rrdbinary/types.go11
-rw-r--r--rrdformat/rrdbinary/unmarshal.go99
-rw-r--r--rrdformat/sniff.go125
4 files changed, 237 insertions, 208 deletions
diff --git a/rrdformat/format.go b/rrdformat/format.go
index 4e68b93..71bd966 100644
--- a/rrdformat/format.go
+++ b/rrdformat/format.go
@@ -10,15 +10,43 @@ import (
"git.lukeshu.com/go/librrd/rrdformat/rrdbinary"
)
+// File format versions:
+//
+// For the most part, the layout of the file format hasn't
+// changed--the version number just indicates which enum members you
+// need to support. The only parser-breaking change is that the
+// 0002->0003 transition added a few bytes to the timestamp, in order
+// to go from second-precision to microsecond-precision.
+//
+// - "0005" (added in rrdtool 1.5.0, 2015-04-16)
+// * added DST_DCOUNTER
+// * added DST_DDERIVE
+// * otherwise identical to "0004"
+//
+// - "0004" (added in rrdtool 1.3.0, 2008-06-10)
+// * added MHWPREDICT function
+// * otherwise identical to "0003"
+//
+// - "0003" (added in the unreleased rrdtool 1.1.x in 2003,
+// present in rrdtool 1.2.0, 2005-04-25)
+// * last-updated timestamp gained a `long` microsecond component
+// * otherwise identical to "0002"
+//
+// - "0002" (never used by a stable release of rrdtool, but was used
+// by development versions of 1.1.x 2001-2003)
+// * added DST_CDEF
+// * added CF_HWPREDICT
+// * added CF_SEASONAL
+// * added CF_DEVPREDICT
+// * added CF_DEVSEASONAL
+// * added CF_FAILURES
+//
+// - "0001" (the original in rrdtool 1.0.0, 1999-07-15)
+
const XMLNS = "https://oss.oetiker.ch/rrdtool/rrdtool-dump.xml"
type RRDValue = rrdbinary.Float
-// rrdtool:
-// rrd_format.h: stat_head_t -- static header of the database
-//
-// javascriptRRD:
-// rrdFile.js: RRDHeader
type Header struct {
Cookie rrdbinary.String `rrdbinary:"size=3" xml:"-"`
Version rrdbinary.String `rrdbinary:"size=4" xml:"version"`
@@ -41,6 +69,11 @@ type RRADef struct {
PDPCnt rrdBinary.Uint
}
+type Timestamp struct {
+ Sec rrdbinary.Time
+ Usec rrdbinary.Int // signed, but always >= 0
+}
+
type PDPPrep struct {
LastDS rrdbinary.String `rrdbinary:"size=30"`
Scratch [10]rrdbinary.Unival
@@ -54,166 +87,29 @@ type RRAPtr struct {
CurRow rrdbinary.Uint
}
-// rrdtool:
-// rrd_format.h: rrd_t One single struct to hold all the others.
-type RRD struct {
+type RRDv0005 = RRDv0004
+type RRDv0004 = RRDv0003
+
+type RRDv0003 struct {
Header Header
DSDefs []DSDef
RRADefs []RRADef
- LastUpdated rrdbinary.Timesstamp
+ LastUpdated Timestamp
PDPPrep TODO
CPDPrep TODO
RRAPtr TODO
Values []RRDValue
}
-type Header struct {
- // identification section
- Cookie []byte // 4 bytes
- Version []byte // 4 bytes (eh, we let the \0-terminator decided how long)
- // padding
- FloatCookie []byte // 8 bytes
-
- Bytes
-
- // structure definition
- DSCnt uint64 // how many different DS provide input to the RRD
- RRACnt uint64 // how many RRAs will be maintained in the RRD
- PDPStep uint64 // PDP interval in seconds
- Parameters [10]Unival
-
- // sniffed metadata
- ByteOrder binary.ByteOrder
- FloatWidth int
- FloatAlign int
- IntWidth int
- IntAlign int
- UnivalWidth int
- UnivalAlign int
-}
+type RRDv0002 = RRDv0001
-func (h *Header) UnmarshalBinary(data []byte) error {
- // magic number cookie
- if !bytes.HasPrefix(data, []byte("RRD\x00")) {
- return newBinError("not an RRD file: wrong magic number", data, 0, 4)
- }
- h.Cookie = data[0:4]
-
- // version string
- null := bytes.IndexByte(data[4:], 0)
- if null < 0 {
- return newBinError("no null-terminator on version string", data, 4, 5)
- }
- null += 4
- h.Version = data[4:null]
- switch string(h.Version) {
- case "0003":
- case "0004":
- case "0005":
- default:
- }
-
- // float cookie
- //
- // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic
- // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that:
- // - a 'double' is 8 bytes wide
- // - the value will be exactly equal, and we don't need to worry about weird rounding.
- h.FloatWidth = 8
- magicFloat := float64(8.642135e130)
- floatAddrPacked := null + 1
- floatAddr32 := ((floatAddrPacked + 3) / 4) * 4
- floatAddr64 := ((floatAddrPacked + 7) / 8) * 8
- var restOffset int
- switch {
- case len(data) < floatAddr32+h.FloatWidth:
- return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
- case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat:
- h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
- h.ByteOrder = binary.LittleEndian
- h.FloatAlign = 4
- restOffset = floatAddr32 + h.FloatWidth
- case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat:
- h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
- h.ByteOrder = binary.BigEndian
- h.FloatAlign = 4
- restOffset = floatAddr32 + h.FloatWidth
- case len(data) < floatAddr64+h.FloatWidth:
- return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
- case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat:
- h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
- h.ByteOrder = binary.LittleEndian
- h.FloatAlign = 8
- restOffset = floatAddr64 + h.FloatWidth
- case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat:
- h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
- h.ByteOrder = binary.BigEndian
- h.FloatAlign = 8
- restOffset = floatAddr64 + h.FloatWidth
- default:
- return newBinError("failed to sniff byte-order and float-alignment",
- data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
- }
-
- switch h.FloatAlign {
- case 4:
- // Assume that if floats are only 32-bit aligned, then everything is 32-bit
- h.IntWidth = 4
- h.IntAlign = 4
- case 8:
- // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints.
-
- // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.)
- //
- // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be
- // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is.
- //
- // By assuming
- // 1. ds_cnt <= math.MaxUint32
- // 2l. rra_cnt > 0 (relevant if little-endian)
- // 2b. ds_cnt > 0 (relevant if big-endian)
- // we can inspect the 4 bytes (marked "big" or "little" below) that are either
- // - the most significant bits of 64-bit ds_cnt, or
- // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian)
- // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt.
- //
- // | | | | | | | big | little |
- // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|
- // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->|
- // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>|
- // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>|
- if len(data) < restOffset+8 {
- return newBinError("unexpected end of file", data, restOffset, 8)
- }
- offset := map[binary.ByteOrder]int{
- binary.BigEndian: restOffset, // 24 in the above diagram
- binary.LittleEndian: restOffset + 4, // 28 in the above diagram
- }[h.ByteOrder]
- if h.ByteOrder.Uint32(data[offset:]) == 0 {
- h.IntWidth = 8
- h.IntAlign = 8
- } else {
- h.IntWidth = 4
- h.IntAlign = 4
- }
- }
-
- return nil
-}
-
-func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
- if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil {
- return err
- }
- if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil {
- return err
- }
- return nil
+type RRDv0001 struct {
+ Header Header
+ DSDefs []DSDef
+ RRADefs []RRADef
+ LastUpdated rrdbinary.Timestamp
+ PDPPrep TODO
+ CPDPrep TODO
+ RRAPtr TODO
+ Values []RRDValue
}
-
-//var _ encoding.BinaryMarshaler = &Header{}
-var _ encoding.BinaryUnmarshaler = &Header{}
-
-var _ xml.Marshaler = &Header{}
-
-//var _ xml.Unmarshaler = &Header{}
diff --git a/rrdformat/rrdbinary/types.go b/rrdformat/rrdbinary/types.go
index 892bb25..36e89fa 100644
--- a/rrdformat/rrdbinary/types.go
+++ b/rrdformat/rrdbinary/types.go
@@ -21,11 +21,12 @@ type Architecture struct {
TimeAlign int
}
-type String string // \0-terminatd
-type Float float64 // 8 bytes
-type Uint uint64 // 4 or 8 bytes
-type Unival uint64 // 8 bytes
-type Timestamp time.time // 8, 12, or 16 bytes
+type String string // \0-terminated
+type Float float64 // 8 bytes
+type Uint uint64 // 4 or 8 bytes
+type _Int int64 // 4 or 8 bytes
+type Unival uint64 // 8 bytes
+type Time int64 // 4 or 8 bytes, only has second-precision
func (u Unival) AsUint64() uint64 { return uint64(u) }
func (u Unival) AsFloat64() float64 { return math.Float64frombits(uint64(u)) }
diff --git a/rrdformat/rrdbinary/unmarshal.go b/rrdformat/rrdbinary/unmarshal.go
index 34fc537..e0f8988 100644
--- a/rrdformat/rrdbinary/unmarshal.go
+++ b/rrdformat/rrdbinary/unmarshal.go
@@ -44,10 +44,12 @@ func (d *unmarshaler) unmarshal(v reflect.Value, tag string) error {
return d.unmarshalFloat(v, tag)
case reflect.TypeOf(Uint(0)):
return d.unmarshalUint(v, tag)
+ case reflect.TypeOf(_Int(0)):
+ return d.unmarshalInt(v, tag)
case reflect.TypeOf(Unival(0)):
return d.unmarshalUnival(v, tag)
- case reflect.TypeOf(Timestamp{}):
- return d.unmarshalTimestamp(v, tag)
+ case reflect.TypeOf(Time(0)):
+ return d.unmarshalTime(v, tag)
default:
switch v.Type().Kind() {
case reflect.Struct:
@@ -210,6 +212,42 @@ func (d *unmarshaler) unmarshalUint(v reflect.Value, tag string) error {
return nil
}
+func (d *unmarshaler) unmarshalInt(v reflect.Value, tag string) error {
+ panicUnless(v.Type() == reflect.TypeOf(_Int(0)))
+ panicUnless(v.CanSet())
+
+ if d.arch.IntWidth != 4 && d.arch.IntWidth != 8 {
+ return archErrorf("rrdbinary does not support IntWidth=%d; only supports 4 or 8", d.arch.IntWidth)
+ }
+ if tag != "" {
+ return typeErrorf("invalid rrdbinary struct tag for int: %q", tag)
+ }
+
+ data := d.data[d.pos:]
+
+ padding := 0
+ if d.pos%d.arch.IntAlign != 0 {
+ padding = d.arch.IntAlign - (d.pos % d.arch.IntAlign)
+ }
+ if len(data) < padding {
+ return d.binErrorf(padding+d.arch.IntWidth, "unexpected end-of-file in %d-byte padding-before-int", padding)
+ }
+ data = data[padding:]
+
+ if len(data) < d.arch.IntWidth {
+ return d.binErrorf(d.arch.IntWidth, "unexpected end-of-file in %d-byte int", d.arch.IntWidth)
+ }
+
+ switch d.arch.IntWidth {
+ case 4:
+ v.SetInt(int64(int32(d.arch.ByteOrder.Uint32(data))))
+ case 8:
+ v.SetInt(int64(d.arch.ByteOrder.Uint64(data)))
+ }
+ d.pos += padding + d.arch.IntWidth
+ return nil
+}
+
func (d *unmarshaler) unmarshalUnival(v reflect.Value, tag string) error {
panicUnless(v.Type() == reflect.TypeOf(Unival(0)))
panicUnless(v.CanSet())
@@ -241,69 +279,38 @@ func (d *unmarshaler) unmarshalUnival(v reflect.Value, tag string) error {
return nil
}
-func (d *unmarshaler) unmarshalTimestamp(v reflect.Value, tag string) error {
- panicUnless(v.Type() == reflect.TypeOf(Timestamp{}))
+func (d *unmarshaler) unmarshalTime(v reflect.Value, tag string) error {
+ panicUnless(v.Type() == reflect.TypeOf(Time(0)))
panicUnless(v.CanSet())
+ if d.arch.TimeWidth != 4 && d.arch.TimeWidth != 8 {
+ return archErrorf("rrdbinary does not support TimeWidth=%d; only supports 4 or 8", d.arch.TimeWidth)
+ }
if tag != "" {
- return typeErrorf("invalid rrdbinary struct tag for timestamp: %q", tag)
+ return typeErrorf("invalid rrdbinary struct tag for time: %q", tag)
}
data := d.data[d.pos:]
- var sec, usec int64
- // seconds -- time_t
- if d.arch.TimeWidth != 4 && d.arch.TimeWidth != 8 {
- return archErrorf("rrdbinary does not support TimeWidth=%d; only supports 4 or 8", d.arch.TimeWidth)
- }
- // padding
padding := 0
if d.pos%d.arch.TimeAlign != 0 {
padding = d.arch.TimeAlign - (d.pos % d.arch.TimeAlign)
}
if len(data) < padding {
- return d.binErrorf(padding+d.arch.TimeWidth, "unexpected end-of-file in %d-byte padding-before-time_t", padding)
+ return d.binErrorf(padding+d.arch.TimeWidth, "unexpected end-of-file in %d-byte padding-before-time", padding)
}
data = data[padding:]
- // value
+
if len(data) < d.arch.TimeWidth {
- return d.binErrorf(d.arch.TimeWidth, "unexpected end-of-file in %d-byte time_t", d.arch.TimeWidth)
+ return d.binErrorf(d.arch.TimeWidth, "unexpected end-of-file in %d-byte time", d.arch.TimeWidth)
}
- switch d.arch.TimeWidth {
- case 4:
- sec = int64(d.arch.ByteOrder.Int32(data))
- case 8:
- sec = d.arch.ByteOrder.Int64(data)
- }
- data = data[d.arch.TimeWidth:]
- // nanoseconds -- long
- if d.arch.IntWidth != 4 && d.arch.IntWidth != 8 {
- return archErrorf("rrdbinary does not support IntWidth=%d; only supports 4 or 8", d.arch.IntWidth)
- }
- // padding
- padding = 0
- if d.pos%d.arch.IntAlign != 0 {
- padding = d.arch.IntAlign - (d.pos % d.arch.IntAlign)
- }
- if len(data) < padding {
- return d.binErrorf(padding+d.arch.IntWidth, "unexpected end-of-file in %d-byte padding-before-int", padding)
- }
- data = data[padding:]
- // value
- if len(data) < d.arch.IntWidth {
- return d.binErrorf(d.arch.IntWidth, "unexpected end-of-file in %d-byte int", d.arch.IntWidth)
- }
- switch d.arch.IntWidth {
+ switch d.arch.TimeWidth {
case 4:
- usec = int64(d.arch.ByteOrder.Int32(data))
+ v.SetInt(int64(int32(d.arch.ByteOrder.Uint32(data))))
case 8:
- usec = d.arch.ByteOrder.Int64(data)
+ v.SetInt(int64(d.arch.ByteOrder.Uint64(data)))
}
- data = data[d.arch.IntWidth:]
-
- // put it all together
- v.Set(reflect.ValueOf(time.Unix(sec, usec*1000)))
- d.pos = len(pos.data) - len(data)
+ d.pos += padding + d.arch.TimeWidth
return nil
}
diff --git a/rrdformat/sniff.go b/rrdformat/sniff.go
new file mode 100644
index 0000000..f8f3397
--- /dev/null
+++ b/rrdformat/sniff.go
@@ -0,0 +1,125 @@
+func (h *Header) UnmarshalBinary(data []byte) error {
+ // magic number cookie
+ if !bytes.HasPrefix(data, []byte("RRD\x00")) {
+ return newBinError("not an RRD file: wrong magic number", data, 0, 4)
+ }
+ h.Cookie = data[0:4]
+
+ // version string
+ null := bytes.IndexByte(data[4:], 0)
+ if null < 0 {
+ return newBinError("no null-terminator on version string", data, 4, 5)
+ }
+ null += 4
+ h.Version = data[4:null]
+ switch string(h.Version) {
+ case "0003":
+ case "0004":
+ case "0005":
+ default:
+ }
+
+ // float cookie
+ //
+ // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic
+ // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that:
+ // - a 'double' is 8 bytes wide
+ // - the value will be exactly equal, and we don't need to worry about weird rounding.
+ h.FloatWidth = 8
+ magicFloat := float64(8.642135e130)
+ floatAddrPacked := null + 1
+ floatAddr32 := ((floatAddrPacked + 3) / 4) * 4
+ floatAddr64 := ((floatAddrPacked + 7) / 8) * 8
+ var restOffset int
+ switch {
+ case len(data) < floatAddr32+h.FloatWidth:
+ return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat:
+ h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+ h.ByteOrder = binary.LittleEndian
+ h.FloatAlign = 4
+ restOffset = floatAddr32 + h.FloatWidth
+ case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat:
+ h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+ h.ByteOrder = binary.BigEndian
+ h.FloatAlign = 4
+ restOffset = floatAddr32 + h.FloatWidth
+ case len(data) < floatAddr64+h.FloatWidth:
+ return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat:
+ h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+ h.ByteOrder = binary.LittleEndian
+ h.FloatAlign = 8
+ restOffset = floatAddr64 + h.FloatWidth
+ case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat:
+ h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+ h.ByteOrder = binary.BigEndian
+ h.FloatAlign = 8
+ restOffset = floatAddr64 + h.FloatWidth
+ default:
+ return newBinError("failed to sniff byte-order and float-alignment",
+ data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ }
+
+ switch h.FloatAlign {
+ case 4:
+ // Assume that if floats are only 32-bit aligned, then everything is 32-bit
+ h.IntWidth = 4
+ h.IntAlign = 4
+ case 8:
+ // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints.
+
+ // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.)
+ //
+ // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be
+ // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is.
+ //
+ // By assuming
+ // 1. ds_cnt <= math.MaxUint32
+ // 2l. rra_cnt > 0 (relevant if little-endian)
+ // 2b. ds_cnt > 0 (relevant if big-endian)
+ // we can inspect the 4 bytes (marked "big" or "little" below) that are either
+ // - the most significant bits of 64-bit ds_cnt, or
+ // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian)
+ // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt.
+ //
+ // | | | | | | | big | little |
+ // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|
+ // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->|
+ // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>|
+ // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>|
+ if len(data) < restOffset+8 {
+ return newBinError("unexpected end of file", data, restOffset, 8)
+ }
+ offset := map[binary.ByteOrder]int{
+ binary.BigEndian: restOffset, // 24 in the above diagram
+ binary.LittleEndian: restOffset + 4, // 28 in the above diagram
+ }[h.ByteOrder]
+ if h.ByteOrder.Uint32(data[offset:]) == 0 {
+ h.IntWidth = 8
+ h.IntAlign = 8
+ } else {
+ h.IntWidth = 4
+ h.IntAlign = 4
+ }
+ }
+
+ return nil
+}
+
+func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+ if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil {
+ return err
+ }
+ if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil {
+ return err
+ }
+ return nil
+}
+
+//var _ encoding.BinaryMarshaler = &Header{}
+var _ encoding.BinaryUnmarshaler = &Header{}
+
+var _ xml.Marshaler = &Header{}
+
+//var _ xml.Unmarshaler = &Header{}