From a5cba20c1e3b0956737327ef9214fd2cbc221add Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 26 Jan 2020 09:05:07 -0500 Subject: initial commit --- go.mod | 5 ++ go.sum | 10 +++ rrdformat/errors_binary.go | 75 +++++++++++++++++ rrdformat/errors_binary_test.go | 34 ++++++++ rrdformat/format.go | 173 ++++++++++++++++++++++++++++++++++++++++ rrdformat/format_test.go | 14 ++++ 6 files changed, 311 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 rrdformat/errors_binary.go create mode 100644 rrdformat/errors_binary_test.go create mode 100644 rrdformat/format.go create mode 100644 rrdformat/format_test.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2989deb --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module git.lukeshu.com/go/librrd + +go 1.13 + +require github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e863f51 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/rrdformat/errors_binary.go b/rrdformat/errors_binary.go new file mode 100644 index 0000000..7329927 --- /dev/null +++ b/rrdformat/errors_binary.go @@ -0,0 +1,75 @@ +package rrdformat + +import ( + "fmt" + "io" +) + +type BinaryError struct { + msg string + ctxPos int + ctxDat []byte + ctxEOF bool +} + +func newBinError(msg string, ctxFile []byte, ctxStart, ctxLen int) error { + if ctxStart+ctxLen > len(ctxFile) { + ctxLen = len(ctxFile) - ctxStart + } + return BinaryError{ + msg: msg, + ctxPos: ctxStart, + ctxDat: ctxFile[ctxStart : ctxStart+ctxLen], + ctxEOF: ctxStart+ctxLen == len(ctxFile), + } +} + +func (e BinaryError) Error() string { + return "invalid RRD: " + e.msg +} + +var cAsciiEscapes = map[byte]byte{ + 0x00: '0', + 0x07: 'a', + 0x08: 'b', + 0x09: 't', + 0x0A: 'n', + 0x0B: 'v', + 0x0C: 'f', + 0x0D: 'r', +} + +func (e BinaryError) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + io.WriteString(s, e.Error()) + if s.Flag('+') { + fmt.Fprintf(s, "\n\tat byte %d:", e.ctxPos) + io.WriteString(s, "\n\t\tascii:") + for _, byte := range e.ctxDat { + if ' ' <= byte && byte <= '~' { + fmt.Fprintf(s, " %c", byte) + } else if c, ok := cAsciiEscapes[byte]; ok { + fmt.Fprintf(s, " \\%c", c) + } else { + io.WriteString(s, " ??") + } + } + if e.ctxEOF { + io.WriteString(s, " ") + } + io.WriteString(s, "\n\t\thex :") + for _, byte := range e.ctxDat { + fmt.Fprintf(s, " %02x", byte) + } + if e.ctxEOF { + io.WriteString(s, " ") + } + io.WriteString(s, "\n") + } + case 's': + io.WriteString(s, e.Error()) + case 'q': + fmt.Fprintf(s, "%q", e.Error()) + } +} diff --git a/rrdformat/errors_binary_test.go b/rrdformat/errors_binary_test.go new file mode 100644 index 0000000..f420a1b --- /dev/null +++ b/rrdformat/errors_binary_test.go @@ -0,0 +1,34 @@ +package rrdformat + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBinaryError(t *testing.T) { + assert := assert.New(t) + + bad404 := []byte(` + hex : 52 +`) +} diff --git a/rrdformat/format.go b/rrdformat/format.go new file mode 100644 index 0000000..0233a5c --- /dev/null +++ b/rrdformat/format.go @@ -0,0 +1,173 @@ +package rrdformat + +import ( + "bytes" + "encoding" + "encoding/binary" + "encoding/xml" + "math" +) + +type Unival uint64 + +func (u Unival) AsUint64() uint64 { return uint64(u) } +func (u Unival) AsFloat64() float64 { return math.Float64frombits(uint64(u)) } + +const XMLNS = "https://oss.oetiker.ch/rrdtool/rrdtool-dump.xml" + +// rrdtool: +// rrd_format.h: stat_head_t -- static header of the database +// +// javascriptRRD: +// rrdFile.js: RRDHeader +type Header struct { + // identification section + Cookie []byte // 4 bytes + Version []byte // 4 bytes (eh, we let the \0-terminator decided how long) + // padding + FloatCookie []byte // 8 bytes + + Bytes + + // structure definition + DSCnt uint64 // how many different DS provide input to the RRD + RRACnt uint64 // how many RRAs will be maintained in the RRD + PDPStep uint64 // PDP interval in seconds + Parameters [10]Unival + + // sniffed metadata + ByteOrder binary.ByteOrder + FloatWidth int + FloatAlign int + IntWidth int + IntAlign int + UnivalWidth int + UnivalAlign int +} + +func (h *Header) UnmarshalBinary(data []byte) error { + // magic number cookie + if !bytes.HasPrefix(data, []byte("RRD\x00")) { + return newBinError("not an RRD file: wrong magic number", data, 0, 4) + } + h.Cookie = data[0:4] + + // version string + null := bytes.IndexByte(data[4:], 0) + if null < 0 { + return newBinError("no null-terminator on version string", data, 4, 5) + } + null += 4 + h.Version = data[4:null] + switch string(h.Version) { + case "0003": + case "0004": + case "0005": + default: + } + + // float cookie + // + // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic + // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that: + // - a 'double' is 8 bytes wide + + // - the value will be exactly equal, and we don't need to worry about weird rounding. + h.FloatWidth = 8 + magicFloat := float64(8.642135e130) + floatAddrPacked := null + 1 + floatAddr32 := ((floatAddrPacked + 3) / 4) * 4 + floatAddr64 := ((floatAddrPacked + 7) / 8) * 8 + var restOffset int + switch { + case len(data) < floatAddr32+h.FloatWidth: + return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat: + h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] + h.ByteOrder = binary.LittleEndian + h.FloatAlign = 4 + restOffset = floatAddr32 + h.FloatWidth + case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat: + h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth] + h.ByteOrder = binary.BigEndian + h.FloatAlign = 4 + restOffset = floatAddr32 + h.FloatWidth + case len(data) < floatAddr64+h.FloatWidth: + return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat: + h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] + h.ByteOrder = binary.LittleEndian + h.FloatAlign = 8 + restOffset = floatAddr64 + h.FloatWidth + case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat: + h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth] + h.ByteOrder = binary.BigEndian + h.FloatAlign = 8 + restOffset = floatAddr64 + h.FloatWidth + default: + return newBinError("failed to sniff byte-order and float-alignment", + data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked) + } + + switch h.FloatAlign { + case 4: + // Assume that if floats are only 32-bit aligned, then everything is 32-bit + h.IntWidth = 4 + h.IntAlign = 4 + case 8: + // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints. + + // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.) + // + // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be + // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is. + // + // By assuming + // 1. ds_cnt <= math.MaxUint32 + // 2l. rra_cnt > 0 (relevant if little-endian) + // 2b. ds_cnt > 0 (relevant if big-endian) + // we can inspect the 4 bytes (marked "big" or "little" below) that are either + // - the most significant bits of 64-bit ds_cnt, or + // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian) + // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt. + // + // | | | | | | | big | little | + // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31| + // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->| + // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>| + // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>| + if len(data) < restOffset+8 { + return newBinError("unexpected end of file", data, restOffset, 8) + } + offset := map[binary.ByteOrder]int{ + binary.BigEndian: restOffset, // 24 in the above diagram + binary.LittleEndian: restOffset + 4, // 28 in the above diagram + }[h.ByteOrder] + if h.ByteOrder.Uint32(data[offset:]) == 0 { + h.IntWidth = 8 + h.IntAlign = 8 + } else { + h.IntWidth = 4 + h.IntAlign = 4 + } + } + + return nil +} + +func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil { + return err + } + if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil { + return err + } + return nil +} + +//var _ encoding.BinaryMarshaler = &Header{} +var _ encoding.BinaryUnmarshaler = &Header{} + +var _ xml.Marshaler = &Header{} + +//var _ xml.Unmarshaler = &Header{} diff --git a/rrdformat/format_test.go b/rrdformat/format_test.go new file mode 100644 index 0000000..e2c1aa0 --- /dev/null +++ b/rrdformat/format_test.go @@ -0,0 +1,14 @@ +package rrdformat + +import ( + "encoding/xml" + "testing" +) + +func TestXML(t *testing.T) { + out, err := xml.Marshal(&Header{ + Version: []byte("0003"), + PDPStep: 300, + }) + t.Log(string(out), err) +} -- cgit v1.2.3