summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@lukeshu.com>2020-01-26 09:05:07 -0500
committerLuke Shumaker <lukeshu@lukeshu.com>2020-01-26 09:05:07 -0500
commita5cba20c1e3b0956737327ef9214fd2cbc221add (patch)
treefc19e2d97243449676fc5d2a5a6f57ad356010e6
initial commit
-rw-r--r--go.mod5
-rw-r--r--go.sum10
-rw-r--r--rrdformat/errors_binary.go75
-rw-r--r--rrdformat/errors_binary_test.go34
-rw-r--r--rrdformat/format.go173
-rw-r--r--rrdformat/format_test.go14
6 files changed, 311 insertions, 0 deletions
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..2989deb
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,5 @@
+module git.lukeshu.com/go/librrd
+
+go 1.13
+
+require github.com/stretchr/testify v1.4.0
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..e863f51
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/rrdformat/errors_binary.go b/rrdformat/errors_binary.go
new file mode 100644
index 0000000..7329927
--- /dev/null
+++ b/rrdformat/errors_binary.go
@@ -0,0 +1,75 @@
+package rrdformat
+
+import (
+ "fmt"
+ "io"
+)
+
+type BinaryError struct {
+ msg string
+ ctxPos int
+ ctxDat []byte
+ ctxEOF bool
+}
+
+func newBinError(msg string, ctxFile []byte, ctxStart, ctxLen int) error {
+ if ctxStart+ctxLen > len(ctxFile) {
+ ctxLen = len(ctxFile) - ctxStart
+ }
+ return BinaryError{
+ msg: msg,
+ ctxPos: ctxStart,
+ ctxDat: ctxFile[ctxStart : ctxStart+ctxLen],
+ ctxEOF: ctxStart+ctxLen == len(ctxFile),
+ }
+}
+
+func (e BinaryError) Error() string {
+ return "invalid RRD: " + e.msg
+}
+
+var cAsciiEscapes = map[byte]byte{
+ 0x00: '0',
+ 0x07: 'a',
+ 0x08: 'b',
+ 0x09: 't',
+ 0x0A: 'n',
+ 0x0B: 'v',
+ 0x0C: 'f',
+ 0x0D: 'r',
+}
+
+func (e BinaryError) Format(s fmt.State, verb rune) {
+ switch verb {
+ case 'v':
+ io.WriteString(s, e.Error())
+ if s.Flag('+') {
+ fmt.Fprintf(s, "\n\tat byte %d:", e.ctxPos)
+ io.WriteString(s, "\n\t\tascii:")
+ for _, byte := range e.ctxDat {
+ if ' ' <= byte && byte <= '~' {
+ fmt.Fprintf(s, " %c", byte)
+ } else if c, ok := cAsciiEscapes[byte]; ok {
+ fmt.Fprintf(s, " \\%c", c)
+ } else {
+ io.WriteString(s, " ??")
+ }
+ }
+ if e.ctxEOF {
+ io.WriteString(s, " <EOF>")
+ }
+ io.WriteString(s, "\n\t\thex :")
+ for _, byte := range e.ctxDat {
+ fmt.Fprintf(s, " %02x", byte)
+ }
+ if e.ctxEOF {
+ io.WriteString(s, " <EOF>")
+ }
+ io.WriteString(s, "\n")
+ }
+ case 's':
+ io.WriteString(s, e.Error())
+ case 'q':
+ fmt.Fprintf(s, "%q", e.Error())
+ }
+}
diff --git a/rrdformat/errors_binary_test.go b/rrdformat/errors_binary_test.go
new file mode 100644
index 0000000..f420a1b
--- /dev/null
+++ b/rrdformat/errors_binary_test.go
@@ -0,0 +1,34 @@
+package rrdformat
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestBinaryError(t *testing.T) {
+ assert := assert.New(t)
+
+ bad404 := []byte(`<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"`)
+ err := newBinError("not an RRD file: wrong magic number", bad404, 0, 4)
+ assert.Equal(err.Error(), `invalid RRD: not an RRD file: wrong magic number`)
+ assert.Equal(fmt.Sprintf("%v", err), `invalid RRD: not an RRD file: wrong magic number`)
+ assert.Equal(fmt.Sprintf("%q", err), `"invalid RRD: not an RRD file: wrong magic number"`)
+ assert.Equal(fmt.Sprintf("%+v", err), `invalid RRD: not an RRD file: wrong magic number
+ at byte 0:
+ ascii: < ! D O
+ hex : 3c 21 44 4f
+`)
+
+ badShort := []byte{'R'}
+ err = newBinError("not an RRD file: wrong magic number", badShort, 0, 4)
+ assert.Equal(err.Error(), `invalid RRD: not an RRD file: wrong magic number`)
+ assert.Equal(fmt.Sprintf("%v", err), `invalid RRD: not an RRD file: wrong magic number`)
+ assert.Equal(fmt.Sprintf("%q", err), `"invalid RRD: not an RRD file: wrong magic number"`)
+ assert.Equal(fmt.Sprintf("%+v", err), `invalid RRD: not an RRD file: wrong magic number
+ at byte 0:
+ ascii: R <EOF>
+ hex : 52 <EOF>
+`)
+}
diff --git a/rrdformat/format.go b/rrdformat/format.go
new file mode 100644
index 0000000..0233a5c
--- /dev/null
+++ b/rrdformat/format.go
@@ -0,0 +1,173 @@
+package rrdformat
+
+import (
+ "bytes"
+ "encoding"
+ "encoding/binary"
+ "encoding/xml"
+ "math"
+)
+
+type Unival uint64
+
+func (u Unival) AsUint64() uint64 { return uint64(u) }
+func (u Unival) AsFloat64() float64 { return math.Float64frombits(uint64(u)) }
+
+const XMLNS = "https://oss.oetiker.ch/rrdtool/rrdtool-dump.xml"
+
+// rrdtool:
+// rrd_format.h: stat_head_t -- static header of the database
+//
+// javascriptRRD:
+// rrdFile.js: RRDHeader
+type Header struct {
+ // identification section
+ Cookie []byte // 4 bytes
+ Version []byte // 4 bytes (eh, we let the \0-terminator decided how long)
+ // padding
+ FloatCookie []byte // 8 bytes
+
+ Bytes
+
+ // structure definition
+ DSCnt uint64 // how many different DS provide input to the RRD
+ RRACnt uint64 // how many RRAs will be maintained in the RRD
+ PDPStep uint64 // PDP interval in seconds
+ Parameters [10]Unival
+
+ // sniffed metadata
+ ByteOrder binary.ByteOrder
+ FloatWidth int
+ FloatAlign int
+ IntWidth int
+ IntAlign int
+ UnivalWidth int
+ UnivalAlign int
+}
+
+func (h *Header) UnmarshalBinary(data []byte) error {
+ // magic number cookie
+ if !bytes.HasPrefix(data, []byte("RRD\x00")) {
+ return newBinError("not an RRD file: wrong magic number", data, 0, 4)
+ }
+ h.Cookie = data[0:4]
+
+ // version string
+ null := bytes.IndexByte(data[4:], 0)
+ if null < 0 {
+ return newBinError("no null-terminator on version string", data, 4, 5)
+ }
+ null += 4
+ h.Version = data[4:null]
+ switch string(h.Version) {
+ case "0003":
+ case "0004":
+ case "0005":
+ default:
+ }
+
+ // float cookie
+ //
+ // Assume IEEE 754 doubles. C doesn't assume 754 doubles, but anything that doesn't use 754 doubles is exotic
+ // enough that I'm OK saying "you're going to need to use `rrdtool dump`". This lets us assume that:
+ // - a 'double' is 8 bytes wide
+
+ // - the value will be exactly equal, and we don't need to worry about weird rounding.
+ h.FloatWidth = 8
+ magicFloat := float64(8.642135e130)
+ floatAddrPacked := null + 1
+ floatAddr32 := ((floatAddrPacked + 3) / 4) * 4
+ floatAddr64 := ((floatAddrPacked + 7) / 8) * 8
+ var restOffset int
+ switch {
+ case len(data) < floatAddr32+h.FloatWidth:
+ return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr32:])) == magicFloat:
+ h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+ h.ByteOrder = binary.LittleEndian
+ h.FloatAlign = 4
+ restOffset = floatAddr32 + h.FloatWidth
+ case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr32:])) == magicFloat:
+ h.FloatCookie = data[floatAddr32 : floatAddr32+h.FloatWidth]
+ h.ByteOrder = binary.BigEndian
+ h.FloatAlign = 4
+ restOffset = floatAddr32 + h.FloatWidth
+ case len(data) < floatAddr64+h.FloatWidth:
+ return newBinError("unexpected end of file", data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ case math.Float64frombits(binary.LittleEndian.Uint64(data[floatAddr64:])) == magicFloat:
+ h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+ h.ByteOrder = binary.LittleEndian
+ h.FloatAlign = 8
+ restOffset = floatAddr64 + h.FloatWidth
+ case math.Float64frombits(binary.BigEndian.Uint64(data[floatAddr64:])) == magicFloat:
+ h.FloatCookie = data[floatAddr64 : floatAddr64+h.FloatWidth]
+ h.ByteOrder = binary.BigEndian
+ h.FloatAlign = 8
+ restOffset = floatAddr64 + h.FloatWidth
+ default:
+ return newBinError("failed to sniff byte-order and float-alignment",
+ data, floatAddrPacked, floatAddr64+h.FloatWidth-floatAddrPacked)
+ }
+
+ switch h.FloatAlign {
+ case 4:
+ // Assume that if floats are only 32-bit aligned, then everything is 32-bit
+ h.IntWidth = 4
+ h.IntAlign = 4
+ case 8:
+ // If floats are 64-bit aligned, then this might be all-in on 64-bit, or it might 32-bit ints.
+
+ // (The following heuristic is borrowed from javascriptRRD, and adjusted to also work with big-endian.)
+ //
+ // The next 2 things after the float_cookie are ds_cnt and rra_cnt (both 'unsigned long'--which may be
+ // either 32 or 64 bit). We'll inspect the bytes a bit to guess how long a long is.
+ //
+ // By assuming
+ // 1. ds_cnt <= math.MaxUint32
+ // 2l. rra_cnt > 0 (relevant if little-endian)
+ // 2b. ds_cnt > 0 (relevant if big-endian)
+ // we can inspect the 4 bytes (marked "big" or "little" below) that are either
+ // - the most significant bits of 64-bit ds_cnt, or
+ // - the entirety of 32-bit rra_cnt (if littlen-endian) or 32-bit ds_cnt (if big-endian)
+ // If we see that those 4 bytes are all 0, then we assume that it's part of a 64-bit ds_cnt.
+ //
+ // | | | | | | | big | little |
+ // |00|01|02|03|04|05|06|07|08|09|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|
+ // 32 | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<--ds_cnt->|<-rra_cnt->|
+ // 64le | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<1111----ds_cnt---0000>|
+ // 64be | R| R| D|\0| 0| 0| 0| 3|\0| |<----doublecookie----->|<0000----ds_cnt---1111>|
+ if len(data) < restOffset+8 {
+ return newBinError("unexpected end of file", data, restOffset, 8)
+ }
+ offset := map[binary.ByteOrder]int{
+ binary.BigEndian: restOffset, // 24 in the above diagram
+ binary.LittleEndian: restOffset + 4, // 28 in the above diagram
+ }[h.ByteOrder]
+ if h.ByteOrder.Uint32(data[offset:]) == 0 {
+ h.IntWidth = 8
+ h.IntAlign = 8
+ } else {
+ h.IntWidth = 4
+ h.IntAlign = 4
+ }
+ }
+
+ return nil
+}
+
+func (h *Header) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+ if err := e.EncodeElement(h.Version, xml.StartElement{Name: xml.Name{Local: "version", Space: XMLNS}}); err != nil {
+ return err
+ }
+ if err := e.EncodeElement(h.PDPStep, xml.StartElement{Name: xml.Name{Local: "step", Space: XMLNS}}); err != nil {
+ return err
+ }
+ return nil
+}
+
+//var _ encoding.BinaryMarshaler = &Header{}
+var _ encoding.BinaryUnmarshaler = &Header{}
+
+var _ xml.Marshaler = &Header{}
+
+//var _ xml.Unmarshaler = &Header{}
diff --git a/rrdformat/format_test.go b/rrdformat/format_test.go
new file mode 100644
index 0000000..e2c1aa0
--- /dev/null
+++ b/rrdformat/format_test.go
@@ -0,0 +1,14 @@
+package rrdformat
+
+import (
+ "encoding/xml"
+ "testing"
+)
+
+func TestXML(t *testing.T) {
+ out, err := xml.Marshal(&Header{
+ Version: []byte("0003"),
+ PDPStep: 300,
+ })
+ t.Log(string(out), err)
+}