Initial commit

Author: swinkelhofer

.github/workflows/go.yml

@@ -0,0 +1,38 @@
+name: Go
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    
+    - name: Get the version
+      id: release_name
+      run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3)
+      
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+
+    - name: Test
+      run: go test -v ./...
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      with:
+        tag_name: ${{ steps.release_name.outputs.VERSION }}
+        release_name: Release ${{ steps.release_name.outputs.VERSION }}
+        draft: false
+        prerelease: false
+      
+    - name: Push
+      run: GOPROXY=proxy.golang.org go list -m github.com/arivum/msgpack@${{ steps.release_name.outputs.VERSION }}

.gitignore

@@ -0,0 +1,2 @@
+*.pprof
+*.swp

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 arivum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,3 @@
+
+prepend-license:
+	go run github.com/arivum/json2msgpackStreamer/scripts/prepend-license

README.md

@@ -0,0 +1,46 @@
+# msgpack
+
+> :warning: DO NOT USE, STILL WIP!
+
+At the minute, this module contains use-case-specific msgpack decoding improved for maximum performance.
+
+[See package documentation](https://pkg.go.dev/github.com/arivum/msgpack)
+
+## How to use
+
+```go
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/arivum/json2msgpackStreamer"
+	"github.com/arivum/msgpack"
+)
+
+func main() {
+
+	conv := json2msgpackStreamer.NewJSON2MsgPackStreamer(strings.NewReader(
+		`{"a": "testvar", "b": ["b", "c"], "c": null, "d": false, "e": true, "g": -10, "f": -2.1, "h": "this is a longer text that needs a broader length indicator", "i": "sldkfj"}
+{"a": 4, "b": 18000000043000000233, "b2":  -9000000043000000233, "c": null, "d": false, "e": true, "g": -10, "f": 2.0, "h": "this is a longer text that needs a broader length indicator", "i": "sldkfj"}
+{"a": "testvar", "b": ["b", "c"], "c": null}
+
+{"a":"this is larger map test","b":"b","c":"c","d":"d","e":"e","f":"f","g":"g","h":"h","i":"i","j":"j","k":"k","l":"l","m":"m","n":"n","o":"o","p":"p","q": 1}
+
+{"cmplx": {"a":"this is larger map test","b":"b","c":"c","d":"d","e":"e","f":"f","g":"g","h":"h","i":"i","j":"j","k":"k","l":"l","m":"m","n":"n","o":"o","p":"p","q": 1}}
+
+["this is a larger slice test", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"]
+
+{"cmplx": ["2nd", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"]}
+`,
+	))
+
+	d := msgpack.NewDecoder(conv)
+	for entry := range d.Stream() {
+		fmt.Printf("%+v\n", entry)
+	}
+	fmt.Println(d.LastError())
+}
+
+```

accumulated_buffer.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022, arivum.
+ * All rights reserved.
+ * SPDX-License-Identifier: MIT
+ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/MIT
+ */
+
+package msgpack
+
+func newPreallocBuf() *preallocBuf {
+	return &preallocBuf{
+		buf:   make([]byte, 1024),
+		index: 0,
+	}
+}
+
+func (g *preallocBuf) allocateBuffer(len int) []byte {
+	if g.index+len < 1024 {
+		g.index += len
+		return g.buf[g.index-len : g.index]
+	}
+	if len >= 1024 {
+		return make([]byte, len)
+	}
+	g.buf = make([]byte, 1024)
+	g.index = len
+	return g.buf[0:len]
+}

cmd/example/main.go

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2022, arivum.
+ * All rights reserved.
+ * SPDX-License-Identifier: MIT
+ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/MIT
+ */
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func (j *JSONFileInputAdapter) readLoop(w io.WriteCloser, r io.Reader) {
+	var (
+		err   error
+		start time.Time
+	)
+	defer func() {
+		_ = w.Close()
+		//close(j.finished)
+	}()
+
+	start = time.Now()
+	if _, err = io.Copy(w, readerFunc(func(p []byte) (int, error) {
+		var (
+			err error
+			n   int
+		)
+
+		if n, err = r.Read(p); err == nil {
+			j.readBytes += float64(n)
+		}
+		return n, err
+	})); err != nil {
+		logrus.Error(err)
+	}
+	logrus.Infof("processed file %s with an avg of %s", j.Filename, j.avgThroughput(time.Since(start)))
+}
+
+func (j *JSONFileInputAdapter) avgThroughput(duration time.Duration) string {
+	var (
+		throughput = float64(j.filesize) / float64(duration.Seconds())
+		multiplier = 0
+		units      = []string{
+			"B/s",
+			"kB/s",
+			"MB/s",
+			"GB/s",
+		}
+	)
+
+	for throughput/1024 > 1 {
+		throughput /= 1024
+		multiplier++
+	}
+
+	return fmt.Sprintf("%.2f %s", throughput, units[multiplier])
+}
+
+func (j *JSONFileInputAdapter) getMBperSec(resolution time.Duration) float64 {
+
+	var (
+		avg = (j.readBytes - j.oldReadBytes) / (1 << 20) / resolution.Seconds()
+	)
+
+	j.oldReadBytes = j.readBytes
+	return avg
+}
+
+// func (j *JSONFileInputAdapter) printStats() {
+// 	var (
+// 		readBytes  = 0
+// 		resolution = 5 * time.Second
+// 	)
+
+// 	for {
+// 		select {
+// 		case <-j.finished:
+// 			return
+// 		default:
+// 			time.Sleep(resolution)
+// 			logrus.Infof("file %s processed: %.2f%%, avg: %.2fMB/s", j.Filename, j.readBytes*100/float64(j.filesize), (j.readBytes-float64(readBytes))/(1<<20)/resolution.Seconds())
+// 			readBytes = int(j.readBytes)
+// 		}
+// 	}
+// }

cmd/performance/helpers.go

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2022, arivum.
+ * All rights reserved.
+ * SPDX-License-Identifier: MIT
+ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/MIT
+ */
+
+package main
+
+import (
+	"io"
+	"os"
+)
+
+func (j *JSONFileInputAdapter) GetReader() (io.Reader, error) {
+	var (
+		file, err = os.OpenFile(j.Filename, os.O_RDONLY, 0600)
+		r, w      = io.Pipe()
+		fInfo     os.FileInfo
+	)
+
+	if fInfo, err = os.Stat(j.Filename); err != nil {
+		return nil, err
+	}
+
+	j.filesize = fInfo.Size()
+
+	go j.readLoop(w, file)
+	//go j.printStats()
+
+	return r, err
+}