refactor: delete package c2h5oh/datasize and create human readable functions

Author: updevru

frac/common/info.go

@@ -6,7 +6,7 @@ import (
 	"path"
 	"time"
 
-	"github.com/c2h5oh/datasize"
+	"github.com/ozontech/seq-db/util"
 
 	"github.com/ozontech/seq-db/buildinfo"
 	"github.com/ozontech/seq-db/config"
@@ -58,8 +58,8 @@ func NewInfo(filename string, docsOnDisk, metaOnDisk uint64) *Info {
 func (s *Info) String() string {
 	return fmt.Sprintf(
 		"raw docs=%s, disk docs=%s",
-		datasize.ByteSize(s.DocsRaw).HR(),
-		datasize.ByteSize(s.DocsOnDisk).HR(),
+		util.ByteToHumanReadable(s.DocsRaw),
+		util.ByteToHumanReadable(s.DocsOnDisk),
 	)
 }
 

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.17
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.13
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5
-	github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee
 	github.com/cep21/circuit/v3 v3.2.2
 	github.com/felixge/fgprof v0.9.5
 	github.com/golang/mock v1.6.0

go.sum

@@ -71,8 +71,6 @@ github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
 github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I=
-github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cactus/go-statsd-client v3.1.1+incompatible/go.mod h1:cMRcwZDklk7hXp+Law83urTHUiHMzCev/r4JMYr/zU0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cep21/circuit/v3 v3.2.2 h1:Z3M1n0+YbEvEvJsjCNV8oYNr2vLhB39drQCdawbS/kk=

storeapi/docs_stream.go

@@ -3,7 +3,7 @@ package storeapi
 import (
 	"context"
 
-	"github.com/c2h5oh/datasize"
+	"github.com/ozontech/seq-db/util"
 	"go.uber.org/zap"
 
 	"github.com/ozontech/seq-db/config"
@@ -107,7 +107,7 @@ func (d *docsStream) calcChunkSize(docs [][]byte, prevChunkSize int) int {
 			zap.Int("total_len", d.totalIDs),
 			zap.Int("prev_chunk_size", prevChunkSize),
 			zap.Int("new_chunk_size", newChunkSize),
-			zap.String("batch_size", datasize.ByteSize(batchSize).HumanReadable()),
+			zap.String("batch_size", util.ByteToHumanReadable(uint64(batchSize))),
 		)
 	}
 

util/bytes.go

@@ -0,0 +1,62 @@
+package util
+
+import (
+	"fmt"
+
+	"github.com/alecthomas/units"
+)
+
+func ByteToHumanReadable(n uint64) string {
+	switch {
+	case n >= uint64(units.EiB):
+		return fmt.Sprintf("%d EiB", n/uint64(units.EiB))
+	case n >= uint64(units.PiB):
+		return fmt.Sprintf("%d PiB", n/uint64(units.PiB))
+	case n >= uint64(units.TiB):
+		return fmt.Sprintf("%d TiB", n/uint64(units.TiB))
+	case n >= uint64(units.GiB):
+		return fmt.Sprintf("%d GiB", n/uint64(units.GiB))
+	case n >= uint64(units.MiB):
+		return fmt.Sprintf("%d MiB", n/uint64(units.MiB))
+	case n >= uint64(units.KiB):
+		return fmt.Sprintf("%d KiB", n/uint64(units.KiB))
+	default:
+		return fmt.Sprintf("%d B", n)
+	}
+}
+
+func ByteToEBytes(b int64) float64 {
+	v := b / int64(units.EiB)
+	r := b % int64(units.EiB)
+	return float64(v) + float64(r)/float64(units.EiB)
+}
+
+func ByteToPBytes(b int64) float64 {
+	v := b / int64(units.PiB)
+	r := b % int64(units.PiB)
+	return float64(v) + float64(r)/float64(units.PiB)
+}
+
+func ByteToTBytes(b int64) float64 {
+	v := b / int64(units.TiB)
+	r := b % int64(units.TiB)
+	return float64(v) + float64(r)/float64(units.TiB)
+}
+
+func ByteToGBytes(b int64) float64 {
+	v := b / int64(units.GiB)
+	r := b % int64(units.GiB)
+	return float64(v) + float64(r)/float64(units.GiB)
+}
+
+func ByteToMBytes(b int64) float64 {
+	v := b / int64(units.MiB)
+	r := b % int64(units.MiB)
+	return float64(v) + float64(r)/float64(units.MiB)
+}
+
+func ByteToKBytes(b int64) float64 {
+	v := b / int64(units.KiB)
+	r := b % int64(units.KiB)
+	return float64(v) + float64(r)/float64(units.KiB)
+}

util/bytes_test.go

@@ -0,0 +1,85 @@
+package util
+
+import (
+	"math"
+	"testing"
+)
+
+// Добавлены проверки границ и больших значений.
+func TestByteToHumanReadable(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    uint64
+		expected string
+	}{
+		{
+			name:     "ZeroBytes",
+			input:    0,
+			expected: "0 B",
+		},
+		{
+			name:     "Bytes",
+			input:    512,
+			expected: "512 B",
+		},
+		{
+			name:     "Exactly1KiB",
+			input:    1024,
+			expected: "1 KiB",
+		},
+		{
+			name:     "JustBelow1MiB",
+			input:    1024*1024 - 1,
+			expected: "1023 KiB",
+		},
+		{
+			name:     "Exactly1MiB",
+			input:    1024 * 1024,
+			expected: "1 MiB",
+		},
+		{
+			name:     "Megabytes",
+			input:    10 * 1024 * 1024,
+			expected: "10 MiB",
+		},
+		{
+			name:     "Gigabytes",
+			input:    15 * 1024 * 1024 * 1024,
+			expected: "15 GiB",
+		},
+		{
+			name:     "Terabytes",
+			input:    5 * 1024 * 1024 * 1024 * 1024,
+			expected: "5 TiB",
+		},
+		{
+			name:     "Petabytes",
+			input:    2 * 1024 * 1024 * 1024 * 1024 * 1024,
+			expected: "2 PiB",
+		},
+		{
+			name:     "Exabytes",
+			input:    1 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
+			expected: "1 EiB",
+		},
+		{
+			name:     "SmallFractionalKB",
+			input:    1025,
+			expected: "1 KiB",
+		},
+		{
+			name:     "VeryLargeValue_NoOverflow",
+			input:    math.MaxUint64,
+			expected: "15 EiB",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := ByteToHumanReadable(tt.input)
+			if result != tt.expected {
+				t.Errorf("ByteToHumanReadable(%d): expected %s, got %s", tt.input, tt.expected, result)
+			}
+		})
+	}
+}

util/log_helpers.go

@@ -8,7 +8,7 @@ import (
 
 // ZapUint64AsSizeStr forms string zap.Field with val converted to size string.
 func ZapUint64AsSizeStr(key string, val uint64) zap.Field {
-	return zap.String(key, SizeStr(val))
+	return zap.String(key, ByteToHumanReadable(val))
 }
 
 // ZapFloat64WithPrec forms float64 zap.Field with removed numerics after prec

util/util.go

@@ -10,7 +10,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/c2h5oh/datasize"
 	"go.uber.org/zap"
 
 	"github.com/ozontech/seq-db/consts"
@@ -27,10 +26,6 @@ func StringToByteUnsafe(str string) []byte { // this works fine
 	return buf
 }
 
-func SizeStr(bytes uint64) string {
-	return datasize.ByteSize(bytes).HR()
-}
-
 func RunEvery(done <-chan struct{}, runInterval time.Duration, actionFn func()) {
 	runTicker := time.NewTicker(runInterval)
 	defer runTicker.Stop()
@@ -73,20 +68,20 @@ func Float64ToPrec(val float64, prec uint32) float64 {
 
 // SizeToUnit converts size in bytes to unit and returns as float64.
 func SizeToUnit(sizeVal uint64, unit string) float64 {
-	val := datasize.ByteSize(sizeVal)
+	val := int64(sizeVal)
 	switch strings.ToLower(unit) {
 	case "kb":
-		return val.KBytes()
+		return ByteToKBytes(val)
 	case "mb":
-		return val.MBytes()
+		return ByteToMBytes(val)
 	case "gb":
-		return val.GBytes()
+		return ByteToGBytes(val)
 	case "tb":
-		return val.TBytes()
+		return ByteToTBytes(val)
 	case "pb":
-		return val.PBytes()
+		return ByteToPBytes(val)
 	case "eb":
-		return val.EBytes()
+		return ByteToEBytes(val)
 	default:
 		logger.Panic("unsupported unit", zap.String("unit", unit))
 		panic("_")