diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 6dc262d..1a50b19 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@v4 - name: Build artifacts run: | - make release-assets + make release-assets docker-build-static - name: Upload linux_amd64.tar.gz if: hashFiles('linux_amd64.tar.gz') != '' uses: actions/upload-release-asset@v1 @@ -41,6 +41,14 @@ jobs: asset_path: ./linux_amd64_dbg.tar.gz asset_name: linux_amd64_dbg.tar.gz asset_content_type: application/tar+gzip + - name: Upload linux_amd64_static.tar.gz + if: hashFiles('linux_amd64_static.tar.gz') != '' + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./linux_amd64_static.tar.gz + asset_name: linux_amd64_static.tar.gz + asset_content_type: application/tar+gzip - name: Upload linux_arm64.tar.gz if: hashFiles('linux_arm64.tar.gz') != '' uses: actions/upload-release-asset@v1 diff --git a/Makefile b/Makefile index 648515c..03bcc37 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,8 @@ BUILD_LDFLAGS=-s -w ## Run tests test: test-unit + +# Build statically linked binary in docker. +docker-build-static: + docker run --platform linux/amd64 -v $(PWD):/code -w /code --rm golang:alpine /bin/sh -c 'apk add build-base && GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build --ldflags "-linkmode external -extldflags=-static" -tags cgo_zstd -o ./bin/catp ./cmd/catp && ./bin/catp' + cd ./bin && tar zcvf ../linux_amd64_static.tar.gz * diff --git a/cmd/catp/README.md b/cmd/catp/README.md index 82cf9d0..789fd9a 100644 --- a/cmd/catp/README.md +++ b/cmd/catp/README.md @@ -11,52 +11,53 @@ go install github.com/bool64/progress/cmd/catp@latest or download from [releases](https://github.com/bool64/progress/releases). ``` -wget https://github.com/bool64/progress/releases/latest/download/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz && rm linux_amd64.tar.gz +wget https://github.com/bool64/progress/releases/latest/download/linux_amd64_static.tar.gz && tar xf linux_amd64_static.tar.gz && rm linux_amd64_static.tar.gz ./catp -version ``` ## Usage ``` -catp dev, go1.22rc1 CGO_ZSTD - -catp prints contents of files to STDOUT or dir/file output, -while printing current progress status to STDERR. +catp prints contents of files to STDOUT or dir/file output, +while printing current progress status to STDERR. It can decompress data from .gz and .zst files. Use dash (-) as PATH to read STDIN. Usage of catp: catp [OPTIONS] PATH ... -dbg-cpu-prof string - write first 10 seconds of CPU profile to file + write first 10 seconds of CPU profile to file -dbg-mem-prof string - write heap profile to file after 10 seconds + write heap profile to file after 10 seconds + -l count lines -no-progress - disable progress printing + disable progress printing -out-dir string - output to directory instead of STDOUT - files will be written to out dir with original base names - disables output flag + output to directory instead of STDOUT + files will be written to out dir with original base names + disables output flag -output string - output to file (can have .gz or .zst ext for compression) instead of STDOUT + output to file (can have .gz or .zst ext for compression) instead of STDOUT -parallel int - number of parallel readers if multiple files are provided - lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file) - use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU) (default 1) + number of parallel readers if multiple files are provided + lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file) + use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU) (default 1) -pass value - filter matching, may contain multiple AND patterns separated by ^, - if filter matches, line is passed to the output (unless filtered out by -skip) - each -pass value is added with OR logic, - for example, you can use "-pass bar^baz -pass foo" to only keep lines that have (bar AND baz) OR foo + filter matching, may contain multiple AND patterns separated by ^, + if filter matches, line is passed to the output (unless filtered out by -skip) + each -pass value is added with OR logic, + for example, you can use "-pass bar^baz -pass foo" to only keep lines that have (bar AND baz) OR foo -progress-json string - write current progress to a file + write current progress to a file + -rate-limit float + output rate limit lines per second -skip value - filter matching, may contain multiple AND patterns separated by ^, - if filter matches, line is removed from the output (even if it passed -pass) - each -skip value is added with OR logic, - for example, you can use "-skip quux^baz -skip fooO" to skip lines that have (quux AND baz) OR fooO + filter matching, may contain multiple AND patterns separated by ^, + if filter matches, line is removed from the output (even if it passed -pass) + each -skip value is added with OR logic, + for example, you can use "-skip quux^baz -skip fooO" to skip lines that have (quux AND baz) OR fooO -version - print version and exit + print version and exit ``` ## Examples diff --git a/cmd/catp/catp/app.go b/cmd/catp/catp/app.go index 24589bc..830ffcd 100644 --- a/cmd/catp/catp/app.go +++ b/cmd/catp/catp/app.go @@ -4,6 +4,7 @@ package catp import ( "bufio" "bytes" + "context" "encoding/json" "errors" "flag" @@ -23,6 +24,7 @@ import ( "github.com/bool64/progress" "github.com/klauspost/compress/zstd" gzip "github.com/klauspost/pgzip" + "golang.org/x/time/rate" ) var versionExtra []string @@ -57,7 +59,11 @@ type runner struct { lastStatusTime int64 lastBytesUncompressed int64 + rateLimit float64 + limiter *rate.Limiter + noProgress bool + countLines bool hasOptions bool options Options @@ -198,10 +204,19 @@ func (r *runner) scanFile(filename string, rd io.Reader, out io.Writer) { lines := 0 buf := make([]byte, 64*1024) + linesPush := 1000 + if r.rateLimit < 100 { + linesPush = 1 + } + for s.Scan() { lines++ - if lines >= 1000 { + if r.limiter != nil { + _ = r.limiter.Wait(context.Background()) //nolint:errcheck // No failure condition here. + } + + if lines >= linesPush { atomic.AddInt64(&r.currentLines, int64(lines)) lines = 0 } @@ -390,7 +405,11 @@ func (r *runner) cat(filename string) (err error) { //nolint:gocyclo }) } - if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 || r.hasOptions { + if r.rateLimit > 0 { + r.limiter = rate.NewLimiter(rate.Limit(r.rateLimit), 100) + } + + if len(r.pass) > 0 || len(r.skip) > 0 || r.parallel > 1 || r.hasOptions || r.countLines || r.rateLimit > 0 { r.scanFile(filename, rd, out) } else { r.readFile(rd, out) @@ -502,6 +521,8 @@ func Main(options ...func(o *Options)) error { //nolint:funlen,cyclop,gocognit,g memProfile := flag.String("dbg-mem-prof", "", "write heap profile to file after 10 seconds") output := flag.String("output", "", "output to file (can have .gz or .zst ext for compression) instead of STDOUT") flag.BoolVar(&r.noProgress, "no-progress", false, "disable progress printing") + flag.BoolVar(&r.countLines, "l", false, "count lines") + flag.Float64Var(&r.rateLimit, "rate-limit", 0, "output rate limit lines per second") progressJSON := flag.String("progress-json", "", "write current progress to a file") ver := flag.Bool("version", false, "print version and exit") diff --git a/go.mod b/go.mod index 06fef44..4e5ffaa 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module github.com/bool64/progress -go 1.22 +go 1.23.0 require ( github.com/DataDog/zstd v1.5.7 github.com/bool64/dev v0.2.40 github.com/klauspost/compress v1.18.0 github.com/klauspost/pgzip v1.2.6 + golang.org/x/time v0.12.0 ) diff --git a/go.sum b/go.sum index a79ea7a..a28fd52 100644 --- a/go.sum +++ b/go.sum @@ -6,3 +6,5 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=