diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 7c56268..c6d02b4 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -21,8 +21,11 @@ jobs: - name: Run Tests via Justfile env: AWS_S3_BUCKET: testbucket + AWS_S3_REGION: us-east-1 + AWS_REGION: us-east-1 + AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: minioadmin AWS_SECRET_ACCESS_KEY: minioadmin AWS_S3_ENDPOINT: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "true" - run: nix develop --command just test-local \ No newline at end of file + run: nix develop --command just test-local diff --git a/Justfile b/Justfile index c9f55c8..58c6887 100644 --- a/Justfile +++ b/Justfile @@ -3,6 +3,10 @@ # Variables MINIO_CONTAINER_NAME := "simples3-minio" MINIO_DATA_DIR := ".minio-data" +MINIO_ENDPOINT := "http://127.0.0.1:9000" +MINIO_REGION := "us-east-1" +MINIO_ACCESS_KEY := "minioadmin" +MINIO_SECRET_KEY := "minioadmin" AWS_S3_BUCKET := "testbucket" # Default command - lists all available recipes @@ -17,7 +21,7 @@ test: test-local: setup @echo "🧪 Running tests with local MinIO..." @sleep 2 - @go test -v ./... + @AWS_S3_BUCKET={{AWS_S3_BUCKET}} AWS_S3_REGION={{MINIO_REGION}} AWS_REGION={{MINIO_REGION}} AWS_DEFAULT_REGION={{MINIO_REGION}} AWS_S3_ACCESS_KEY={{MINIO_ACCESS_KEY}} AWS_S3_SECRET_KEY={{MINIO_SECRET_KEY}} AWS_ACCESS_KEY_ID={{MINIO_ACCESS_KEY}} AWS_SECRET_ACCESS_KEY={{MINIO_SECRET_KEY}} AWS_S3_ENDPOINT={{MINIO_ENDPOINT}} AWS_EC2_METADATA_DISABLED=true SIMPLES3_CLI_INTEGRATION=1 go test -v ./... # --- Go Module Management --- @@ -61,7 +65,11 @@ minio-reset: minio-clean minio-up setup: minio-up @echo "⚙️ Setting up development environment..." @sleep 3 - @aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://{{AWS_S3_BUCKET}} || true + @if command -v aws >/dev/null 2>&1; then \ + AWS_ACCESS_KEY_ID={{MINIO_ACCESS_KEY}} AWS_SECRET_ACCESS_KEY={{MINIO_SECRET_KEY}} AWS_DEFAULT_REGION={{MINIO_REGION}} aws --endpoint-url {{MINIO_ENDPOINT}} s3 mb s3://{{AWS_S3_BUCKET}} || true; \ + else \ + echo "ℹ️ aws CLI not installed; Go tests will bootstrap {{AWS_S3_BUCKET}} automatically"; \ + fi @echo "✅ Development environment ready!" dev-env: setup diff --git a/README.md b/README.md index 5ffb5ae..ba4a8ca 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,180 @@ using AWS Signature Version 4. go get github.com/rhnvrm/simples3 ``` +## CLI + +A first-cut `simples3` CLI is available under `cmd/simples3`. + +### Build + +```sh +go build ./cmd/simples3 +``` + +### Auth and endpoint resolution + +The CLI resolves credentials and region from: +1. explicit flags such as `--region` and `--endpoint` +2. standard AWS environment variables like `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, and `AWS_PROFILE` +3. shared AWS config files (`~/.aws/credentials` and `~/.aws/config`) + +Custom S3-compatible endpoints such as MinIO can be passed with `--endpoint`. + +### Machine contract + +- Default text output is for humans and may change. +- `--json` is the stable automation path. JSON success and error payloads are written to stdout. +- Exit codes: + - `0` success + - `1` runtime failure + - `2` usage or validation failure + - `3` partial failure for bulk commands when `--continue-on-error` is used +- JSON errors use this envelope: + +```json +{ + "command": "cp", + "ok": false, + "error": { + "type": "usage", + "message": "source and destination are required", + "exitCode": 2 + } +} +``` + +Bulk commands (`cp`, `mv`, `rm`, `sync`) return per-operation JSON with a summary: + +```json +{ + "command": "sync", + "ok": true, + "summary": { + "total": 3, + "changed": 1, + "deleted": 1, + "unchanged": 1, + "dryRun": true, + "noop": false + }, + "operations": [ + { + "action": "sync", + "source": "./dist/app.js", + "destination": "s3://my-bucket/site/app.js", + "status": "would-sync" + } + ] +} +``` + +### Commands + +```text +ls list buckets or objects +cp copy local files and S3 objects +rm remove S3 objects and prefixes +mb make bucket +rb remove bucket +mv move local files and S3 objects +presign generate a presigned object URL +sync synchronize source to destination +tags get, set, or delete object tags +versioning get or set bucket versioning +versions list object versions and delete markers +lifecycle get, set, or delete bucket lifecycle configuration +acl get or set bucket or object ACLs +``` + +### Transfer and sync flags + +`cp`, `mv`, `rm`, and `sync` share these automation-oriented flags: + +- `--dry-run` +- `--continue-on-error` +- `--concurrency ` +- `--retries ` +- `--include ` / `--exclude ` where applicable + +`sync --plan` implies `--dry-run` and includes unchanged entries in JSON output. + +`sync --delete` is only allowed for recursive syncs. + +`rm` has a few safety guards: +- `simples3 rm s3://bucket` is refused; use `rb` for buckets +- prefix-like targets require `--recursive` +- `--version-id` is only valid for single-object deletes + +### Metadata and lifecycle inputs + +- `tags set` accepts repeatable `--tag key=value` flags or `--tags-file PATH|-` with a JSON object map. +- `lifecycle set` accepts `--file PATH|-` with lower-camel JSON. +- `acl set` accepts either `--acl ` or `--policy-file PATH|-`. + +Lifecycle document example: + +```json +{ + "rules": [ + { + "id": "expire-logs", + "status": "Enabled", + "filter": { + "prefix": "logs/" + }, + "expiration": { + "days": 30 + } + } + ] +} +``` + +ACL policy example: + +```json +{ + "owner": { + "id": "owner-id", + "displayName": "owner" + }, + "grants": [ + { + "grantee": { + "type": "CanonicalUser", + "id": "owner-id", + "displayName": "owner" + }, + "permission": "FULL_CONTROL" + } + ] +} +``` + +### Examples + +```sh +simples3 ls +simples3 ls s3://my-bucket/prefix/ +simples3 cp ./notes.txt s3://my-bucket/docs/ +simples3 cp --recursive ./public s3://my-bucket/site/ +simples3 cp --json --continue-on-error --concurrency 4 ./batch s3://my-bucket/inbox/ +simples3 cp s3://my-bucket/archive/report.csv ./report.csv +simples3 rm --recursive s3://my-bucket/tmp/ +simples3 rm --json --continue-on-error s3://my-bucket/tmp/ +simples3 mv s3://my-bucket/inbox/file.txt s3://my-bucket/archive/file.txt +simples3 presign --expires 15m s3://my-bucket/path/file.txt +simples3 sync --plan --json ./dist s3://my-bucket/site/ +simples3 sync --delete ./dist s3://my-bucket/site/ +simples3 tags set --tag env=prod --tag team=platform s3://my-bucket/path/file.txt +simples3 tags get --json s3://my-bucket/path/file.txt +simples3 versioning set --status enabled s3://my-bucket +simples3 versions --json s3://my-bucket/path/file.txt +simples3 lifecycle set --file lifecycle.json s3://my-bucket +simples3 acl get --json s3://my-bucket +simples3 acl set --policy-file acl.json s3://my-bucket +``` + ## Quick Start ```go @@ -775,7 +949,7 @@ The library includes comprehensive tests that run against a local MinIO instance # Run all tests (without MinIO) just test -# Run tests with local MinIO +# Run tests with local MinIO (includes CLI integration tests) just test-local # Run specific test @@ -809,7 +983,7 @@ export AWS_S3_BUCKET="testbucket" ## Contributing -Contributions welcome! Check [ROADMAP.md](ROADMAP.md) for planned features. Please add tests and ensure `just test-local` passes before submitting PRs. +Contributions welcome! Check [ROADMAP.md](ROADMAP.md) for planned features. Please add tests and ensure `just test-local` passes before submitting PRs. That MinIO-backed run now includes the `cmd/simples3` CLI integration harness in addition to the library tests. ## Author diff --git a/cmd/simples3/command_acl.go b/cmd/simples3/command_acl.go new file mode 100644 index 0000000..6fae8e1 --- /dev/null +++ b/cmd/simples3/command_acl.go @@ -0,0 +1,148 @@ +package main + +import ( + "fmt" + + "github.com/rhnvrm/simples3" +) + +type aclResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Target string `json:"target"` + Status string `json:"status"` + CannedACL string `json:"cannedAcl,omitempty"` + Policy *accessControlPolicyDocument `json:"policy,omitempty"` +} + +func (rt *runtime) runACL(args []string) error { + if len(args) == 0 { + return usageErrorf("acl requires a subcommand: get or set") + } + switch args[0] { + case "get": + return rt.runACLGet(args[1:]) + case "set": + return rt.runACLSet(args[1:]) + default: + return usageErrorf("unknown acl subcommand %q", args[0]) + } +} + +func (rt *runtime) runACLGet(args []string) error { + var flags awsFlags + var versionID string + fs := newFlagSet("acl get", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 acl get [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] [--version-id ID] s3://bucket[/key]\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&versionID, "version-id", "", "object version ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("acl get requires exactly one bucket or object URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() { + return usageErrorf("acl get target must be an S3 URI like s3://bucket or s3://bucket/key") + } + if versionID != "" && loc.key == "" { + return usageErrorf("--version-id requires an object target") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + client := settings.newClient() + var policy simples3.AccessControlPolicy + if loc.key == "" { + policy, err = client.GetBucketAcl(loc.bucket) + } else { + policy, err = client.GetObjectAcl(simples3.GetObjectAclInput{Bucket: loc.bucket, ObjectKey: loc.key, VersionId: versionID}) + } + if err != nil { + return err + } + doc := aclDocumentFromPolicy(policy) + result := aclResult{Command: "acl", OK: true, Target: loc.String(), Status: "loaded", Policy: &doc} + if flags.json { + return writeJSON(rt.stdout, result) + } + return writeJSON(rt.stdout, doc) +} + +func (rt *runtime) runACLSet(args []string) error { + var flags awsFlags + var cannedACL string + var policyFile string + var versionID string + fs := newFlagSet("acl set", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 acl set [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] [--version-id ID] (--acl VALUE | --policy-file PATH|-) s3://bucket[/key]\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&cannedACL, "acl", "", "canned ACL value") + fs.StringVar(&policyFile, "policy-file", "", "JSON file or - for stdin with ACL policy") + fs.StringVar(&versionID, "version-id", "", "object version ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("acl set requires exactly one bucket or object URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() { + return usageErrorf("acl set target must be an S3 URI like s3://bucket or s3://bucket/key") + } + if versionID != "" && loc.key == "" { + return usageErrorf("--version-id requires an object target") + } + if (cannedACL == "") == (policyFile == "") { + return usageErrorf("acl set requires exactly one of --acl or --policy-file") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + client := settings.newClient() + result := aclResult{Command: "acl", OK: true, Target: loc.String(), Status: "updated", CannedACL: cannedACL} + if policyFile != "" { + var doc accessControlPolicyDocument + if err := decodeJSONSource(rt, policyFile, &doc); err != nil { + return err + } + result.Policy = &doc + if loc.key == "" { + err = client.PutBucketAcl(simples3.PutBucketAclInput{Bucket: loc.bucket, AccessControlPolicy: doc.toPolicy()}) + } else { + err = client.PutObjectAcl(simples3.PutObjectAclInput{Bucket: loc.bucket, ObjectKey: loc.key, VersionId: versionID, AccessControlPolicy: doc.toPolicy()}) + } + if err != nil { + return err + } + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "updated acl %s", loc.String()) + return nil + } + if loc.key == "" { + err = client.PutBucketAcl(simples3.PutBucketAclInput{Bucket: loc.bucket, CannedACL: cannedACL}) + } else { + err = client.PutObjectAcl(simples3.PutObjectAclInput{Bucket: loc.bucket, ObjectKey: loc.key, VersionId: versionID, CannedACL: cannedACL}) + } + if err != nil { + return err + } + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "updated acl %s", loc.String()) + return nil +} diff --git a/cmd/simples3/command_bucket.go b/cmd/simples3/command_bucket.go new file mode 100644 index 0000000..d0fd6df --- /dev/null +++ b/cmd/simples3/command_bucket.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + + "github.com/rhnvrm/simples3" +) + +type bucketResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Bucket string `json:"bucket"` + Location string `json:"location,omitempty"` + Status string `json:"status"` +} + +func (rt *runtime) runMakeBucket(args []string) error { + var flags awsFlags + fs := newFlagSet("mb", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 mb [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if len(fs.Args()) != 1 { + fs.Usage() + return usageErrorf("mb requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Args()[0]) + if err != nil { + return err + } + if !loc.isS3() || loc.bucket == "" || loc.key != "" { + return usageErrorf("mb target must be a bucket URI like s3://my-bucket") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + output, err := settings.newClient().CreateBucket(simples3.CreateBucketInput{Bucket: loc.bucket, Region: settings.Region}) + if err != nil { + return err + } + result := bucketResult{Command: "mb", OK: true, Bucket: loc.bucket, Location: output.Location, Status: "created"} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "created %s", loc.String()) + return nil +} + +func (rt *runtime) runRemoveBucket(args []string) error { + var flags awsFlags + fs := newFlagSet("rb", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 rb [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if len(fs.Args()) != 1 { + fs.Usage() + return usageErrorf("rb requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Args()[0]) + if err != nil { + return err + } + if !loc.isS3() || loc.bucket == "" || loc.key != "" { + return usageErrorf("rb target must be a bucket URI like s3://my-bucket") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + if err := settings.newClient().DeleteBucket(simples3.DeleteBucketInput{Bucket: loc.bucket}); err != nil { + return err + } + result := bucketResult{Command: "rb", OK: true, Bucket: loc.bucket, Status: "deleted"} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "deleted %s", loc.String()) + return nil +} diff --git a/cmd/simples3/command_cp.go b/cmd/simples3/command_cp.go new file mode 100644 index 0000000..557fd77 --- /dev/null +++ b/cmd/simples3/command_cp.go @@ -0,0 +1,154 @@ +package main + +import ( + "fmt" +) + +type copyFlags struct { + awsFlags + recursive bool + dryRun bool + continueOnError bool + concurrency int + retries int + includes stringListFlag + excludes stringListFlag + acl string + sse string + sseKMS string + versionID string +} + +func (rt *runtime) runCopy(args []string) error { + flags := copyFlags{} + fs := newFlagSet("cp", func() { + fmt.Fprint(rt.stderr, `Usage: simples3 cp [flags] + +Copy a local file or S3 object between local paths and s3:// locations. + +Flags: + --recursive copy directories or S3 prefixes recursively + --dry-run print planned operations without executing them + --include include glob pattern (repeatable) + --exclude exclude glob pattern (repeatable) + --continue-on-error keep processing remaining operations after failures + --concurrency number of operations to run in parallel (default 1) + --retries retry transient failures this many times (default 2) + --acl canned ACL for uploads to S3 + --sse server-side encryption mode for S3 destinations + --sse-kms-key-id KMS key ID when using aws:kms + --version-id source object version for exact S3 downloads + --profile AWS profile name + --region AWS region + --endpoint custom S3 endpoint URL + --json emit JSON output +`) + }) + addAWSFlags(fs, &flags.awsFlags) + fs.BoolVar(&flags.recursive, "recursive", false, "copy recursively") + fs.BoolVar(&flags.dryRun, "dry-run", false, "print operations without executing") + fs.Var(&flags.includes, "include", "include glob pattern") + fs.Var(&flags.excludes, "exclude", "exclude glob pattern") + fs.BoolVar(&flags.continueOnError, "continue-on-error", false, "continue processing after failures") + fs.IntVar(&flags.concurrency, "concurrency", 1, "number of parallel operations") + fs.IntVar(&flags.retries, "retries", 2, "number of retries for transient failures") + fs.StringVar(&flags.acl, "acl", "", "canned ACL for uploads") + fs.StringVar(&flags.sse, "sse", "", "server-side encryption mode") + fs.StringVar(&flags.sseKMS, "sse-kms-key-id", "", "KMS key ID") + fs.StringVar(&flags.versionID, "version-id", "", "source object version ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 2 { + return usageErrorf("cp requires a source and destination") + } + flags.sse = normalizeSSE(flags.sse, flags.sseKMS) + + source, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + destination, err := parseLocation(fs.Arg(1)) + if err != nil { + return err + } + if source.isLocal() && destination.isLocal() { + return usageErrorf("local-to-local copies are not supported") + } + if destination.isLocal() && (flags.acl != "" || flags.sse != "" || flags.sseKMS != "") { + return usageErrorf("--acl/--sse flags require an S3 destination") + } + if err := validateRecursiveSource(source, flags.recursive); err != nil { + return err + } + if flags.versionID != "" { + if source.isLocal() { + return usageErrorf("--version-id requires an S3 source") + } + if flags.recursive { + return usageErrorf("--version-id is only supported for single-object copies") + } + if destination.isS3() { + return usageErrorf("--version-id is currently only supported for downloads to local paths") + } + } + + if flags.concurrency < 1 { + return usageErrorf("--concurrency must be at least 1") + } + if flags.retries < 0 { + return usageErrorf("--retries cannot be negative") + } + + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + matcher, err := newMatcher([]string(flags.includes), []string(flags.excludes)) + if err != nil { + return err + } + client := settings.newClient() + entries, err := collectSourceEntries(client, source, flags.recursive, matcher, flags.versionID) + if err != nil { + return err + } + if err := ensureSingleSourceEntry(source, flags.recursive, entries); err != nil { + return err + } + + ops := make([]plannedOperation, 0, len(entries)) + options := copyOptions{ + executionOptions: executionOptions{DryRun: flags.dryRun, Concurrency: flags.concurrency, Retries: flags.retries, ContinueOnError: flags.continueOnError}, + EmitProgress: !flags.json, + ACL: flags.acl, + SSE: flags.sse, + SSEKMS: flags.sseKMS, + } + for _, entry := range entries { + target, err := resolveTarget(entry, destination, flags.recursive) + if err != nil { + return err + } + result := operationResult{Action: "copy", Source: entrySourceString(entry), Destination: targetString(target), Status: "copied", Size: entry.Size, DryRun: flags.dryRun} + if flags.dryRun { + result.Status = "would-copy" + } + entryCopy := entry + targetCopy := target + ops = append(ops, plannedOperation{result: result, run: func() error { + return copyEntry(client, entryCopy, targetCopy, options, rt) + }}) + } + results := executePlannedOperations(ops, options.executionOptions) + if !flags.json { + printOperations(rt.stdout, results) + } + if err := operationsPartialFailure("cp", results); err != nil { + return err + } + if flags.json { + return writeOperationsJSON(rt.stdout, "cp", results) + } + return nil +} diff --git a/cmd/simples3/command_lifecycle.go b/cmd/simples3/command_lifecycle.go new file mode 100644 index 0000000..a595df2 --- /dev/null +++ b/cmd/simples3/command_lifecycle.go @@ -0,0 +1,146 @@ +package main + +import ( + "fmt" + + "github.com/rhnvrm/simples3" +) + +type lifecycleResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Bucket string `json:"bucket"` + Status string `json:"status"` + Configuration *lifecycleConfigurationDocument `json:"configuration,omitempty"` +} + +func (rt *runtime) runLifecycle(args []string) error { + if len(args) == 0 { + return usageErrorf("lifecycle requires a subcommand: get, set, or delete") + } + switch args[0] { + case "get": + return rt.runLifecycleGet(args[1:]) + case "set": + return rt.runLifecycleSet(args[1:]) + case "delete", "rm": + return rt.runLifecycleDelete(args[1:]) + default: + return usageErrorf("unknown lifecycle subcommand %q", args[0]) + } +} + +func (rt *runtime) runLifecycleGet(args []string) error { + var flags awsFlags + fs := newFlagSet("lifecycle get", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 lifecycle get [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("lifecycle get requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key != "" { + return usageErrorf("lifecycle get target must be a bucket URI like s3://my-bucket") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + config, err := settings.newClient().GetBucketLifecycle(loc.bucket) + if err != nil { + return err + } + doc := lifecycleDocumentFromConfiguration(config) + result := lifecycleResult{Command: "lifecycle", OK: true, Bucket: loc.bucket, Status: "loaded", Configuration: &doc} + if flags.json { + return writeJSON(rt.stdout, result) + } + return writeJSON(rt.stdout, doc) +} + +func (rt *runtime) runLifecycleSet(args []string) error { + var flags awsFlags + var file string + fs := newFlagSet("lifecycle set", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 lifecycle set [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] --file PATH|- s3://bucket\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&file, "file", "", "JSON file or - for stdin with lifecycle configuration") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("lifecycle set requires exactly one bucket URI") + } + if file == "" { + return usageErrorf("lifecycle set requires --file") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key != "" { + return usageErrorf("lifecycle set target must be a bucket URI like s3://my-bucket") + } + var doc lifecycleConfigurationDocument + if err := decodeJSONSource(rt, file, &doc); err != nil { + return err + } + if len(doc.Rules) == 0 { + return usageErrorf("lifecycle set requires at least one rule") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + if err := settings.newClient().PutBucketLifecycle(simples3.PutBucketLifecycleInput{Bucket: loc.bucket, Configuration: doc.toConfiguration()}); err != nil { + return err + } + result := lifecycleResult{Command: "lifecycle", OK: true, Bucket: loc.bucket, Status: "updated", Configuration: &doc} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "updated lifecycle %s", loc.String()) + return nil +} + +func (rt *runtime) runLifecycleDelete(args []string) error { + var flags awsFlags + fs := newFlagSet("lifecycle delete", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 lifecycle delete [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("lifecycle delete requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key != "" { + return usageErrorf("lifecycle delete target must be a bucket URI like s3://my-bucket") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + if err := settings.newClient().DeleteBucketLifecycle(simples3.DeleteBucketInput{Bucket: loc.bucket}); err != nil { + return err + } + result := lifecycleResult{Command: "lifecycle", OK: true, Bucket: loc.bucket, Status: "deleted"} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "deleted lifecycle %s", loc.String()) + return nil +} diff --git a/cmd/simples3/command_ls.go b/cmd/simples3/command_ls.go new file mode 100644 index 0000000..f727741 --- /dev/null +++ b/cmd/simples3/command_ls.go @@ -0,0 +1,120 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/rhnvrm/simples3" +) + +type listSummary struct { + BucketCount int `json:"bucketCount,omitempty"` + ObjectCount int `json:"objectCount,omitempty"` + CommonPrefixCount int `json:"commonPrefixCount,omitempty"` +} + +type listJSONOutput struct { + Command string `json:"command"` + OK bool `json:"ok"` + Target string `json:"target,omitempty"` + Summary listSummary `json:"summary"` + Buckets []simples3.Bucket `json:"buckets,omitempty"` + Prefix string `json:"prefix,omitempty"` + Objects []objectEntry `json:"objects,omitempty"` + Common []string `json:"commonPrefixes,omitempty"` +} + +type objectEntry struct { + Key string `json:"key"` + Size int64 `json:"size"` + LastModified string `json:"lastModified"` + StorageClass string `json:"storageClass,omitempty"` +} + +func (rt *runtime) runList(args []string) error { + var flags awsFlags + var recursive bool + fs := newFlagSet("ls", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 ls [--profile PROFILE] [--region REGION] [--endpoint URL] [--recursive] [--json] [s3://bucket[/prefix]]\n") + }) + addAWSFlags(fs, &flags) + fs.BoolVar(&recursive, "recursive", false, "list recursively") + if err := parseFlagSet(fs, args); err != nil { + return err + } + remaining := fs.Args() + if len(remaining) > 1 { + fs.Usage() + return usageErrorf("ls accepts at most one target") + } + + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + client := settings.newClient() + + if len(remaining) == 0 { + result, err := client.ListBuckets(simples3.ListBucketsInput{}) + if err != nil { + return err + } + if flags.json { + return writeJSON(rt.stdout, listJSONOutput{Command: "ls", OK: true, Summary: listSummary{BucketCount: len(result.Buckets)}, Buckets: result.Buckets}) + } + for _, bucket := range result.Buckets { + printLine(rt.stdout, "%s\t%s", bucket.CreationDate.Format(time.RFC3339), bucket.Name) + } + return nil + } + + loc, err := parseLocation(remaining[0]) + if err != nil { + return err + } + if !loc.isS3() { + return usageErrorf("ls target must be an s3:// URI") + } + + if recursive { + seq, finish := client.ListAll(simples3.ListInput{Bucket: loc.bucket, Prefix: loc.s3Prefix()}) + output := listJSONOutput{Command: "ls", OK: true, Target: loc.String(), Prefix: loc.s3Prefix()} + for object := range seq { + entry := objectEntry{Key: object.Key, Size: object.Size, LastModified: object.LastModified, StorageClass: object.StorageClass} + if flags.json { + output.Objects = append(output.Objects, entry) + } else { + printLine(rt.stdout, "%12d %s", entry.Size, entry.Key) + } + } + if err := finish(); err != nil { + return err + } + if flags.json { + output.Summary = listSummary{ObjectCount: len(output.Objects)} + return writeJSON(rt.stdout, output) + } + return nil + } + + result, err := client.List(simples3.ListInput{Bucket: loc.bucket, Prefix: loc.s3Prefix(), Delimiter: "/"}) + if err != nil { + return err + } + output := listJSONOutput{Command: "ls", OK: true, Target: loc.String(), Prefix: loc.s3Prefix(), Common: result.CommonPrefixes} + for _, object := range result.Objects { + output.Objects = append(output.Objects, objectEntry{Key: object.Key, Size: object.Size, LastModified: object.LastModified, StorageClass: object.StorageClass}) + } + if flags.json { + output.Summary = listSummary{ObjectCount: len(output.Objects), CommonPrefixCount: len(output.Common)} + return writeJSON(rt.stdout, output) + } + for _, prefix := range output.Common { + printLine(rt.stdout, "DIR\t%s", strings.TrimSuffix(prefix, "/")+"/") + } + for _, object := range output.Objects { + printLine(rt.stdout, "%12d %s", object.Size, object.Key) + } + return nil +} diff --git a/cmd/simples3/command_mv.go b/cmd/simples3/command_mv.go new file mode 100644 index 0000000..5c7ac41 --- /dev/null +++ b/cmd/simples3/command_mv.go @@ -0,0 +1,134 @@ +package main + +import "fmt" + +type moveFlags struct { + copyFlags +} + +func (rt *runtime) runMove(args []string) error { + flags := moveFlags{} + fs := newFlagSet("mv", func() { + fmt.Fprint(rt.stderr, `Usage: simples3 mv [flags] + +Move a local file or S3 object between local paths and s3:// locations. + +Flags: + --recursive move directories or S3 prefixes recursively + --dry-run print planned operations without executing them + --include include glob pattern (repeatable) + --exclude exclude glob pattern (repeatable) + --continue-on-error keep processing remaining operations after failures + --concurrency number of operations to run in parallel (default 1) + --retries retry transient failures this many times (default 2) + --acl canned ACL for uploads to S3 + --sse server-side encryption mode for S3 destinations + --sse-kms-key-id KMS key ID when using aws:kms + --profile AWS profile name + --region AWS region + --endpoint custom S3 endpoint URL + --json emit JSON output +`) + }) + addAWSFlags(fs, &flags.awsFlags) + fs.BoolVar(&flags.recursive, "recursive", false, "move recursively") + fs.BoolVar(&flags.dryRun, "dry-run", false, "print operations without executing") + fs.Var(&flags.includes, "include", "include glob pattern") + fs.Var(&flags.excludes, "exclude", "exclude glob pattern") + fs.BoolVar(&flags.continueOnError, "continue-on-error", false, "continue processing after failures") + fs.IntVar(&flags.concurrency, "concurrency", 1, "number of parallel operations") + fs.IntVar(&flags.retries, "retries", 2, "number of retries for transient failures") + fs.StringVar(&flags.acl, "acl", "", "canned ACL for uploads") + fs.StringVar(&flags.sse, "sse", "", "server-side encryption mode") + fs.StringVar(&flags.sseKMS, "sse-kms-key-id", "", "KMS key ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 2 { + return usageErrorf("mv requires a source and destination") + } + flags.sse = normalizeSSE(flags.sse, flags.sseKMS) + + source, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + destination, err := parseLocation(fs.Arg(1)) + if err != nil { + return err + } + if source.isLocal() && destination.isLocal() { + return usageErrorf("local-to-local moves are not supported") + } + if source.String() == destination.String() { + return usageErrorf("source and destination are the same") + } + if destination.isLocal() && (flags.acl != "" || flags.sse != "" || flags.sseKMS != "") { + return usageErrorf("--acl/--sse flags require an S3 destination") + } + if err := validateRecursiveSource(source, flags.recursive); err != nil { + return err + } + + if flags.concurrency < 1 { + return usageErrorf("--concurrency must be at least 1") + } + if flags.retries < 0 { + return usageErrorf("--retries cannot be negative") + } + + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + matcher, err := newMatcher([]string(flags.includes), []string(flags.excludes)) + if err != nil { + return err + } + client := settings.newClient() + entries, err := collectSourceEntries(client, source, flags.recursive, matcher, "") + if err != nil { + return err + } + if err := ensureSingleSourceEntry(source, flags.recursive, entries); err != nil { + return err + } + + ops := make([]plannedOperation, 0, len(entries)) + options := copyOptions{ + executionOptions: executionOptions{DryRun: flags.dryRun, Concurrency: flags.concurrency, Retries: flags.retries, ContinueOnError: flags.continueOnError}, + EmitProgress: !flags.json, + ACL: flags.acl, + SSE: flags.sse, + SSEKMS: flags.sseKMS, + } + for _, entry := range entries { + target, err := resolveTarget(entry, destination, flags.recursive) + if err != nil { + return err + } + result := operationResult{Action: "move", Source: entrySourceString(entry), Destination: targetString(target), Status: "moved", Size: entry.Size, DryRun: flags.dryRun} + if flags.dryRun { + result.Status = "would-move" + } + entryCopy := entry + targetCopy := target + ops = append(ops, plannedOperation{result: result, run: func() error { + if err := copyEntry(client, entryCopy, targetCopy, options, rt); err != nil { + return err + } + return deleteSourceEntry(client, entryCopy, source) + }}) + } + results := executePlannedOperations(ops, options.executionOptions) + if !flags.json { + printOperations(rt.stdout, results) + } + if err := operationsPartialFailure("mv", results); err != nil { + return err + } + if flags.json { + return writeOperationsJSON(rt.stdout, "mv", results) + } + return nil +} diff --git a/cmd/simples3/command_presign.go b/cmd/simples3/command_presign.go new file mode 100644 index 0000000..0b75c11 --- /dev/null +++ b/cmd/simples3/command_presign.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/rhnvrm/simples3" +) + +type presignResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Target string `json:"target"` + Method string `json:"method"` + Expires string `json:"expires"` + URL string `json:"url"` +} + +func (rt *runtime) runPresign(args []string) error { + var flags awsFlags + var method string + var expires string + var disposition string + fs := newFlagSet("presign", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 presign [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] [--method GET|PUT] [--expires 1h] [--response-content-disposition VALUE] s3://bucket/key\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&method, "method", "GET", "HTTP method for the presigned URL (GET or PUT)") + fs.StringVar(&expires, "expires", "1h", "URL expiry duration (for example 15m, 1h)") + fs.StringVar(&disposition, "response-content-disposition", "", "optional response-content-disposition for GET URLs") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if len(fs.Args()) != 1 { + fs.Usage() + return usageErrorf("presign requires exactly one object URI") + } + loc, err := parseLocation(fs.Args()[0]) + if err != nil { + return err + } + if !loc.isS3() || loc.key == "" { + return usageErrorf("presign target must be an object URI like s3://bucket/key") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + method = strings.ToUpper(method) + if method != "GET" && method != "PUT" { + return usageErrorf("unsupported presign method %q", method) + } + duration, err := time.ParseDuration(expires) + if err != nil { + return usageErrorf("invalid --expires value %q: %v", expires, err) + } + seconds := int(duration.Seconds()) + if seconds <= 0 { + return usageErrorf("expires must be greater than zero") + } + url := settings.newClient().GeneratePresignedURL(simples3.PresignedInput{Bucket: loc.bucket, ObjectKey: loc.key, Method: method, ExpirySeconds: seconds, ResponseContentDisposition: disposition}) + if url == "" { + return fmt.Errorf("failed to generate presigned URL") + } + result := presignResult{Command: "presign", OK: true, Target: loc.String(), Method: method, Expires: duration.String(), URL: url} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "%s", url) + return nil +} diff --git a/cmd/simples3/command_rm.go b/cmd/simples3/command_rm.go new file mode 100644 index 0000000..719fdb4 --- /dev/null +++ b/cmd/simples3/command_rm.go @@ -0,0 +1,118 @@ +package main + +import "fmt" + +type removeFlags struct { + awsFlags + recursive bool + dryRun bool + continueOnError bool + concurrency int + retries int + includes stringListFlag + excludes stringListFlag + versionID string +} + +func (rt *runtime) runRemove(args []string) error { + flags := removeFlags{} + fs := newFlagSet("rm", func() { + fmt.Fprint(rt.stderr, `Usage: simples3 rm [flags] + +Remove an S3 object or prefix. + +Flags: + --recursive remove prefixes recursively + --dry-run print planned deletions without executing them + --continue-on-error keep processing remaining deletions after failures + --concurrency number of deletions to run in parallel (default 1) + --retries retry transient failures this many times (default 2) + --include include glob pattern for recursive deletes (repeatable) + --exclude exclude glob pattern for recursive deletes (repeatable) + --version-id delete a specific object version + --profile AWS profile name + --region AWS region + --endpoint custom S3 endpoint URL + --json emit JSON output +`) + }) + addAWSFlags(fs, &flags.awsFlags) + fs.BoolVar(&flags.recursive, "recursive", false, "remove recursively") + fs.BoolVar(&flags.dryRun, "dry-run", false, "print deletions without executing") + fs.BoolVar(&flags.continueOnError, "continue-on-error", false, "continue processing after failures") + fs.IntVar(&flags.concurrency, "concurrency", 1, "number of parallel deletions") + fs.IntVar(&flags.retries, "retries", 2, "number of retries for transient failures") + fs.Var(&flags.includes, "include", "include glob pattern") + fs.Var(&flags.excludes, "exclude", "exclude glob pattern") + fs.StringVar(&flags.versionID, "version-id", "", "object version ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("rm requires exactly one S3 path") + } + + target, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if target.isLocal() { + return usageErrorf("rm only supports s3:// locations") + } + if !flags.recursive && target.key == "" { + return usageErrorf("refusing to remove bucket root; use rb for buckets or --recursive for object prefixes") + } + if flags.versionID != "" && flags.recursive { + return usageErrorf("--version-id is only supported for single-object deletes") + } + if !flags.recursive && target.hasTrailing { + return usageErrorf("%s looks like a prefix; use --recursive", target.String()) + } + + if flags.concurrency < 1 { + return usageErrorf("--concurrency must be at least 1") + } + if flags.retries < 0 { + return usageErrorf("--retries cannot be negative") + } + + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + matcher, err := newMatcher([]string(flags.includes), []string(flags.excludes)) + if err != nil { + return err + } + client := settings.newClient() + + execution := executionOptions{DryRun: flags.dryRun, Concurrency: flags.concurrency, Retries: flags.retries, ContinueOnError: flags.continueOnError} + results := []operationResult{} + if flags.recursive { + entries, err := collectSourceEntries(client, target, true, matcher, "") + if err != nil { + return err + } + results = executeDeleteBatches(client, entries, target.bucket, execution) + } else { + entry := sourceEntry{Kind: locationKindS3, Bucket: target.bucket, Key: target.key, Version: flags.versionID, Relative: target.key} + ops := []plannedOperation{{ + result: buildDeleteResult(entry, flags.dryRun), + run: func() error { + return deleteSourceEntry(client, entry, target) + }, + }} + results = executePlannedOperations(ops, execution) + } + + if !flags.json { + printOperations(rt.stdout, results) + } + if err := operationsPartialFailure("rm", results); err != nil { + return err + } + if flags.json { + return writeOperationsJSON(rt.stdout, "rm", results) + } + return nil +} diff --git a/cmd/simples3/command_sync.go b/cmd/simples3/command_sync.go new file mode 100644 index 0000000..caf0e51 --- /dev/null +++ b/cmd/simples3/command_sync.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "os" +) + +type syncFlags struct { + awsFlags + recursive bool + dryRun bool + plan bool + delete bool + continueOnError bool + concurrency int + retries int + includes stringListFlag + excludes stringListFlag + acl string + sse string + sseKMS string +} + +func (rt *runtime) runSync(args []string) error { + flags := syncFlags{} + fs := newFlagSet("sync", func() { + fmt.Fprint(rt.stderr, `Usage: simples3 sync [flags] + +Synchronize source to destination. This is a one-way copy from source to destination. + +Flags: + --recursive recurse into directories or S3 prefixes + --dry-run print planned operations without executing them + --plan emit a full sync plan, including unchanged entries (implies --dry-run) + --delete delete destination files missing from source + --include include glob pattern (repeatable) + --exclude exclude glob pattern (repeatable) + --continue-on-error keep processing remaining operations after failures + --concurrency number of operations to run in parallel (default 1) + --retries retry transient failures this many times (default 2) + --acl canned ACL for uploads to S3 + --sse server-side encryption mode for S3 destinations + --sse-kms-key-id KMS key ID when using aws:kms + --profile AWS profile name + --region AWS region + --endpoint custom S3 endpoint URL + --json emit JSON output +`) + }) + addAWSFlags(fs, &flags.awsFlags) + fs.BoolVar(&flags.recursive, "recursive", false, "sync recursively") + fs.BoolVar(&flags.dryRun, "dry-run", false, "print operations without executing") + fs.BoolVar(&flags.plan, "plan", false, "show a full sync plan including unchanged entries") + fs.BoolVar(&flags.delete, "delete", false, "delete destination entries missing from source") + fs.Var(&flags.includes, "include", "include glob pattern") + fs.Var(&flags.excludes, "exclude", "exclude glob pattern") + fs.BoolVar(&flags.continueOnError, "continue-on-error", false, "continue processing after failures") + fs.IntVar(&flags.concurrency, "concurrency", 1, "number of parallel operations") + fs.IntVar(&flags.retries, "retries", 2, "number of retries for transient failures") + fs.StringVar(&flags.acl, "acl", "", "canned ACL for uploads") + fs.StringVar(&flags.sse, "sse", "", "server-side encryption mode") + fs.StringVar(&flags.sseKMS, "sse-kms-key-id", "", "KMS key ID") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 2 { + return usageErrorf("sync requires a source and destination") + } + flags.sse = normalizeSSE(flags.sse, flags.sseKMS) + flags.dryRun = flags.dryRun || flags.plan + + source, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + destination, err := parseLocation(fs.Arg(1)) + if err != nil { + return err + } + if source.isLocal() && destination.isLocal() { + return usageErrorf("local-to-local sync is not supported") + } + if destination.isLocal() && (flags.acl != "" || flags.sse != "" || flags.sseKMS != "") { + return usageErrorf("--acl/--sse flags require an S3 destination") + } + + if flags.concurrency < 1 { + return usageErrorf("--concurrency must be at least 1") + } + if flags.retries < 0 { + return usageErrorf("--retries cannot be negative") + } + + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + matcher, err := newMatcher([]string(flags.includes), []string(flags.excludes)) + if err != nil { + return err + } + client := settings.newClient() + recursive, err := inferSyncRecursion(source, flags.recursive) + if err != nil { + return err + } + if flags.delete && !recursive { + return usageErrorf("--delete is only supported for recursive syncs") + } + + entries, err := collectSourceEntries(client, source, recursive, matcher, "") + if err != nil { + return err + } + + results := []operationResult{} + seen := map[string]struct{}{} + options := copyOptions{ + executionOptions: executionOptions{DryRun: flags.dryRun, Concurrency: flags.concurrency, Retries: flags.retries, ContinueOnError: flags.continueOnError}, + EmitProgress: !flags.json, + ACL: flags.acl, + SSE: flags.sse, + SSEKMS: flags.sseKMS, + } + sourceOps := make([]plannedOperation, 0, len(entries)) + for _, entry := range entries { + seen[entry.Relative] = struct{}{} + target, err := resolveTarget(entry, destination, recursive) + if err != nil { + return err + } + meta, err := fetchTargetMeta(client, target) + if err != nil { + return err + } + if !needsSync(entry, target, meta) { + if flags.plan { + sourceOps = append(sourceOps, plannedOperation{result: operationResult{Action: "sync", Source: entrySourceString(entry), Destination: targetString(target), Status: "unchanged", Size: entry.Size, DryRun: flags.dryRun}}) + } + continue + } + result := operationResult{Action: "sync", Source: entrySourceString(entry), Destination: targetString(target), Status: "synced", Size: entry.Size, DryRun: flags.dryRun} + if flags.dryRun { + result.Status = "would-sync" + } + entryCopy := entry + targetCopy := target + sourceOps = append(sourceOps, plannedOperation{result: result, run: func() error { + return copyEntry(client, entryCopy, targetCopy, options, rt) + }}) + } + results = append(results, executePlannedOperations(sourceOps, options.executionOptions)...) + + if flags.delete { + targetEntries, err := collectTargetEntries(client, destination, true, matcher) + if err != nil { + return err + } + deleteOps := make([]plannedOperation, 0, len(targetEntries)) + for _, targetEntry := range targetEntries { + if _, ok := seen[targetEntry.Relative]; ok { + continue + } + targetRef := entryAsTarget(targetEntry) + result := operationResult{Action: "delete", Source: targetString(targetRef), Status: "deleted", Size: targetEntry.Size, DryRun: flags.dryRun} + if flags.dryRun { + result.Status = "would-delete" + } + targetCopy := targetRef + deleteOps = append(deleteOps, plannedOperation{result: result, run: func() error { + return deleteTargetRef(client, targetCopy) + }}) + } + results = append(results, executePlannedOperations(deleteOps, options.executionOptions)...) + } + + if !flags.json { + printOperations(rt.stdout, results) + } + if err := operationsPartialFailure("sync", results); err != nil { + return err + } + if flags.json { + return writeOperationsJSON(rt.stdout, "sync", results) + } + return nil +} + +func inferSyncRecursion(source location, requested bool) (bool, error) { + if source.isS3() { + return requested || source.key == "" || source.hasTrailing, nil + } + info, err := os.Stat(source.path) + if err != nil { + return false, err + } + return requested || info.IsDir(), nil +} + +func entryAsTarget(entry sourceEntry) targetRef { + if entry.Kind == locationKindLocal { + return targetRef{Kind: locationKindLocal, Local: entry.Local} + } + return targetRef{Kind: locationKindS3, Bucket: entry.Bucket, Key: entry.Key} +} diff --git a/cmd/simples3/command_tags.go b/cmd/simples3/command_tags.go new file mode 100644 index 0000000..1624127 --- /dev/null +++ b/cmd/simples3/command_tags.go @@ -0,0 +1,158 @@ +package main + +import ( + "fmt" + + "github.com/rhnvrm/simples3" +) + +type tagsResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Target string `json:"target"` + Status string `json:"status"` + Tags map[string]string `json:"tags"` +} + +type tagsFlags struct { + awsFlags +} + +func (rt *runtime) runTags(args []string) error { + if len(args) == 0 { + return usageErrorf("tags requires a subcommand: get, set, or delete") + } + switch args[0] { + case "get": + return rt.runTagsGet(args[1:]) + case "set": + return rt.runTagsSet(args[1:]) + case "delete", "rm": + return rt.runTagsDelete(args[1:]) + default: + return usageErrorf("unknown tags subcommand %q", args[0]) + } +} + +func (rt *runtime) runTagsGet(args []string) error { + flags := tagsFlags{} + fs := newFlagSet("tags get", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 tags get [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket/key\n") + }) + addAWSFlags(fs, &flags.awsFlags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("tags get requires exactly one object URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key == "" { + return usageErrorf("tags get target must be an object URI like s3://bucket/key") + } + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + output, err := settings.newClient().GetObjectTagging(simples3.GetObjectTaggingInput{Bucket: loc.bucket, ObjectKey: loc.key}) + if err != nil { + return err + } + tags := output.Tags + if tags == nil { + tags = map[string]string{} + } + result := tagsResult{Command: "tags", OK: true, Target: loc.String(), Status: "loaded", Tags: tags} + if flags.json { + return writeJSON(rt.stdout, result) + } + writeTagLines(rt.stdout, output.Tags) + return nil +} + +func (rt *runtime) runTagsSet(args []string) error { + flags := tagsFlags{} + var tagsFile string + var tags keyValueFlag + fs := newFlagSet("tags set", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 tags set [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] [--tag key=value ...] [--tags-file PATH|-] s3://bucket/key\n") + }) + addAWSFlags(fs, &flags.awsFlags) + fs.Var(&tags, "tag", "tag key=value (repeatable)") + fs.StringVar(&tagsFile, "tags-file", "", "JSON file or - for stdin with tag map") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("tags set requires exactly one object URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key == "" { + return usageErrorf("tags set target must be an object URI like s3://bucket/key") + } + mergedTags := map[string]string{} + if tagsFile != "" { + if err := decodeJSONSource(rt, tagsFile, &mergedTags); err != nil { + return err + } + } + for key, value := range tags { + mergedTags[key] = value + } + if len(mergedTags) == 0 { + return usageErrorf("tags set requires at least one --tag or --tags-file input") + } + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + if err := settings.newClient().PutObjectTagging(simples3.PutObjectTaggingInput{Bucket: loc.bucket, ObjectKey: loc.key, Tags: mergedTags}); err != nil { + return err + } + result := tagsResult{Command: "tags", OK: true, Target: loc.String(), Status: "updated", Tags: mergedTags} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "updated tags %s", loc.String()) + return nil +} + +func (rt *runtime) runTagsDelete(args []string) error { + flags := tagsFlags{} + fs := newFlagSet("tags delete", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 tags delete [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket/key\n") + }) + addAWSFlags(fs, &flags.awsFlags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("tags delete requires exactly one object URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key == "" { + return usageErrorf("tags delete target must be an object URI like s3://bucket/key") + } + settings, err := rt.resolveAWSSettings(flags.awsFlags) + if err != nil { + return err + } + if err := settings.newClient().DeleteObjectTagging(simples3.DeleteObjectTaggingInput{Bucket: loc.bucket, ObjectKey: loc.key}); err != nil { + return err + } + result := tagsResult{Command: "tags", OK: true, Target: loc.String(), Status: "deleted", Tags: map[string]string{}} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "deleted tags %s", loc.String()) + return nil +} diff --git a/cmd/simples3/command_test.go b/cmd/simples3/command_test.go new file mode 100644 index 0000000..785e446 --- /dev/null +++ b/cmd/simples3/command_test.go @@ -0,0 +1,1044 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func newTestRuntime(t *testing.T, env map[string]string) (*runtime, *bytes.Buffer, *bytes.Buffer) { + t.Helper() + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + rt := &runtime{ + stdin: strings.NewReader(""), + stdout: stdout, + stderr: stderr, + getenv: func(key string) string { return env[key] }, + homeDir: func() (string, error) { return t.TempDir(), nil }, + } + return rt, stdout, stderr +} + +func decodeJSON(t *testing.T, data string, target any) { + t.Helper() + if err := json.Unmarshal([]byte(data), target); err != nil { + t.Fatalf("failed to decode JSON %s: %v", data, err) + } +} + +func TestRunListBucketsJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + 1test + + bucket-one2026-01-02T03:04:05Z + +`)) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"ls", "--json", "--endpoint", server.URL}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, "bucket-one") { + t.Fatalf("expected bucket output, got %s", output) + } + if !strings.Contains(output, `"command": "ls"`) || !strings.Contains(output, `"ok": true`) { + t.Fatalf("expected JSON envelope fields, got %s", output) + } +} + +func TestRunMakeAndRemoveBucket(t *testing.T) { + requests := make([]string, 0, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method+" "+r.URL.Path) + if r.Method == http.MethodPut { + w.Header().Set("Location", "/example-bucket") + w.WriteHeader(http.StatusOK) + return + } + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusMethodNotAllowed) + })) + defer server.Close() + + env := map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + } + + rt, stdout, stderr := newTestRuntime(t, env) + if code := rt.run([]string{"mb", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("mb failed: code=%d stderr=%s", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "created s3://example-bucket") { + t.Fatalf("unexpected mb stdout: %s", stdout.String()) + } + + stdout.Reset() + stderr.Reset() + if code := rt.run([]string{"rb", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("rb failed: code=%d stderr=%s", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "deleted s3://example-bucket") { + t.Fatalf("unexpected rb stdout: %s", stdout.String()) + } + + if len(requests) != 2 || requests[0] != "PUT /example-bucket" || requests[1] != "DELETE /example-bucket" { + t.Fatalf("unexpected requests: %#v", requests) + } +} + +func TestRunPresign(t *testing.T) { + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"presign", "--endpoint", "https://objects.example.test", "--method", "GET", "--expires", "15m", "s3://example-bucket/path/file.txt"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, "X-Amz-Algorithm=AWS4-HMAC-SHA256") { + t.Fatalf("expected presigned URL output, got %s", output) + } + if !strings.Contains(output, "example-bucket") { + t.Fatalf("expected bucket in URL, got %s", output) + } +} + +func TestRunCopyUploadLocalToS3(t *testing.T) { + sourceFile := filepath.Join(t.TempDir(), "upload.txt") + if err := os.WriteFile(sourceFile, []byte("hello upload"), 0o644); err != nil { + t.Fatal(err) + } + + var requestPath string + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + body = string(payload) + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"cp", "--endpoint", server.URL, sourceFile, "s3://example-bucket/uploads/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + if requestPath != "/example-bucket/uploads/upload.txt" { + t.Fatalf("unexpected request path: %s", requestPath) + } + if body != "hello upload" { + t.Fatalf("unexpected uploaded body: %q", body) + } + if !strings.Contains(stdout.String(), "copied ") { + t.Fatalf("expected copy output, got %s", stdout.String()) + } +} + +func TestRunCopyDownloadS3ToLocal(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-bucket/path/file.txt" { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "18") + w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT") + w.WriteHeader(http.StatusOK) + return + } + if r.Method == http.MethodGet { + _, _ = w.Write([]byte("downloaded content")) + return + } + w.WriteHeader(http.StatusMethodNotAllowed) + })) + defer server.Close() + + destinationDir := t.TempDir() + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"cp", "--endpoint", server.URL, "s3://example-bucket/path/file.txt", destinationDir + string(filepath.Separator)}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + data, err := os.ReadFile(filepath.Join(destinationDir, "file.txt")) + if err != nil { + t.Fatal(err) + } + if string(data) != "downloaded content" { + t.Fatalf("unexpected downloaded content: %q", string(data)) + } + if !strings.Contains(stdout.String(), "copied ") { + t.Fatalf("expected copy output, got %s", stdout.String()) + } +} + +func TestRunRemoveRecursiveDryRun(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if r.Method == http.MethodGet && r.URL.Query().Get("list-type") == "2" { + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + example-bucket + false + 1 + + prefix/one.txt + 2026-01-02T03:04:05Z + etag + 12 + STANDARD + +`)) + return + } + w.WriteHeader(http.StatusMethodNotAllowed) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"rm", "--recursive", "--dry-run", "--endpoint", server.URL, "s3://example-bucket/prefix/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + if requestCount != 1 { + t.Fatalf("expected one list request, got %d", requestCount) + } + if !strings.Contains(stdout.String(), "would-delete s3://example-bucket/prefix/one.txt") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunRemoveSingleObjectRetriesTransientFailure(t *testing.T) { + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Path != "/example-bucket/object.txt" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + attempts++ + if attempts == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("temporary failure")) + return + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"rm", "--retries", "1", "--endpoint", server.URL, "s3://example-bucket/object.txt"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if attempts != 2 { + t.Fatalf("expected 2 delete attempts, got %d", attempts) + } + if !strings.Contains(stdout.String(), "deleted s3://example-bucket/object.txt") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunMoveLocalToS3RemovesSource(t *testing.T) { + sourceFile := filepath.Join(t.TempDir(), "move.txt") + if err := os.WriteFile(sourceFile, []byte("move me"), 0o644); err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.URL.Path != "/example-bucket/archive/move.txt" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"mv", "--endpoint", server.URL, sourceFile, "s3://example-bucket/archive/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + if _, err := os.Stat(sourceFile); !os.IsNotExist(err) { + t.Fatalf("expected source file to be removed, stat err=%v", err) + } + if !strings.Contains(stdout.String(), "moved ") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunCopyRequiresRecursiveForS3PrefixSource(t *testing.T) { + rt, _, stderr := newTestRuntime(t, nil) + code := rt.run([]string{"cp", "s3://example-bucket/prefix/", filepath.Join(t.TempDir(), "out")}) + if code == 0 { + t.Fatalf("expected cp to fail without --recursive") + } + if !strings.Contains(stderr.String(), "use --recursive") { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } +} + +func TestRunMoveRequiresRecursiveForS3PrefixSource(t *testing.T) { + rt, _, stderr := newTestRuntime(t, nil) + code := rt.run([]string{"mv", "s3://example-bucket/prefix/", "s3://example-bucket/dst"}) + if code == 0 { + t.Fatalf("expected mv to fail without --recursive") + } + if !strings.Contains(stderr.String(), "use --recursive") { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } +} + +func TestRunSyncUploadsAndDeletes(t *testing.T) { + sourceDir := t.TempDir() + if err := os.WriteFile(filepath.Join(sourceDir, "new.txt"), []byte("fresh"), 0o644); err != nil { + t.Fatal(err) + } + + requests := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) + switch { + case r.Method == http.MethodHead && r.URL.Path == "/example-bucket/prefix/new.txt": + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodPut && r.URL.Path == "/example-bucket/prefix/new.txt": + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Query().Get("list-type") == "2": + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + example-bucket + false + 1 + + prefix/stale.txt + 2026-01-02T03:04:05Z + etag + 9 + STANDARD + +`)) + case r.Method == http.MethodDelete && r.URL.Path == "/example-bucket/prefix/stale.txt": + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"sync", "--delete", "--endpoint", server.URL, sourceDir, "s3://example-bucket/prefix/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + joined := strings.Join(requests, "\n") + if !strings.Contains(joined, "HEAD /example-bucket/prefix/new.txt?") || !strings.Contains(joined, "PUT /example-bucket/prefix/new.txt?") || !strings.Contains(joined, "DELETE /example-bucket/prefix/stale.txt?") { + t.Fatalf("unexpected request sequence:\n%s", joined) + } + if !strings.Contains(stdout.String(), "synced ") || !strings.Contains(stdout.String(), "deleted s3://example-bucket/prefix/stale.txt") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunCopyDryRunJSONIncludesSummary(t *testing.T) { + sourceFile := filepath.Join(t.TempDir(), "upload.txt") + if err := os.WriteFile(sourceFile, []byte("hello upload"), 0o644); err != nil { + t.Fatal(err) + } + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"cp", "--dry-run", "--json", sourceFile, "s3://example-bucket/uploads/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, `"command": "cp"`) || !strings.Contains(output, `"dryRun": true`) || !strings.Contains(output, `"changed": 1`) { + t.Fatalf("unexpected JSON output: %s", output) + } +} + +func TestRunJSONUsageErrorIncludesExitCode(t *testing.T) { + rt, stdout, stderr := newTestRuntime(t, nil) + code := rt.run([]string{"cp", "--json", filepath.Join(t.TempDir(), "a"), filepath.Join(t.TempDir(), "b")}) + if code != 2 { + t.Fatalf("expected exit code 2, got %d (stderr=%s)", code, stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr in JSON mode, got %s", stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, `"ok": false`) || !strings.Contains(output, `"type": "usage"`) || !strings.Contains(output, `"exitCode": 2`) { + t.Fatalf("unexpected JSON error output: %s", output) + } +} + +func TestRunSingleDashJSONUsageErrorIncludesExitCode(t *testing.T) { + rt, stdout, stderr := newTestRuntime(t, nil) + code := rt.run([]string{"cp", "-json", filepath.Join(t.TempDir(), "a"), filepath.Join(t.TempDir(), "b")}) + if code != 2 { + t.Fatalf("expected exit code 2, got %d (stderr=%s)", code, stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr in JSON mode, got %s", stderr.String()) + } + output := stdout.String() + if !strings.Contains(output, `"ok": false`) || !strings.Contains(output, `"type": "usage"`) || !strings.Contains(output, `"exitCode": 2`) { + t.Fatalf("unexpected JSON error output: %s", output) + } +} + +func TestRunTextUsageErrorReturnsExitCodeTwo(t *testing.T) { + rt, stdout, stderr := newTestRuntime(t, nil) + code := rt.run([]string{"cp", filepath.Join(t.TempDir(), "a"), filepath.Join(t.TempDir(), "b")}) + if code != 2 { + t.Fatalf("expected exit code 2, got %d", code) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout, got %s", stdout.String()) + } + if !strings.Contains(stderr.String(), "local-to-local copies are not supported") { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } +} + +func TestRunSyncUpdatesS3TargetWhenOnlyModTimeDiffers(t *testing.T) { + sourceDir := t.TempDir() + sourcePath := filepath.Join(sourceDir, "same-size.txt") + if err := os.WriteFile(sourcePath, []byte("fresh"), 0o644); err != nil { + t.Fatal(err) + } + sourceTime := time.Date(2026, 1, 3, 4, 5, 6, 0, time.UTC) + if err := os.Chtimes(sourcePath, sourceTime, sourceTime); err != nil { + t.Fatal(err) + } + + requests := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method+" "+r.URL.Path+"?"+r.URL.RawQuery) + switch { + case r.Method == http.MethodHead && r.URL.Path == "/example-bucket/prefix/same-size.txt": + w.Header().Set("Content-Length", "5") + w.Header().Set("Last-Modified", "Mon, 02 Jan 2006 15:04:05 GMT") + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPut && r.URL.Path == "/example-bucket/prefix/same-size.txt": + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"sync", "--endpoint", server.URL, sourceDir, "s3://example-bucket/prefix/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stderr=%s", code, stderr.String()) + } + joined := strings.Join(requests, "\n") + if !strings.Contains(joined, "HEAD /example-bucket/prefix/same-size.txt?") || !strings.Contains(joined, "PUT /example-bucket/prefix/same-size.txt?") { + t.Fatalf("expected sync to update same-size object when modtime differs, got:\n%s", joined) + } + if !strings.Contains(stdout.String(), "synced ") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunCopyRetriesTransientUploadFailure(t *testing.T) { + sourceFile := filepath.Join(t.TempDir(), "retry.txt") + if err := os.WriteFile(sourceFile, []byte("retry me"), 0o644); err != nil { + t.Fatal(err) + } + + attempts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.URL.Path != "/example-bucket/uploads/retry.txt" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + attempts++ + if attempts == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte("temporary failure")) + return + } + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"cp", "--retries", "1", "--endpoint", server.URL, sourceFile, "s3://example-bucket/uploads/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if attempts != 2 { + t.Fatalf("expected 2 attempts, got %d", attempts) + } + if !strings.Contains(stdout.String(), "copied ") { + t.Fatalf("unexpected output: %s", stdout.String()) + } +} + +func TestRunCopyContinueOnErrorJSONReturnsPartialFailure(t *testing.T) { + sourceDir := t.TempDir() + if err := os.WriteFile(filepath.Join(sourceDir, "bad.txt"), []byte("bad"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sourceDir, "ok.txt"), []byte("ok"), 0o644); err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch r.URL.Path { + case "/example-bucket/prefix/bad.txt": + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + case "/example-bucket/prefix/ok.txt": + w.Header().Set("ETag", "etag") + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"cp", "--recursive", "--continue-on-error", "--retries", "0", "--json", "--endpoint", server.URL, sourceDir, "s3://example-bucket/prefix/"}) + if code != 3 { + t.Fatalf("expected exit code 3, got %d (stdout=%s stderr=%s)", code, stdout.String(), stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr, got %s", stderr.String()) + } + var output struct { + Command string `json:"command"` + OK bool `json:"ok"` + Error commandErrorDetail `json:"error"` + Data operationsData `json:"data"` + } + decodeJSON(t, stdout.String(), &output) + if output.Command != "cp" || output.OK { + t.Fatalf("unexpected partial failure envelope: %+v", output) + } + if output.Error.Type != string(cliErrorPartial) || output.Error.ExitCode != 3 { + t.Fatalf("unexpected error detail: %+v", output.Error) + } + if output.Data.Summary.Changed != 1 || output.Data.Summary.Failed != 1 || output.Data.Summary.Total != 2 { + t.Fatalf("unexpected summary: %+v", output.Data.Summary) + } + if len(output.Data.Operations) != 2 { + t.Fatalf("expected 2 operations, got %d", len(output.Data.Operations)) + } + if output.Data.Operations[0].Status != "failed" || output.Data.Operations[0].Error == "" { + t.Fatalf("expected first operation to fail with details, got %+v", output.Data.Operations[0]) + } + if output.Data.Operations[1].Status != "copied" { + t.Fatalf("expected second operation to succeed, got %+v", output.Data.Operations[1]) + } +} + +func TestRunSyncPlanJSONIncludesUnchangedEntries(t *testing.T) { + sourceDir := t.TempDir() + sourcePath := filepath.Join(sourceDir, "same.txt") + if err := os.WriteFile(sourcePath, []byte("same"), 0o644); err != nil { + t.Fatal(err) + } + sourceTime := time.Date(2026, 1, 2, 15, 4, 5, 0, time.UTC) + if err := os.Chtimes(sourcePath, sourceTime, sourceTime); err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && r.URL.Path == "/example-bucket/prefix/same.txt" { + w.Header().Set("Content-Length", "4") + w.Header().Set("Last-Modified", sourceTime.Format(http.TimeFormat)) + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusMethodNotAllowed) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"sync", "--plan", "--json", "--endpoint", server.URL, sourceDir, "s3://example-bucket/prefix/"}) + if code != 0 { + t.Fatalf("unexpected exit code %d, stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var output operationsOutput + decodeJSON(t, stdout.String(), &output) + if output.Command != "sync" || !output.OK { + t.Fatalf("unexpected output: %+v", output) + } + if output.Summary.Total != 1 || output.Summary.Unchanged != 1 || !output.Summary.Noop || !output.Summary.DryRun { + t.Fatalf("unexpected summary: %+v", output.Summary) + } + if len(output.Operations) != 1 || output.Operations[0].Status != "unchanged" { + t.Fatalf("unexpected operations: %+v", output.Operations) + } +} + +func TestRunRemoveRecursiveContinueOnErrorJSONReturnsPartialFailure(t *testing.T) { + deleteRequests := 0 + var deleteBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Query().Get("list-type") == "2": + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + example-bucket + false + 2 + + prefix/fail.txt + 2026-01-02T03:04:05Z + etag + 4 + STANDARD + + + prefix/ok.txt + 2026-01-02T03:04:05Z + etag + 2 + STANDARD + +`)) + case r.Method == http.MethodPost && r.URL.RawQuery == "delete": + deleteRequests++ + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + deleteBody = string(payload) + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + prefix/ok.txt + prefix/fail.txtInternalErrordelete failed +`)) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{ + "AWS_ACCESS_KEY_ID": "test-access", + "AWS_SECRET_ACCESS_KEY": "test-secret", + }) + code := rt.run([]string{"rm", "--recursive", "--continue-on-error", "--retries", "0", "--json", "--endpoint", server.URL, "s3://example-bucket/prefix/"}) + if code != 3 { + t.Fatalf("expected exit code 3, got %d (stdout=%s stderr=%s)", code, stdout.String(), stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr, got %s", stderr.String()) + } + if deleteRequests != 1 { + t.Fatalf("expected one batched delete request, got %d", deleteRequests) + } + if !strings.Contains(deleteBody, "prefix/fail.txt") || !strings.Contains(deleteBody, "prefix/ok.txt") { + t.Fatalf("unexpected delete body: %s", deleteBody) + } + var output struct { + Command string `json:"command"` + OK bool `json:"ok"` + Error commandErrorDetail `json:"error"` + Data operationsData `json:"data"` + } + decodeJSON(t, stdout.String(), &output) + if output.Command != "rm" || output.OK { + t.Fatalf("unexpected partial failure envelope: %+v", output) + } + if output.Data.Summary.Deleted != 1 || output.Data.Summary.Failed != 1 || output.Data.Summary.Total != 2 { + t.Fatalf("unexpected summary: %+v", output.Data.Summary) + } + if len(output.Data.Operations) != 2 { + t.Fatalf("expected 2 operations, got %d", len(output.Data.Operations)) + } + if output.Data.Operations[0].Status != "failed" || output.Data.Operations[1].Status != "deleted" { + t.Fatalf("unexpected operations: %+v", output.Data.Operations) + } + if output.Data.Operations[0].Error != "InternalError: delete failed" { + t.Fatalf("unexpected failure detail: %+v", output.Data.Operations[0]) + } +} + +func TestRunTagsCommands(t *testing.T) { + var putBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-bucket/object.txt" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if _, ok := r.URL.Query()["tagging"]; !ok { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch r.Method { + case http.MethodPut: + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + putBody = string(payload) + w.WriteHeader(http.StatusOK) + case http.MethodGet: + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + + envtest + teamplatform + +`)) + case http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + tagsFile := filepath.Join(t.TempDir(), "tags.json") + if err := os.WriteFile(tagsFile, []byte(`{"env":"test","team":"platform"}`), 0o644); err != nil { + t.Fatal(err) + } + + env := map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"} + rt, stdout, stderr := newTestRuntime(t, env) + if code := rt.run([]string{"tags", "set", "--json", "--endpoint", server.URL, "--tags-file", tagsFile, "s3://example-bucket/object.txt"}); code != 0 { + t.Fatalf("tags set failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(putBody, "env") || !strings.Contains(putBody, "team") { + t.Fatalf("unexpected tagging PUT body: %s", putBody) + } + var setOutput tagsResult + decodeJSON(t, stdout.String(), &setOutput) + if setOutput.Status != "updated" || setOutput.Tags["env"] != "test" { + t.Fatalf("unexpected tags set output: %+v", setOutput) + } + + rt, stdout, stderr = newTestRuntime(t, env) + if code := rt.run([]string{"tags", "get", "--json", "--endpoint", server.URL, "s3://example-bucket/object.txt"}); code != 0 { + t.Fatalf("tags get failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var getOutput tagsResult + decodeJSON(t, stdout.String(), &getOutput) + if getOutput.Status != "loaded" || getOutput.Tags["team"] != "platform" { + t.Fatalf("unexpected tags get output: %+v", getOutput) + } + + rt, stdout, stderr = newTestRuntime(t, env) + if code := rt.run([]string{"tags", "delete", "--json", "--endpoint", server.URL, "s3://example-bucket/object.txt"}); code != 0 { + t.Fatalf("tags delete failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var deleteOutput tagsResult + decodeJSON(t, stdout.String(), &deleteOutput) + if deleteOutput.Status != "deleted" { + t.Fatalf("unexpected tags delete output: %+v", deleteOutput) + } +} + +func TestRunVersioningCommands(t *testing.T) { + var putBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-bucket" || r.URL.RawQuery != "versioning" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch r.Method { + case http.MethodPut: + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + putBody = string(payload) + w.WriteHeader(http.StatusOK) + case http.MethodGet: + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + Enabled +`)) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + env := map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"} + rt, stdout, stderr := newTestRuntime(t, env) + if code := rt.run([]string{"versioning", "set", "--json", "--endpoint", server.URL, "--status", "enabled", "s3://example-bucket"}); code != 0 { + t.Fatalf("versioning set failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(putBody, "Enabled") { + t.Fatalf("unexpected versioning PUT body: %s", putBody) + } + var setOutput versioningResult + decodeJSON(t, stdout.String(), &setOutput) + if setOutput.Status != "Enabled" { + t.Fatalf("unexpected versioning set output: %+v", setOutput) + } + + rt, stdout, stderr = newTestRuntime(t, env) + if code := rt.run([]string{"versioning", "get", "--json", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("versioning get failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var getOutput versioningResult + decodeJSON(t, stdout.String(), &getOutput) + if getOutput.Status != "Enabled" { + t.Fatalf("unexpected versioning get output: %+v", getOutput) + } +} + +func TestRunVersionsJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if r.Method != http.MethodGet || r.URL.Path != "/example-bucket" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if _, ok := query["versions"]; !ok || query.Get("prefix") != "prefix/" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + example-bucket + prefix/ + false + + prefix/object.txt + v2 + true + 2026-01-02T03:04:05Z + etag + 12 + STANDARD + owner-idowner + + + prefix/object.txt + v1 + false + 2026-01-01T03:04:05Z + owner-idowner + +`)) + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"}) + if code := rt.run([]string{"versions", "--json", "--endpoint", server.URL, "s3://example-bucket/prefix/"}); code != 0 { + t.Fatalf("versions failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var output versionsResult + decodeJSON(t, stdout.String(), &output) + if output.Bucket != "example-bucket" || len(output.Versions) != 1 || output.Versions[0].VersionID != "v2" || len(output.DeleteMarkers) != 1 { + t.Fatalf("unexpected versions output: %+v", output) + } +} + +func TestRunLifecycleCommands(t *testing.T) { + var putBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-bucket" || r.URL.RawQuery != "lifecycle" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch r.Method { + case http.MethodPut: + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + putBody = string(payload) + w.WriteHeader(http.StatusOK) + case http.MethodGet: + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + + expire + Enabled + 30 + +`)) + case http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + rt, stdout, stderr := newTestRuntime(t, map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"}) + rt.stdin = strings.NewReader(`{"rules":[{"id":"expire","status":"Enabled","expiration":{"days":30}}]}`) + if code := rt.run([]string{"lifecycle", "set", "--json", "--endpoint", server.URL, "--file", "-", "s3://example-bucket"}); code != 0 { + t.Fatalf("lifecycle set failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(putBody, "expire") || !strings.Contains(putBody, "30") { + t.Fatalf("unexpected lifecycle PUT body: %s", putBody) + } + var setOutput lifecycleResult + decodeJSON(t, stdout.String(), &setOutput) + if setOutput.Status != "updated" || setOutput.Configuration == nil || len(setOutput.Configuration.Rules) != 1 || setOutput.Configuration.Rules[0].Expiration == nil || setOutput.Configuration.Rules[0].Expiration.Days != 30 { + t.Fatalf("unexpected lifecycle set output: %+v", setOutput) + } + + rt, stdout, stderr = newTestRuntime(t, map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"}) + if code := rt.run([]string{"lifecycle", "get", "--json", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("lifecycle get failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var getOutput lifecycleResult + decodeJSON(t, stdout.String(), &getOutput) + if getOutput.Status != "loaded" || getOutput.Configuration == nil || len(getOutput.Configuration.Rules) != 1 || getOutput.Configuration.Rules[0].ID != "expire" { + t.Fatalf("unexpected lifecycle get output: %+v", getOutput) + } + + rt, stdout, stderr = newTestRuntime(t, map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"}) + if code := rt.run([]string{"lifecycle", "delete", "--json", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("lifecycle delete failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var deleteOutput lifecycleResult + decodeJSON(t, stdout.String(), &deleteOutput) + if deleteOutput.Status != "deleted" { + t.Fatalf("unexpected lifecycle delete output: %+v", deleteOutput) + } +} + +func TestRunACLCommands(t *testing.T) { + var putBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-bucket" || r.URL.RawQuery != "acl" { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch r.Method { + case http.MethodPut: + payload, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + putBody = string(payload) + w.WriteHeader(http.StatusOK) + case http.MethodGet: + w.Header().Set("Content-Type", "application/xml") + _, _ = w.Write([]byte(` + + owner-idowner + + + + owner-id + owner + + FULL_CONTROL + + +`)) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + policyFile := filepath.Join(t.TempDir(), "acl.json") + policy := `{"owner":{"id":"owner-id","displayName":"owner"},"grants":[{"grantee":{"type":"CanonicalUser","id":"owner-id","displayName":"owner"},"permission":"FULL_CONTROL"}]}` + if err := os.WriteFile(policyFile, []byte(policy), 0o644); err != nil { + t.Fatal(err) + } + env := map[string]string{"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret"} + rt, stdout, stderr := newTestRuntime(t, env) + if code := rt.run([]string{"acl", "set", "--json", "--endpoint", server.URL, "--policy-file", policyFile, "s3://example-bucket"}); code != 0 { + t.Fatalf("acl set failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(putBody, "FULL_CONTROL") { + t.Fatalf("unexpected ACL PUT body: %s", putBody) + } + var setOutput aclResult + decodeJSON(t, stdout.String(), &setOutput) + if setOutput.Status != "updated" || setOutput.Policy == nil || len(setOutput.Policy.Grants) != 1 || setOutput.Policy.Owner.ID != "owner-id" { + t.Fatalf("unexpected acl set output: %+v", setOutput) + } + + rt, stdout, stderr = newTestRuntime(t, env) + if code := rt.run([]string{"acl", "get", "--json", "--endpoint", server.URL, "s3://example-bucket"}); code != 0 { + t.Fatalf("acl get failed: code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + var getOutput aclResult + decodeJSON(t, stdout.String(), &getOutput) + if getOutput.Status != "loaded" || getOutput.Policy == nil || getOutput.Policy.Owner.ID != "owner-id" || len(getOutput.Policy.Grants) != 1 { + t.Fatalf("unexpected acl get output: %+v", getOutput) + } +} diff --git a/cmd/simples3/command_versioning.go b/cmd/simples3/command_versioning.go new file mode 100644 index 0000000..a453fda --- /dev/null +++ b/cmd/simples3/command_versioning.go @@ -0,0 +1,132 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/rhnvrm/simples3" +) + +type versioningResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Bucket string `json:"bucket"` + Status string `json:"status"` + MFADelete string `json:"mfaDelete,omitempty"` +} + +func (rt *runtime) runVersioning(args []string) error { + if len(args) == 0 { + return usageErrorf("versioning requires a subcommand: get or set") + } + switch args[0] { + case "get": + return rt.runVersioningGet(args[1:]) + case "set": + return rt.runVersioningSet(args[1:]) + default: + return usageErrorf("unknown versioning subcommand %q", args[0]) + } +} + +func (rt *runtime) runVersioningGet(args []string) error { + var flags awsFlags + fs := newFlagSet("versioning get", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 versioning get [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("versioning get requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key != "" { + return usageErrorf("versioning get target must be a bucket URI like s3://my-bucket") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + output, err := settings.newClient().GetBucketVersioning(loc.bucket) + if err != nil { + return err + } + result := versioningResult{Command: "versioning", OK: true, Bucket: loc.bucket, Status: firstNonEmpty(output.Status, "Unversioned"), MFADelete: output.MfaDelete} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "status: %s", result.Status) + if result.MFADelete != "" { + printLine(rt.stdout, "mfa-delete: %s", result.MFADelete) + } + return nil +} + +func (rt *runtime) runVersioningSet(args []string) error { + var flags awsFlags + var status string + var mfaDelete string + fs := newFlagSet("versioning set", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 versioning set [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] --status enabled|suspended [--mfa-delete enabled|disabled] s3://bucket\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&status, "status", "", "versioning status: enabled or suspended") + fs.StringVar(&mfaDelete, "mfa-delete", "", "MFA delete status: enabled or disabled") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("versioning set requires exactly one bucket URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() || loc.key != "" { + return usageErrorf("versioning set target must be a bucket URI like s3://my-bucket") + } + status = normalizeVersioningStatus(status) + if status == "" { + return usageErrorf("versioning set requires --status enabled|suspended") + } + if status != "Enabled" && status != "Suspended" { + return usageErrorf("invalid --status value %q", status) + } + mfaDelete = normalizeVersioningStatus(mfaDelete) + if mfaDelete != "" && mfaDelete != "Enabled" && mfaDelete != "Disabled" { + return usageErrorf("invalid --mfa-delete value %q", mfaDelete) + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + if err := settings.newClient().PutBucketVersioning(simples3.PutBucketVersioningInput{Bucket: loc.bucket, Status: status, MfaDelete: mfaDelete}); err != nil { + return err + } + result := versioningResult{Command: "versioning", OK: true, Bucket: loc.bucket, Status: status, MFADelete: mfaDelete} + if flags.json { + return writeJSON(rt.stdout, result) + } + printLine(rt.stdout, "updated versioning %s to %s", loc.String(), status) + return nil +} + +func normalizeVersioningStatus(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "enabled": + return "Enabled" + case "disabled": + return "Disabled" + case "suspended": + return "Suspended" + case "": + return "" + default: + return value + } +} diff --git a/cmd/simples3/command_versions.go b/cmd/simples3/command_versions.go new file mode 100644 index 0000000..f20690e --- /dev/null +++ b/cmd/simples3/command_versions.go @@ -0,0 +1,139 @@ +package main + +import ( + "fmt" + + "github.com/rhnvrm/simples3" +) + +type versionItem struct { + Key string `json:"key"` + VersionID string `json:"versionId"` + IsLatest bool `json:"isLatest"` + LastModified string `json:"lastModified"` + Size int64 `json:"size"` + StorageClass string `json:"storageClass,omitempty"` + OwnerID string `json:"ownerId,omitempty"` + OwnerName string `json:"ownerName,omitempty"` +} + +type deleteMarkerItem struct { + Key string `json:"key"` + VersionID string `json:"versionId"` + IsLatest bool `json:"isLatest"` + LastModified string `json:"lastModified"` + OwnerID string `json:"ownerId,omitempty"` + OwnerName string `json:"ownerName,omitempty"` +} + +type versionsResult struct { + Command string `json:"command"` + OK bool `json:"ok"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + Delimiter string `json:"delimiter,omitempty"` + MaxKeys int64 `json:"maxKeys,omitempty"` + IsTruncated bool `json:"isTruncated"` + NextKeyMarker string `json:"nextKeyMarker,omitempty"` + NextVersionIDMarker string `json:"nextVersionIdMarker,omitempty"` + CommonPrefixes []string `json:"commonPrefixes,omitempty"` + Versions []versionItem `json:"versions,omitempty"` + DeleteMarkers []deleteMarkerItem `json:"deleteMarkers,omitempty"` +} + +func (rt *runtime) runVersions(args []string) error { + var flags awsFlags + var delimiter string + var maxKeys int64 + var keyMarker string + var versionIDMarker string + fs := newFlagSet("versions", func() { + fmt.Fprint(rt.stderr, "Usage: simples3 versions [--profile PROFILE] [--region REGION] [--endpoint URL] [--json] [--delimiter /] [--max-keys N] [--key-marker KEY] [--version-id-marker VID] s3://bucket[/prefix]\n") + }) + addAWSFlags(fs, &flags) + fs.StringVar(&delimiter, "delimiter", "", "delimiter to group keys") + fs.Int64Var(&maxKeys, "max-keys", 0, "maximum versions to return") + fs.StringVar(&keyMarker, "key-marker", "", "key marker for pagination") + fs.StringVar(&versionIDMarker, "version-id-marker", "", "version ID marker for pagination") + if err := parseFlagSet(fs, args); err != nil { + return err + } + if fs.NArg() != 1 { + return usageErrorf("versions requires exactly one S3 bucket or prefix URI") + } + loc, err := parseLocation(fs.Arg(0)) + if err != nil { + return err + } + if !loc.isS3() { + return usageErrorf("versions target must be an S3 URI like s3://bucket or s3://bucket/prefix") + } + settings, err := rt.resolveAWSSettings(flags) + if err != nil { + return err + } + prefix := loc.key + if loc.hasTrailing { + prefix = loc.s3Prefix() + } + output, err := settings.newClient().ListVersions(simples3.ListVersionsInput{ + Bucket: loc.bucket, + Prefix: prefix, + Delimiter: delimiter, + MaxKeys: maxKeys, + KeyMarker: keyMarker, + VersionIdMarker: versionIDMarker, + }) + if err != nil { + return err + } + result := versionsResult{ + Command: "versions", + OK: true, + Bucket: loc.bucket, + Prefix: output.Prefix, + Delimiter: output.Delimiter, + MaxKeys: output.MaxKeys, + IsTruncated: output.IsTruncated, + NextKeyMarker: output.NextKeyMarker, + NextVersionIDMarker: output.NextVersionIdMarker, + CommonPrefixes: output.CommonPrefixes, + Versions: make([]versionItem, 0, len(output.Versions)), + DeleteMarkers: make([]deleteMarkerItem, 0, len(output.DeleteMarkers)), + } + for _, version := range output.Versions { + result.Versions = append(result.Versions, versionItem{ + Key: version.Key, + VersionID: version.VersionId, + IsLatest: version.IsLatest, + LastModified: version.LastModified, + Size: version.Size, + StorageClass: version.StorageClass, + OwnerID: version.Owner.ID, + OwnerName: version.Owner.DisplayName, + }) + } + for _, marker := range output.DeleteMarkers { + result.DeleteMarkers = append(result.DeleteMarkers, deleteMarkerItem{ + Key: marker.Key, + VersionID: marker.VersionId, + IsLatest: marker.IsLatest, + LastModified: marker.LastModified, + OwnerID: marker.Owner.ID, + OwnerName: marker.Owner.DisplayName, + }) + } + if flags.json { + return writeJSON(rt.stdout, result) + } + for _, prefix := range result.CommonPrefixes { + printLine(rt.stdout, "prefix %s", prefix) + } + for _, version := range result.Versions { + printLine(rt.stdout, "version %s version=%s latest=%t size=%d", version.Key, version.VersionID, version.IsLatest, version.Size) + } + for _, marker := range result.DeleteMarkers { + printLine(rt.stdout, "delete-marker %s version=%s latest=%t", marker.Key, marker.VersionID, marker.IsLatest) + } + return nil +} diff --git a/cmd/simples3/config.go b/cmd/simples3/config.go new file mode 100644 index 0000000..21a5558 --- /dev/null +++ b/cmd/simples3/config.go @@ -0,0 +1,168 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/rhnvrm/simples3" +) + +type awsFlags struct { + profile string + region string + endpoint string + json bool +} + +type awsSettings struct { + Profile string + Region string + AccessKey string + SecretKey string + SessionToken string + Endpoint string +} + +type iniSections map[string]map[string]string + +func addAWSFlags(fs interface { + StringVar(*string, string, string, string) + BoolVar(*bool, string, bool, string) +}, flags *awsFlags) { + fs.StringVar(&flags.profile, "profile", "", "AWS profile name") + fs.StringVar(&flags.region, "region", "", "AWS region") + fs.StringVar(&flags.endpoint, "endpoint", "", "custom S3 endpoint URL") + fs.BoolVar(&flags.json, "json", false, "emit JSON output") +} + +func (rt *runtime) resolveAWSSettings(flags awsFlags) (awsSettings, error) { + homeDir, err := rt.homeDir() + if err != nil { + return awsSettings{}, err + } + + profile := firstNonEmpty(flags.profile, rt.getenv("AWS_PROFILE"), "default") + credentialsPath := firstNonEmpty(rt.getenv("AWS_SHARED_CREDENTIALS_FILE"), filepath.Join(homeDir, ".aws", "credentials")) + configPath := firstNonEmpty(rt.getenv("AWS_CONFIG_FILE"), filepath.Join(homeDir, ".aws", "config")) + envAccessKey := firstNonEmpty(rt.getenv("AWS_ACCESS_KEY_ID"), rt.getenv("AWS_ACCESS_KEY")) + envSecretKey := firstNonEmpty(rt.getenv("AWS_SECRET_ACCESS_KEY"), rt.getenv("AWS_SECRET_KEY")) + envSessionToken := rt.getenv("AWS_SESSION_TOKEN") + envRegion := firstNonEmpty(flags.region, rt.getenv("AWS_REGION"), rt.getenv("AWS_DEFAULT_REGION")) + + credentials, err := parseINIFile(credentialsPath) + if err != nil { + if envAccessKey == "" || envSecretKey == "" { + return awsSettings{}, err + } + credentials = iniSections{} + } + config, err := parseINIFile(configPath) + if err != nil { + if envRegion == "" { + return awsSettings{}, err + } + config = iniSections{} + } + + credsSection := credentials[profile] + configSection := config[awsConfigSection(profile)] + + settings := awsSettings{ + Profile: profile, + Region: firstNonEmpty(envRegion, configValue(configSection, "region"), "us-east-1"), + AccessKey: firstNonEmpty(envAccessKey, configValue(credsSection, "aws_access_key_id")), + SecretKey: firstNonEmpty(envSecretKey, configValue(credsSection, "aws_secret_access_key")), + SessionToken: firstNonEmpty(envSessionToken, configValue(credsSection, "aws_session_token")), + Endpoint: firstNonEmpty(flags.endpoint, rt.getenv("AWS_ENDPOINT_URL_S3"), rt.getenv("AWS_ENDPOINT_URL"), rt.getenv("AWS_S3_ENDPOINT")), + } + + if settings.AccessKey == "" || settings.SecretKey == "" { + return awsSettings{}, errors.New("missing AWS credentials: set AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or configure an AWS credentials file") + } + + return settings, nil +} + +func (settings awsSettings) newClient() *simples3.S3 { + client := simples3.New(settings.Region, settings.AccessKey, settings.SecretKey) + if settings.SessionToken != "" { + client.SetToken(settings.SessionToken) + } + if settings.Endpoint != "" { + client.SetEndpoint(settings.Endpoint) + } + return client +} + +func parseINIFile(path string) (iniSections, error) { + sections := iniSections{} + file, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return sections, nil + } + return nil, err + } + defer file.Close() + + current := "" + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") { + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + current = strings.TrimSpace(line[1 : len(line)-1]) + if _, ok := sections[current]; !ok { + sections[current] = map[string]string{} + } + continue + } + if current == "" { + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + sections[current][strings.TrimSpace(strings.ToLower(key))] = trimINIValue(value) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + return sections, nil +} + +func trimINIValue(value string) string { + trimmed := strings.TrimSpace(value) + trimmed = strings.Trim(trimmed, `"`) + return trimmed +} + +func awsConfigSection(profile string) string { + if profile == "" || profile == "default" { + return "default" + } + return "profile " + profile +} + +func configValue(section map[string]string, key string) string { + if section == nil { + return "" + } + return section[strings.ToLower(key)] +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/cmd/simples3/config_test.go b/cmd/simples3/config_test.go new file mode 100644 index 0000000..4dd7b31 --- /dev/null +++ b/cmd/simples3/config_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveAWSSettingsPrecedence(t *testing.T) { + dir := t.TempDir() + awsDir := filepath.Join(dir, ".aws") + if err := os.MkdirAll(awsDir, 0o755); err != nil { + t.Fatal(err) + } + credentialsPath := filepath.Join(awsDir, "credentials") + configPath := filepath.Join(awsDir, "config") + if err := os.WriteFile(credentialsPath, []byte("[default]\naws_access_key_id = file-access\naws_secret_access_key = file-secret\n[work]\naws_access_key_id = work-access\naws_secret_access_key = work-secret\naws_session_token = work-token\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("[default]\nregion = us-west-1\n[profile work]\nregion = ap-south-1\n"), 0o644); err != nil { + t.Fatal(err) + } + + env := map[string]string{ + "AWS_SHARED_CREDENTIALS_FILE": credentialsPath, + "AWS_CONFIG_FILE": configPath, + "AWS_ACCESS_KEY_ID": "env-access", + "AWS_SECRET_ACCESS_KEY": "env-secret", + "AWS_REGION": "eu-central-1", + } + getenv := func(key string) string { return env[key] } + rt := &runtime{getenv: getenv, homeDir: func() (string, error) { return dir, nil }} + + settings, err := rt.resolveAWSSettings(awsFlags{profile: "work", region: "us-east-2"}) + if err != nil { + t.Fatal(err) + } + if settings.AccessKey != "env-access" || settings.SecretKey != "env-secret" { + t.Fatalf("expected env credentials, got %+v", settings) + } + if settings.Region != "us-east-2" { + t.Fatalf("expected flag region, got %s", settings.Region) + } + if settings.SessionToken != "work-token" { + t.Fatalf("expected session token from profile, got %q", settings.SessionToken) + } +} + +func TestResolveAWSSettingsFromFiles(t *testing.T) { + dir := t.TempDir() + awsDir := filepath.Join(dir, ".aws") + if err := os.MkdirAll(awsDir, 0o755); err != nil { + t.Fatal(err) + } + credentialsPath := filepath.Join(awsDir, "credentials") + configPath := filepath.Join(awsDir, "config") + if err := os.WriteFile(credentialsPath, []byte("[default]\naws_access_key_id = file-access\naws_secret_access_key = file-secret\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("[default]\nregion = us-west-2\n"), 0o644); err != nil { + t.Fatal(err) + } + getenv := func(string) string { return "" } + rt := &runtime{getenv: getenv, homeDir: func() (string, error) { return dir, nil }} + + settings, err := rt.resolveAWSSettings(awsFlags{}) + if err != nil { + t.Fatal(err) + } + if settings.AccessKey != "file-access" || settings.SecretKey != "file-secret" || settings.Region != "us-west-2" { + t.Fatalf("unexpected settings: %+v", settings) + } +} + +func TestResolveAWSSettingsIgnoresBrokenFilesWhenEnvSuffices(t *testing.T) { + dir := t.TempDir() + env := map[string]string{ + "AWS_SHARED_CREDENTIALS_FILE": dir, + "AWS_CONFIG_FILE": dir, + "AWS_ACCESS_KEY_ID": "env-access", + "AWS_SECRET_ACCESS_KEY": "env-secret", + "AWS_REGION": "us-east-1", + } + getenv := func(key string) string { return env[key] } + rt := &runtime{getenv: getenv, homeDir: func() (string, error) { return dir, nil }} + + settings, err := rt.resolveAWSSettings(awsFlags{}) + if err != nil { + t.Fatal(err) + } + if settings.AccessKey != "env-access" || settings.SecretKey != "env-secret" || settings.Region != "us-east-1" { + t.Fatalf("unexpected settings: %+v", settings) + } +} diff --git a/cmd/simples3/errors.go b/cmd/simples3/errors.go new file mode 100644 index 0000000..5ce7a94 --- /dev/null +++ b/cmd/simples3/errors.go @@ -0,0 +1,92 @@ +package main + +import ( + "errors" + "fmt" +) + +type cliErrorKind string + +const ( + cliErrorUsage cliErrorKind = "usage" + cliErrorRuntime cliErrorKind = "runtime" + cliErrorPartial cliErrorKind = "partial_failure" +) + +type cliError struct { + kind cliErrorKind + message string + cause error + output any +} + +func (e *cliError) Error() string { + switch { + case e == nil: + return "" + case e.message != "": + return e.message + case e.cause != nil: + return e.cause.Error() + default: + return string(e.kind) + } +} + +func (e *cliError) Unwrap() error { + if e == nil { + return nil + } + return e.cause +} + +func usageErrorf(format string, args ...any) error { + return &cliError{kind: cliErrorUsage, message: fmt.Sprintf(format, args...)} +} + +func partialFailureError(message string, output any) error { + return &cliError{kind: cliErrorPartial, message: message, output: output} +} + +func exitCodeForError(err error) int { + if err == nil || errors.Is(err, flagErrHelp) { + return 0 + } + var cliErr *cliError + if errors.As(err, &cliErr) { + switch cliErr.kind { + case cliErrorUsage: + return 2 + case cliErrorPartial: + return 3 + default: + return 1 + } + } + return 1 +} + +func errorKindForError(err error) cliErrorKind { + var cliErr *cliError + if errors.As(err, &cliErr) { + return cliErr.kind + } + return cliErrorRuntime +} + +func errorOutputForCommand(command string, err error) commandErrorOutput { + output := commandErrorOutput{ + Command: command, + OK: false, + Error: commandErrorDetail{ + Type: string(errorKindForError(err)), + Message: err.Error(), + ExitCode: exitCodeForError(err), + }, + } + var cliErr *cliError + if errors.As(err, &cliErr) && cliErr.output != nil { + output.Data = cliErr.output + } + return output +} diff --git a/cmd/simples3/filter.go b/cmd/simples3/filter.go new file mode 100644 index 0000000..157e04f --- /dev/null +++ b/cmd/simples3/filter.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + gopath "path" + "path/filepath" + "strings" +) + +type matcher struct { + includes []string + excludes []string +} + +func newMatcher(includes, excludes []string) (matcher, error) { + for _, pattern := range append(append([]string{}, includes...), excludes...) { + if _, err := gopath.Match(pattern, "x"); err != nil { + return matcher{}, fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + } + return matcher{includes: includes, excludes: excludes}, nil +} + +func (m matcher) Match(name string) bool { + normalized := normalizeMatchPath(name) + if len(m.includes) > 0 && !matchesAny(m.includes, normalized) { + return false + } + if matchesAny(m.excludes, normalized) { + return false + } + return true +} + +func matchesAny(patterns []string, name string) bool { + for _, pattern := range patterns { + matched, err := gopath.Match(pattern, name) + if err == nil && matched { + return true + } + } + return false +} + +func normalizeMatchPath(name string) string { + name = filepath.ToSlash(name) + name = strings.TrimPrefix(name, "./") + return strings.TrimPrefix(name, "/") +} diff --git a/cmd/simples3/filter_test.go b/cmd/simples3/filter_test.go new file mode 100644 index 0000000..846da18 --- /dev/null +++ b/cmd/simples3/filter_test.go @@ -0,0 +1,19 @@ +package main + +import "testing" + +func TestMatcher(t *testing.T) { + matcher, err := newMatcher([]string{"*.txt", "docs/*"}, []string{"secret*"}) + if err != nil { + t.Fatal(err) + } + if !matcher.Match("notes.txt") { + t.Fatalf("expected notes.txt to match") + } + if matcher.Match("image.png") { + t.Fatalf("expected image.png not to match") + } + if matcher.Match("secret-plan.txt") { + t.Fatalf("expected secret-plan.txt to be excluded") + } +} diff --git a/cmd/simples3/flags.go b/cmd/simples3/flags.go new file mode 100644 index 0000000..ff06835 --- /dev/null +++ b/cmd/simples3/flags.go @@ -0,0 +1,39 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" +) + +var flagErrHelp = errors.New("help requested") + +type stringListFlag []string + +func (f *stringListFlag) String() string { + return fmt.Sprintf("%v", []string(*f)) +} + +func (f *stringListFlag) Set(value string) error { + *f = append(*f, value) + return nil +} + +func newFlagSet(name string, usage func()) *flag.FlagSet { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.Usage = usage + return fs +} + +func parseFlagSet(fs *flag.FlagSet, args []string) error { + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + fs.Usage() + return flagErrHelp + } + return usageErrorf("%v", err) + } + return nil +} diff --git a/cmd/simples3/integration_test.go b/cmd/simples3/integration_test.go new file mode 100644 index 0000000..17bf7e1 --- /dev/null +++ b/cmd/simples3/integration_test.go @@ -0,0 +1,457 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rhnvrm/simples3" +) + +const cliIntegrationEnv = "SIMPLES3_CLI_INTEGRATION" + +type cliIntegrationHarness struct { + t *testing.T + env map[string]string + client *simples3.S3 + bucket string + homeDir string + endpoint string +} + +func newCLIIntegrationHarness(t *testing.T) *cliIntegrationHarness { + t.Helper() + + if os.Getenv(cliIntegrationEnv) != "1" { + t.Skipf("set %s=1 to run MinIO-backed CLI integration tests", cliIntegrationEnv) + } + + endpoint := os.Getenv("AWS_S3_ENDPOINT") + accessKey := firstNonEmpty(os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_S3_ACCESS_KEY")) + secretKey := firstNonEmpty(os.Getenv("AWS_SECRET_ACCESS_KEY"), os.Getenv("AWS_S3_SECRET_KEY")) + region := firstNonEmpty(os.Getenv("AWS_REGION"), os.Getenv("AWS_S3_REGION"), "us-east-1") + if endpoint == "" || accessKey == "" || secretKey == "" { + t.Skip("CLI integration tests require AWS_S3_ENDPOINT and AWS credentials for MinIO") + } + + client := simples3.New(region, accessKey, secretKey) + client.SetEndpoint(endpoint) + + bucket := fmt.Sprintf("simples3-cli-%d", time.Now().UnixNano()) + if _, err := client.CreateBucket(simples3.CreateBucketInput{Bucket: bucket, Region: region}); err != nil { + t.Fatalf("create integration bucket %s: %v", bucket, err) + } + + homeDir := t.TempDir() + h := &cliIntegrationHarness{ + t: t, + client: client, + bucket: bucket, + homeDir: homeDir, + endpoint: endpoint, + env: map[string]string{ + "AWS_ACCESS_KEY_ID": accessKey, + "AWS_SECRET_ACCESS_KEY": secretKey, + "AWS_REGION": region, + "AWS_DEFAULT_REGION": region, + "AWS_S3_ENDPOINT": endpoint, + "AWS_EC2_METADATA_DISABLED": "true", + }, + } + + t.Cleanup(func() { + h.cleanupBucket() + }) + + return h +} + +func (h *cliIntegrationHarness) cleanupBucket() { + h.t.Helper() + for attempt := 0; attempt < 5; attempt++ { + removed := false + versions, err := h.client.ListVersions(simples3.ListVersionsInput{Bucket: h.bucket}) + if err == nil { + for _, version := range versions.Versions { + removed = true + if err := h.client.FileDelete(simples3.DeleteInput{Bucket: h.bucket, ObjectKey: version.Key, VersionId: version.VersionId}); err != nil && !strings.Contains(strings.ToLower(err.Error()), "not found") { + h.t.Fatalf("delete cleanup object version %s from %s: %v", version.Key, h.bucket, err) + } + } + for _, marker := range versions.DeleteMarkers { + removed = true + if err := h.client.FileDelete(simples3.DeleteInput{Bucket: h.bucket, ObjectKey: marker.Key, VersionId: marker.VersionId}); err != nil && !strings.Contains(strings.ToLower(err.Error()), "not found") { + h.t.Fatalf("delete cleanup delete-marker %s from %s: %v", marker.Key, h.bucket, err) + } + } + } + + seq, finish := h.client.ListAll(simples3.ListInput{Bucket: h.bucket}) + keys := make([]string, 0) + for object := range seq { + keys = append(keys, object.Key) + } + if err := finish(); err != nil && !strings.Contains(strings.ToLower(err.Error()), "not found") { + h.t.Fatalf("list cleanup bucket %s: %v", h.bucket, err) + } + if len(keys) == 0 && !removed { + break + } + for start := 0; start < len(keys); start += 1000 { + removed = true + end := start + 1000 + if end > len(keys) { + end = len(keys) + } + if _, err := h.client.DeleteObjects(simples3.DeleteObjectsInput{Bucket: h.bucket, Objects: keys[start:end], Quiet: true}); err != nil { + h.t.Fatalf("delete cleanup objects from %s: %v", h.bucket, err) + } + } + if !removed { + break + } + } + if err := h.client.DeleteBucket(simples3.DeleteBucketInput{Bucket: h.bucket}); err != nil && !strings.Contains(strings.ToLower(err.Error()), "not found") { + h.t.Fatalf("delete cleanup bucket %s: %v", h.bucket, err) + } +} + +func (h *cliIntegrationHarness) run(args ...string) (int, string, string) { + return h.runWithInput("", args...) +} + +func (h *cliIntegrationHarness) runWithInput(input string, args ...string) (int, string, string) { + h.t.Helper() + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + rt := &runtime{ + stdin: strings.NewReader(input), + stdout: stdout, + stderr: stderr, + getenv: func(key string) string { return h.env[key] }, + homeDir: func() (string, error) { + return h.homeDir, nil + }, + } + code := rt.run(args) + return code, stdout.String(), stderr.String() +} + +func (h *cliIntegrationHarness) mustRun(args ...string) string { + h.t.Helper() + code, stdout, stderr := h.run(args...) + if code != 0 { + h.t.Fatalf("command failed (%v): stdout=%s stderr=%s", args, stdout, stderr) + } + return stdout +} + +func (h *cliIntegrationHarness) mustRunWithInput(input string, args ...string) string { + h.t.Helper() + code, stdout, stderr := h.runWithInput(input, args...) + if code != 0 { + h.t.Fatalf("command failed (%v): stdout=%s stderr=%s", args, stdout, stderr) + } + return stdout +} + +func (h *cliIntegrationHarness) putObject(key, content string) { + h.t.Helper() + _, err := h.client.FilePut(simples3.UploadInput{ + Bucket: h.bucket, + ObjectKey: key, + ContentType: "text/plain", + Body: strings.NewReader(content), + }) + if err != nil { + h.t.Fatalf("put object %s: %v", key, err) + } +} + +func (h *cliIntegrationHarness) readObject(key string) string { + h.t.Helper() + body, err := h.client.FileDownload(simples3.DownloadInput{Bucket: h.bucket, ObjectKey: key}) + if err != nil { + h.t.Fatalf("download object %s: %v", key, err) + } + defer body.Close() + data, err := io.ReadAll(body) + if err != nil { + h.t.Fatalf("read object %s: %v", key, err) + } + return string(data) +} + +func (h *cliIntegrationHarness) objectExists(key string) bool { + h.t.Helper() + _, err := h.client.FileDetails(simples3.DetailsInput{Bucket: h.bucket, ObjectKey: key}) + return err == nil +} + +func TestCLIIntegrationBucketLifecycle(t *testing.T) { + h := newCLIIntegrationHarness(t) + bucket := fmt.Sprintf("simples3-cli-extra-%d", time.Now().UnixNano()) + + stdout := h.mustRun("mb", "s3://"+bucket) + if !strings.Contains(stdout, "created s3://"+bucket) { + t.Fatalf("unexpected mb output: %s", stdout) + } + + stdout = h.mustRun("ls", "--json") + if !strings.Contains(stdout, bucket) { + t.Fatalf("expected bucket %s in ls output: %s", bucket, stdout) + } + + stdout = h.mustRun("rb", "s3://"+bucket) + if !strings.Contains(stdout, "deleted s3://"+bucket) { + t.Fatalf("unexpected rb output: %s", stdout) + } +} + +func TestCLIIntegrationCopyMoveRemoveList(t *testing.T) { + h := newCLIIntegrationHarness(t) + + sourceDir := t.TempDir() + sourceFile := filepath.Join(sourceDir, "hello.txt") + if err := os.WriteFile(sourceFile, []byte("hello from cli"), 0o644); err != nil { + t.Fatal(err) + } + + h.mustRun("cp", sourceFile, fmt.Sprintf("s3://%s/uploads/", h.bucket)) + if got := h.readObject("uploads/hello.txt"); got != "hello from cli" { + t.Fatalf("unexpected uploaded content: %q", got) + } + + stdout := h.mustRun("ls", "--json", fmt.Sprintf("s3://%s/uploads/", h.bucket)) + if !strings.Contains(stdout, "uploads/hello.txt") { + t.Fatalf("expected uploaded key in ls output: %s", stdout) + } + + downloadDir := t.TempDir() + h.mustRun("cp", fmt.Sprintf("s3://%s/uploads/hello.txt", h.bucket), downloadDir+string(filepath.Separator)) + data, err := os.ReadFile(filepath.Join(downloadDir, "hello.txt")) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello from cli" { + t.Fatalf("unexpected downloaded content: %q", string(data)) + } + + h.mustRun("mv", fmt.Sprintf("s3://%s/uploads/hello.txt", h.bucket), fmt.Sprintf("s3://%s/archive/hello.txt", h.bucket)) + if h.objectExists("uploads/hello.txt") { + t.Fatalf("expected source object to be removed after mv") + } + if got := h.readObject("archive/hello.txt"); got != "hello from cli" { + t.Fatalf("unexpected moved content: %q", got) + } + + stdout = h.mustRun("rm", fmt.Sprintf("s3://%s/archive/hello.txt", h.bucket)) + if !strings.Contains(stdout, "deleted s3://"+h.bucket+"/archive/hello.txt") { + t.Fatalf("unexpected rm output: %s", stdout) + } + if h.objectExists("archive/hello.txt") { + t.Fatalf("expected object to be removed after rm") + } +} + +func TestCLIIntegrationSyncAndPresign(t *testing.T) { + h := newCLIIntegrationHarness(t) + + sourceDir := t.TempDir() + freshFile := filepath.Join(sourceDir, "fresh.txt") + sameSizeFile := filepath.Join(sourceDir, "same-size.txt") + if err := os.WriteFile(freshFile, []byte("fresh-data"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sameSizeFile, []byte("fresh"), 0o644); err != nil { + t.Fatal(err) + } + h.putObject("sync/same-size.txt", "stale") + h.putObject("sync/stale.txt", "remove-me") + time.Sleep(1100 * time.Millisecond) + now := time.Now() + if err := os.Chtimes(sameSizeFile, now, now); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(freshFile, now, now); err != nil { + t.Fatal(err) + } + + stdout := h.mustRun("sync", "--delete", sourceDir, fmt.Sprintf("s3://%s/sync/", h.bucket)) + if !strings.Contains(stdout, "synced ") || !strings.Contains(stdout, "deleted s3://"+h.bucket+"/sync/stale.txt") { + t.Fatalf("unexpected sync output: %s", stdout) + } + if got := h.readObject("sync/same-size.txt"); got != "fresh" { + t.Fatalf("expected sync to replace same-size object, got %q", got) + } + if got := h.readObject("sync/fresh.txt"); got != "fresh-data" { + t.Fatalf("unexpected synced content: %q", got) + } + if h.objectExists("sync/stale.txt") { + t.Fatalf("expected stale object to be deleted by sync") + } + + h.putObject("presign.txt", "presigned hello") + stdout = strings.TrimSpace(h.mustRun("presign", fmt.Sprintf("s3://%s/presign.txt", h.bucket))) + resp, err := http.Get(stdout) + if err != nil { + t.Fatalf("GET presigned URL: %v", err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read presigned response: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 from presigned URL, got %d: %s", resp.StatusCode, string(body)) + } + if string(body) != "presigned hello" { + t.Fatalf("unexpected presigned body: %q", string(body)) + } +} + +func TestCLIIntegrationTags(t *testing.T) { + h := newCLIIntegrationHarness(t) + objectURI := fmt.Sprintf("s3://%s/meta/tagged.txt", h.bucket) + h.putObject("meta/tagged.txt", "hello tags") + + stdout := h.mustRun("tags", "set", "--json", "--tag", "env=test", "--tag", "team=platform", objectURI) + var setOutput tagsResult + decodeJSON(t, stdout, &setOutput) + if setOutput.Status != "updated" || setOutput.Tags["env"] != "test" || setOutput.Tags["team"] != "platform" { + t.Fatalf("unexpected tags set output: %+v", setOutput) + } + + stdout = h.mustRun("tags", "get", "--json", objectURI) + var getOutput tagsResult + decodeJSON(t, stdout, &getOutput) + if getOutput.Status != "loaded" || len(getOutput.Tags) != 2 || getOutput.Tags["team"] != "platform" { + t.Fatalf("unexpected tags get output: %+v", getOutput) + } + + stdout = h.mustRun("tags", "delete", "--json", objectURI) + var deleteOutput tagsResult + decodeJSON(t, stdout, &deleteOutput) + if deleteOutput.Status != "deleted" { + t.Fatalf("unexpected tags delete output: %+v", deleteOutput) + } + + stdout = h.mustRun("tags", "get", "--json", objectURI) + var finalGetOutput tagsResult + decodeJSON(t, stdout, &finalGetOutput) + if len(finalGetOutput.Tags) != 0 { + t.Fatalf("expected tags to be empty after delete, got %+v", finalGetOutput.Tags) + } +} + +func TestCLIIntegrationVersioningAndVersions(t *testing.T) { + h := newCLIIntegrationHarness(t) + bucketURI := fmt.Sprintf("s3://%s", h.bucket) + objectKey := "history/object.txt" + objectURI := fmt.Sprintf("s3://%s/%s", h.bucket, objectKey) + + stdout := h.mustRun("versioning", "set", "--json", "--status", "enabled", bucketURI) + var setOutput versioningResult + decodeJSON(t, stdout, &setOutput) + if setOutput.Status != "Enabled" { + t.Fatalf("unexpected versioning set output: %+v", setOutput) + } + + stdout = h.mustRun("versioning", "get", "--json", bucketURI) + var getOutput versioningResult + decodeJSON(t, stdout, &getOutput) + if getOutput.Status != "Enabled" { + t.Fatalf("unexpected versioning get output: %+v", getOutput) + } + + h.putObject(objectKey, "v1") + time.Sleep(1100 * time.Millisecond) + h.putObject(objectKey, "v2") + + stdout = h.mustRun("versions", "--json", objectURI) + var versionsOutput versionsResult + decodeJSON(t, stdout, &versionsOutput) + if versionsOutput.Bucket != h.bucket || len(versionsOutput.Versions) < 2 { + t.Fatalf("unexpected versions output: %+v", versionsOutput) + } + latestCount := 0 + for _, version := range versionsOutput.Versions { + if version.IsLatest { + latestCount++ + } + } + if latestCount != 1 { + t.Fatalf("expected exactly one latest version, got %d in %+v", latestCount, versionsOutput.Versions) + } +} + +func TestCLIIntegrationLifecycleAndACL(t *testing.T) { + h := newCLIIntegrationHarness(t) + bucketURI := fmt.Sprintf("s3://%s", h.bucket) + objectURI := fmt.Sprintf("s3://%s/acl/object.txt", h.bucket) + lifecycleInput := `{"rules":[{"id":"expire-logs","status":"Enabled","filter":{"prefix":"logs/"},"expiration":{"days":30}}]}` + + stdout := h.mustRunWithInput(lifecycleInput, "lifecycle", "set", "--json", "--file", "-", bucketURI) + var lifecycleSet lifecycleResult + decodeJSON(t, stdout, &lifecycleSet) + if lifecycleSet.Status != "updated" || lifecycleSet.Configuration == nil || len(lifecycleSet.Configuration.Rules) != 1 { + t.Fatalf("unexpected lifecycle set output: %+v", lifecycleSet) + } + + stdout = h.mustRun("lifecycle", "get", "--json", bucketURI) + var lifecycleGet lifecycleResult + decodeJSON(t, stdout, &lifecycleGet) + if lifecycleGet.Status != "loaded" || lifecycleGet.Configuration == nil || len(lifecycleGet.Configuration.Rules) != 1 || lifecycleGet.Configuration.Rules[0].Filter == nil || lifecycleGet.Configuration.Rules[0].Filter.Prefix != "logs/" { + t.Fatalf("unexpected lifecycle get output: %+v", lifecycleGet) + } + + stdout = h.mustRun("lifecycle", "delete", "--json", bucketURI) + var lifecycleDelete lifecycleResult + decodeJSON(t, stdout, &lifecycleDelete) + if lifecycleDelete.Status != "deleted" { + t.Fatalf("unexpected lifecycle delete output: %+v", lifecycleDelete) + } + + h.putObject("acl/object.txt", "hello acl") + stdout = h.mustRun("acl", "get", "--json", bucketURI) + var aclGet aclResult + decodeJSON(t, stdout, &aclGet) + if aclGet.Status != "loaded" || aclGet.Policy == nil || len(aclGet.Policy.Grants) == 0 { + t.Fatalf("unexpected bucket acl get output: %+v", aclGet) + } + + policyBytes, err := json.Marshal(aclGet.Policy) + if err != nil { + t.Fatalf("marshal ACL policy: %v", err) + } + policyFile := filepath.Join(t.TempDir(), "bucket-acl.json") + if err := os.WriteFile(policyFile, policyBytes, 0o644); err != nil { + t.Fatalf("write ACL policy file: %v", err) + } + code, stdout, stderr := h.run("acl", "set", "--json", "--policy-file", policyFile, bucketURI) + if code != 0 { + if strings.Contains(stdout, "501 Not Implemented") || strings.Contains(stderr, "501 Not Implemented") { + t.Skip("backend does not support ACL updates") + } + t.Fatalf("acl set failed: code=%d stdout=%s stderr=%s", code, stdout, stderr) + } + var aclSet aclResult + decodeJSON(t, stdout, &aclSet) + if aclSet.Status != "updated" || aclSet.Policy == nil || len(aclSet.Policy.Grants) == 0 { + t.Fatalf("unexpected bucket acl set output: %+v", aclSet) + } + + stdout = h.mustRun("acl", "get", "--json", objectURI) + var objectACL aclResult + decodeJSON(t, stdout, &objectACL) + if objectACL.Status != "loaded" || objectACL.Policy == nil || len(objectACL.Policy.Grants) == 0 { + t.Fatalf("unexpected object acl get output: %+v", objectACL) + } +} diff --git a/cmd/simples3/location.go b/cmd/simples3/location.go new file mode 100644 index 0000000..a59951f --- /dev/null +++ b/cmd/simples3/location.go @@ -0,0 +1,101 @@ +package main + +import ( + "net/url" + "path" + "path/filepath" + "strings" +) + +type locationKind string + +const ( + locationKindLocal locationKind = "local" + locationKindS3 locationKind = "s3" +) + +type location struct { + kind locationKind + raw string + path string + bucket string + key string + hasTrailing bool +} + +func parseLocation(raw string) (location, error) { + if strings.TrimSpace(raw) == "" { + return location{}, usageErrorf("empty path") + } + if strings.HasPrefix(raw, "s3://") { + parsed, err := url.Parse(raw) + if err != nil { + return location{}, usageErrorf("invalid s3 location %q: %v", raw, err) + } + bucket := parsed.Host + if bucket == "" { + return location{}, usageErrorf("missing bucket in %q", raw) + } + key := strings.TrimPrefix(parsed.EscapedPath(), "/") + decodedKey, err := url.PathUnescape(key) + if err != nil { + return location{}, usageErrorf("invalid s3 key in %q: %v", raw, err) + } + return location{ + kind: locationKindS3, + raw: raw, + bucket: bucket, + key: decodedKey, + hasTrailing: strings.HasSuffix(raw, "/") || decodedKey == "", + }, nil + } + cleaned := filepath.Clean(raw) + return location{ + kind: locationKindLocal, + raw: raw, + path: cleaned, + hasTrailing: strings.HasSuffix(raw, string(filepath.Separator)), + }, nil +} + +func (loc location) String() string { + if loc.kind == locationKindS3 { + if loc.key == "" { + return "s3://" + loc.bucket + } + return "s3://" + loc.bucket + "/" + loc.key + } + return loc.path +} + +func (loc location) isS3() bool { + return loc.kind == locationKindS3 +} + +func (loc location) isLocal() bool { + return loc.kind == locationKindLocal +} + +func (loc location) s3Prefix() string { + if loc.key == "" { + return "" + } + if loc.hasTrailing { + return strings.TrimSuffix(loc.key, "/") + "/" + } + return loc.key +} + +func joinS3Key(parts ...string) string { + clean := make([]string, 0, len(parts)) + for _, part := range parts { + if part == "" { + continue + } + clean = append(clean, strings.Trim(part, "/")) + } + if len(clean) == 0 { + return "" + } + return path.Join(clean...) +} diff --git a/cmd/simples3/location_test.go b/cmd/simples3/location_test.go new file mode 100644 index 0000000..839bb01 --- /dev/null +++ b/cmd/simples3/location_test.go @@ -0,0 +1,32 @@ +package main + +import "testing" + +func TestParseLocationS3(t *testing.T) { + loc, err := parseLocation("s3://example-bucket/path/to/file.txt") + if err != nil { + t.Fatal(err) + } + if !loc.isS3() { + t.Fatalf("expected S3 location") + } + if loc.bucket != "example-bucket" || loc.key != "path/to/file.txt" { + t.Fatalf("unexpected parsed location: %+v", loc) + } +} + +func TestParseLocationLocal(t *testing.T) { + loc, err := parseLocation("./tmp/file.txt") + if err != nil { + t.Fatal(err) + } + if !loc.isLocal() { + t.Fatalf("expected local location") + } +} + +func TestJoinS3Key(t *testing.T) { + if got := joinS3Key("prefix/", "/child", "file.txt"); got != "prefix/child/file.txt" { + t.Fatalf("unexpected key: %s", got) + } +} diff --git a/cmd/simples3/main.go b/cmd/simples3/main.go new file mode 100644 index 0000000..bc326ac --- /dev/null +++ b/cmd/simples3/main.go @@ -0,0 +1,179 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +type runtime struct { + stdin io.Reader + stdout io.Writer + stderr io.Writer + getenv func(string) string + homeDir func() (string, error) +} + +func newRuntime(stdout, stderr io.Writer) *runtime { + return &runtime{ + stdin: os.Stdin, + stdout: stdout, + stderr: stderr, + getenv: os.Getenv, + homeDir: os.UserHomeDir, + } +} + +func main() { + os.Exit(newRuntime(os.Stdout, os.Stderr).run(os.Args[1:])) +} + +func (rt *runtime) run(args []string) int { + if len(args) == 0 { + rt.printUsage() + return 1 + } + + cmd := args[0] + cmdArgs := args[1:] + jsonMode := wantsJSON(cmdArgs) + + var err error + switch cmd { + case "help", "-h", "--help": + rt.printUsage() + return 0 + case "ls": + err = rt.runList(cmdArgs) + case "mb": + err = rt.runMakeBucket(cmdArgs) + case "rb": + err = rt.runRemoveBucket(cmdArgs) + case "presign": + err = rt.runPresign(cmdArgs) + case "cp": + err = rt.runCopy(cmdArgs) + case "rm": + err = rt.runRemove(cmdArgs) + case "mv": + err = rt.runMove(cmdArgs) + case "sync": + err = rt.runSync(cmdArgs) + case "tags": + err = rt.runTags(cmdArgs) + case "versioning": + err = rt.runVersioning(cmdArgs) + case "versions": + err = rt.runVersions(cmdArgs) + case "lifecycle": + err = rt.runLifecycle(cmdArgs) + case "acl": + err = rt.runACL(cmdArgs) + default: + err = usageErrorf("unknown command %q", cmd) + } + + if err == nil { + return 0 + } + if errors.Is(err, flagErrHelp) { + return 0 + } + if jsonMode { + if encodeErr := writeJSON(rt.stdout, errorOutputForCommand(cmd, err)); encodeErr != nil { + fmt.Fprintf(rt.stderr, "error: %v\n", err) + return 1 + } + } else { + fmt.Fprintf(rt.stderr, "error: %v\n", err) + } + return exitCodeForError(err) +} + +func wantsJSON(args []string) bool { + jsonMode := false + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--": + return jsonMode + case arg == "-json" || arg == "--json": + jsonMode = true + case strings.HasPrefix(arg, "-json=") || strings.HasPrefix(arg, "--json="): + _, value, _ := strings.Cut(arg, "=") + parsed, err := strconv.ParseBool(value) + if err != nil { + return true + } + jsonMode = parsed + case strings.HasPrefix(arg, "-"): + name, _, hasInlineValue := strings.Cut(arg, "=") + if expectsFlagValue(name) && !hasInlineValue && i+1 < len(args) { + i++ + } + default: + return jsonMode + } + } + return jsonMode +} + +func expectsFlagValue(name string) bool { + switch name { + case "-profile", "--profile", + "-region", "--region", + "-endpoint", "--endpoint", + "-include", "--include", + "-exclude", "--exclude", + "-acl", "--acl", + "-sse", "--sse", + "-sse-kms-key-id", "--sse-kms-key-id", + "-version-id", "--version-id", + "-method", "--method", + "-expires", "--expires", + "-response-content-disposition", "--response-content-disposition", + "-concurrency", "--concurrency", + "-retries", "--retries", + "-tag", "--tag", + "-tags-file", "--tags-file", + "-status", "--status", + "-mfa-delete", "--mfa-delete", + "-delimiter", "--delimiter", + "-max-keys", "--max-keys", + "-key-marker", "--key-marker", + "-version-id-marker", "--version-id-marker", + "-file", "--file", + "-policy-file", "--policy-file": + return true + default: + return false + } +} + +func (rt *runtime) printUsage() { + fmt.Fprint(rt.stderr, `simples3 - simple S3 CLI + +Usage: + simples3 [flags] [arguments] + +Commands: + ls list buckets or objects + mb make bucket + rb remove bucket + presign generate a presigned object URL + cp copy local files and S3 objects + rm remove S3 objects and prefixes + mv move local files and S3 objects + sync synchronize source to destination + tags get, set, or delete object tags + versioning get or set bucket versioning + versions list object versions and delete markers + lifecycle get, set, or delete bucket lifecycle config + acl get or set bucket or object ACLs + +Run 'simples3 help' to see this message. +`) +} diff --git a/cmd/simples3/output.go b/cmd/simples3/output.go new file mode 100644 index 0000000..801bb1e --- /dev/null +++ b/cmd/simples3/output.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "strings" +) + +type commandEnvelope struct { + Command string `json:"command"` + OK bool `json:"ok"` +} + +type commandErrorDetail struct { + Type string `json:"type"` + Message string `json:"message"` + ExitCode int `json:"exitCode"` +} + +type commandErrorOutput struct { + Command string `json:"command"` + OK bool `json:"ok"` + Error commandErrorDetail `json:"error"` + Data any `json:"data,omitempty"` +} + +type operationSummary struct { + Total int `json:"total"` + Changed int `json:"changed,omitempty"` + Deleted int `json:"deleted,omitempty"` + Failed int `json:"failed,omitempty"` + Unchanged int `json:"unchanged,omitempty"` + DryRun bool `json:"dryRun,omitempty"` + Noop bool `json:"noop,omitempty"` +} + +type operationsData struct { + Summary operationSummary `json:"summary"` + Operations []operationResult `json:"operations"` +} + +type operationsOutput struct { + commandEnvelope + operationsData +} + +func writeJSON(out io.Writer, value any) error { + encoder := json.NewEncoder(out) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +func printLine(out io.Writer, format string, args ...any) { + fmt.Fprintf(out, format+"\n", args...) +} + +func printOperation(out io.Writer, result operationResult) { + suffix := "" + if result.Error != "" { + suffix = " (" + result.Error + ")" + } + if result.Destination != "" { + printLine(out, "%s %s -> %s%s", result.Status, result.Source, result.Destination, suffix) + return + } + printLine(out, "%s %s%s", result.Status, result.Source, suffix) +} + +func printOperations(out io.Writer, results []operationResult) { + for _, result := range results { + printOperation(out, result) + } +} + +func buildOperationsData(results []operationResult) operationsData { + return operationsData{Summary: summarizeOperations(results), Operations: results} +} + +func writeOperationsJSON(out io.Writer, command string, results []operationResult) error { + return writeJSON(out, operationsOutput{ + commandEnvelope: commandEnvelope{Command: command, OK: true}, + operationsData: buildOperationsData(results), + }) +} + +func summarizeOperations(results []operationResult) operationSummary { + summary := operationSummary{Total: len(results)} + for _, result := range results { + if result.DryRun { + summary.DryRun = true + } + switch { + case isDeleteStatus(result.Status): + summary.Deleted++ + case isChangedStatus(result.Status): + summary.Changed++ + case isFailureStatus(result.Status): + summary.Failed++ + case isUnchangedStatus(result.Status): + summary.Unchanged++ + } + } + summary.Noop = summary.Total == 0 || (summary.Changed == 0 && summary.Deleted == 0 && summary.Failed == 0) + return summary +} + +func isChangedStatus(status string) bool { + return status == "copied" || status == "moved" || status == "synced" || status == "would-copy" || status == "would-move" || status == "would-sync" +} + +func isDeleteStatus(status string) bool { + return status == "deleted" || status == "would-delete" +} + +func isFailureStatus(status string) bool { + return status == "failed" || strings.HasPrefix(status, "failed-") +} + +func isUnchangedStatus(status string) bool { + return status == "unchanged" +} + +func hasFailures(results []operationResult) bool { + for _, result := range results { + if isFailureStatus(result.Status) { + return true + } + } + return false +} diff --git a/cmd/simples3/payload.go b/cmd/simples3/payload.go new file mode 100644 index 0000000..86a3d63 --- /dev/null +++ b/cmd/simples3/payload.go @@ -0,0 +1,363 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/rhnvrm/simples3" +) + +type keyValueFlag map[string]string + +func (f *keyValueFlag) String() string { + if f == nil { + return "" + } + pairs := make([]string, 0, len(*f)) + for key, value := range *f { + pairs = append(pairs, key+"="+value) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (f *keyValueFlag) Set(value string) error { + key, val, ok := strings.Cut(value, "=") + if !ok || key == "" { + return fmt.Errorf("expected key=value") + } + if *f == nil { + *f = map[string]string{} + } + (*f)[key] = val + return nil +} + +type ownerDocument struct { + ID string `json:"id,omitempty"` + DisplayName string `json:"displayName,omitempty"` +} + +type granteeDocument struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + DisplayName string `json:"displayName,omitempty"` + URI string `json:"uri,omitempty"` + EmailAddress string `json:"emailAddress,omitempty"` +} + +type grantDocument struct { + Grantee granteeDocument `json:"grantee"` + Permission string `json:"permission"` +} + +type accessControlPolicyDocument struct { + Owner ownerDocument `json:"owner"` + Grants []grantDocument `json:"grants"` +} + +type tagDocument struct { + Key string `json:"key"` + Value string `json:"value"` +} + +type lifecycleAndDocument struct { + Prefix string `json:"prefix,omitempty"` + Tags []tagDocument `json:"tags,omitempty"` +} + +type lifecycleFilterDocument struct { + Prefix string `json:"prefix,omitempty"` + Tag *tagDocument `json:"tag,omitempty"` + And *lifecycleAndDocument `json:"and,omitempty"` +} + +type lifecycleExpirationDocument struct { + Date string `json:"date,omitempty"` + Days int `json:"days,omitempty"` + ExpiredObjectDeleteMarker bool `json:"expiredObjectDeleteMarker,omitempty"` +} + +type lifecycleTransitionDocument struct { + Date string `json:"date,omitempty"` + Days int `json:"days,omitempty"` + StorageClass string `json:"storageClass"` +} + +type lifecycleNoncurrentVersionExpirationDocument struct { + NoncurrentDays int `json:"noncurrentDays"` +} + +type lifecycleNoncurrentVersionTransitionDocument struct { + NoncurrentDays int `json:"noncurrentDays"` + StorageClass string `json:"storageClass"` +} + +type lifecycleAbortIncompleteMultipartUploadDocument struct { + DaysAfterInitiation int `json:"daysAfterInitiation"` +} + +type lifecycleRuleDocument struct { + ID string `json:"id,omitempty"` + Status string `json:"status"` + Filter *lifecycleFilterDocument `json:"filter,omitempty"` + Prefix *string `json:"prefix,omitempty"` + Expiration *lifecycleExpirationDocument `json:"expiration,omitempty"` + Transitions []lifecycleTransitionDocument `json:"transitions,omitempty"` + NoncurrentVersionExpiration *lifecycleNoncurrentVersionExpirationDocument `json:"noncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransitions []lifecycleNoncurrentVersionTransitionDocument `json:"noncurrentVersionTransitions,omitempty"` + AbortIncompleteMultipartUpload *lifecycleAbortIncompleteMultipartUploadDocument `json:"abortIncompleteMultipartUpload,omitempty"` +} + +type lifecycleConfigurationDocument struct { + Rules []lifecycleRuleDocument `json:"rules"` +} + +func ownerDocumentFromOwner(owner simples3.Owner) ownerDocument { + return ownerDocument{ID: owner.ID, DisplayName: owner.DisplayName} +} + +func (doc ownerDocument) toOwner() simples3.Owner { + return simples3.Owner{ID: doc.ID, DisplayName: doc.DisplayName} +} + +func granteeDocumentFromGrantee(grantee simples3.Grantee) granteeDocument { + return granteeDocument{ + Type: grantee.Type, + ID: grantee.ID, + DisplayName: grantee.DisplayName, + URI: grantee.URI, + EmailAddress: grantee.EmailAddress, + } +} + +func (doc granteeDocument) toGrantee() simples3.Grantee { + return simples3.Grantee{ + Type: doc.Type, + ID: doc.ID, + DisplayName: doc.DisplayName, + URI: doc.URI, + EmailAddress: doc.EmailAddress, + } +} + +func grantDocumentFromGrant(grant simples3.Grant) grantDocument { + return grantDocument{Grantee: granteeDocumentFromGrantee(grant.Grantee), Permission: grant.Permission} +} + +func (doc grantDocument) toGrant() simples3.Grant { + return simples3.Grant{Grantee: doc.Grantee.toGrantee(), Permission: doc.Permission} +} + +func aclDocumentFromPolicy(policy simples3.AccessControlPolicy) accessControlPolicyDocument { + grants := make([]grantDocument, 0, len(policy.AccessControlList)) + for _, grant := range policy.AccessControlList { + grants = append(grants, grantDocumentFromGrant(grant)) + } + return accessControlPolicyDocument{Owner: ownerDocumentFromOwner(policy.Owner), Grants: grants} +} + +func (doc accessControlPolicyDocument) toPolicy() *simples3.AccessControlPolicy { + grants := make([]simples3.Grant, 0, len(doc.Grants)) + for _, grant := range doc.Grants { + grants = append(grants, grant.toGrant()) + } + return &simples3.AccessControlPolicy{Owner: doc.Owner.toOwner(), AccessControlList: grants} +} + +func tagDocumentFromTag(tag simples3.Tag) tagDocument { + return tagDocument{Key: tag.Key, Value: tag.Value} +} + +func (doc tagDocument) toTag() simples3.Tag { + return simples3.Tag{Key: doc.Key, Value: doc.Value} +} + +func lifecycleFilterDocumentFromFilter(filter *simples3.LifecycleFilter) *lifecycleFilterDocument { + if filter == nil { + return nil + } + doc := &lifecycleFilterDocument{Prefix: filter.Prefix} + if filter.Tag != nil { + tag := tagDocumentFromTag(*filter.Tag) + doc.Tag = &tag + } + if filter.And != nil { + and := &lifecycleAndDocument{Prefix: filter.And.Prefix, Tags: make([]tagDocument, 0, len(filter.And.Tags))} + for _, tag := range filter.And.Tags { + and.Tags = append(and.Tags, tagDocumentFromTag(tag)) + } + doc.And = and + } + return doc +} + +func (doc *lifecycleFilterDocument) toFilter() *simples3.LifecycleFilter { + if doc == nil { + return nil + } + filter := &simples3.LifecycleFilter{Prefix: doc.Prefix} + if doc.Tag != nil { + tag := doc.Tag.toTag() + filter.Tag = &tag + } + if doc.And != nil { + and := &struct { + Prefix string `xml:"Prefix,omitempty"` + Tags []simples3.Tag `xml:"Tag,omitempty"` + }{ + Prefix: doc.And.Prefix, + Tags: make([]simples3.Tag, 0, len(doc.And.Tags)), + } + for _, tag := range doc.And.Tags { + and.Tags = append(and.Tags, tag.toTag()) + } + filter.And = and + } + return filter +} + +func lifecycleRuleDocumentFromRule(rule simples3.LifecycleRule) lifecycleRuleDocument { + doc := lifecycleRuleDocument{ + ID: rule.ID, + Status: rule.Status, + Filter: lifecycleFilterDocumentFromFilter(rule.Filter), + Prefix: rule.Prefix, + Transitions: make([]lifecycleTransitionDocument, 0, len(rule.Transitions)), + NoncurrentVersionTransitions: make([]lifecycleNoncurrentVersionTransitionDocument, 0, len(rule.NoncurrentVersionTransitions)), + AbortIncompleteMultipartUpload: nil, + } + if rule.Expiration != nil { + doc.Expiration = &lifecycleExpirationDocument{ + Date: rule.Expiration.Date, + Days: rule.Expiration.Days, + ExpiredObjectDeleteMarker: rule.Expiration.ExpiredObjectDeleteMarker, + } + } + for _, transition := range rule.Transitions { + doc.Transitions = append(doc.Transitions, lifecycleTransitionDocument{ + Date: transition.Date, + Days: transition.Days, + StorageClass: transition.StorageClass, + }) + } + if rule.NoncurrentVersionExpiration != nil { + doc.NoncurrentVersionExpiration = &lifecycleNoncurrentVersionExpirationDocument{NoncurrentDays: rule.NoncurrentVersionExpiration.NoncurrentDays} + } + for _, transition := range rule.NoncurrentVersionTransitions { + doc.NoncurrentVersionTransitions = append(doc.NoncurrentVersionTransitions, lifecycleNoncurrentVersionTransitionDocument{ + NoncurrentDays: transition.NoncurrentDays, + StorageClass: transition.StorageClass, + }) + } + if rule.AbortIncompleteMultipartUpload != nil { + doc.AbortIncompleteMultipartUpload = &lifecycleAbortIncompleteMultipartUploadDocument{DaysAfterInitiation: rule.AbortIncompleteMultipartUpload.DaysAfterInitiation} + } + return doc +} + +func (doc lifecycleRuleDocument) toRule() simples3.LifecycleRule { + rule := simples3.LifecycleRule{ + ID: doc.ID, + Status: doc.Status, + Filter: doc.Filter.toFilter(), + Prefix: doc.Prefix, + Transitions: make([]simples3.LifecycleTransition, 0, len(doc.Transitions)), + NoncurrentVersionTransitions: make([]simples3.LifecycleNoncurrentVersionTransition, 0, len(doc.NoncurrentVersionTransitions)), + } + if doc.Expiration != nil { + rule.Expiration = &simples3.LifecycleExpiration{ + Date: doc.Expiration.Date, + Days: doc.Expiration.Days, + ExpiredObjectDeleteMarker: doc.Expiration.ExpiredObjectDeleteMarker, + } + } + for _, transition := range doc.Transitions { + rule.Transitions = append(rule.Transitions, simples3.LifecycleTransition{ + Date: transition.Date, + Days: transition.Days, + StorageClass: transition.StorageClass, + }) + } + if doc.NoncurrentVersionExpiration != nil { + rule.NoncurrentVersionExpiration = &simples3.LifecycleNoncurrentVersionExpiration{NoncurrentDays: doc.NoncurrentVersionExpiration.NoncurrentDays} + } + for _, transition := range doc.NoncurrentVersionTransitions { + rule.NoncurrentVersionTransitions = append(rule.NoncurrentVersionTransitions, simples3.LifecycleNoncurrentVersionTransition{ + NoncurrentDays: transition.NoncurrentDays, + StorageClass: transition.StorageClass, + }) + } + if doc.AbortIncompleteMultipartUpload != nil { + rule.AbortIncompleteMultipartUpload = &simples3.LifecycleAbortIncompleteMultipartUpload{DaysAfterInitiation: doc.AbortIncompleteMultipartUpload.DaysAfterInitiation} + } + return rule +} + +func lifecycleDocumentFromConfiguration(config simples3.LifecycleConfiguration) lifecycleConfigurationDocument { + rules := make([]lifecycleRuleDocument, 0, len(config.Rules)) + for _, rule := range config.Rules { + rules = append(rules, lifecycleRuleDocumentFromRule(rule)) + } + return lifecycleConfigurationDocument{Rules: rules} +} + +func (doc lifecycleConfigurationDocument) toConfiguration() *simples3.LifecycleConfiguration { + rules := make([]simples3.LifecycleRule, 0, len(doc.Rules)) + for _, rule := range doc.Rules { + rules = append(rules, rule.toRule()) + } + return &simples3.LifecycleConfiguration{Rules: rules} +} + +func decodeJSONSource(rt *runtime, path string, target any) error { + reader, closer, err := openInputSource(rt, path) + if err != nil { + return err + } + if closer != nil { + defer closer.Close() + } + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return usageErrorf("invalid JSON in %q: %v", path, err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return usageErrorf("invalid JSON in %q: multiple top-level values", path) + } + return usageErrorf("invalid JSON in %q: %v", path, err) + } + return nil +} + +func openInputSource(rt *runtime, path string) (io.Reader, io.Closer, error) { + if path == "-" { + if rt.stdin == nil { + return nil, nil, usageErrorf("stdin is not available") + } + return rt.stdin, nil, nil + } + file, err := os.Open(path) + if err != nil { + return nil, nil, err + } + return file, file, nil +} + +func writeTagLines(out io.Writer, tags map[string]string) { + keys := make([]string, 0, len(tags)) + for key := range tags { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + printLine(out, "%s=%s", key, tags[key]) + } +} diff --git a/cmd/simples3/transfer.go b/cmd/simples3/transfer.go new file mode 100644 index 0000000..6e2a04e --- /dev/null +++ b/cmd/simples3/transfer.go @@ -0,0 +1,785 @@ +package main + +import ( + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/rhnvrm/simples3" +) + +const ( + multipartThreshold int64 = 64 * 1024 * 1024 + multipartPartSize int64 = 16 * 1024 * 1024 +) + +func normalizeSSE(sse, sseKMS string) string { + if sse == "" && sseKMS != "" { + return "aws:kms" + } + return sse +} + +type sourceEntry struct { + Kind locationKind + Local string + Bucket string + Key string + Version string + Relative string + Size int64 + ModTime time.Time +} + +type targetRef struct { + Kind locationKind + Local string + Bucket string + Key string +} + +type operationResult struct { + Action string `json:"action"` + Source string `json:"source,omitempty"` + Destination string `json:"destination,omitempty"` + Status string `json:"status"` + Size int64 `json:"size,omitempty"` + DryRun bool `json:"dryRun,omitempty"` + Error string `json:"error,omitempty"` +} + +type targetMeta struct { + Exists bool + Size int64 + ModTime time.Time +} + +type executionOptions struct { + DryRun bool + Concurrency int + Retries int + ContinueOnError bool +} + +type copyOptions struct { + executionOptions + EmitProgress bool + ACL string + SSE string + SSEKMS string +} + +type plannedOperation struct { + result operationResult + run func() error +} + +func collectSourceEntries(client *simples3.S3, source location, recursive bool, matcher matcher, versionID string) ([]sourceEntry, error) { + if source.isLocal() { + return collectLocalEntries(source, recursive, matcher) + } + return collectS3Entries(client, source, recursive, matcher, versionID) +} + +func collectLocalEntries(source location, recursive bool, matcher matcher) ([]sourceEntry, error) { + info, err := os.Stat(source.path) + if err != nil { + return nil, err + } + if !info.IsDir() { + base := filepath.Base(source.path) + if !matcher.Match(base) { + return nil, nil + } + return []sourceEntry{{ + Kind: locationKindLocal, + Local: source.path, + Relative: filepath.ToSlash(base), + Size: info.Size(), + ModTime: info.ModTime(), + }}, nil + } + if !recursive { + return nil, usageErrorf("%s is a directory; use --recursive", source.path) + } + + entries := []sourceEntry{} + err = filepath.WalkDir(source.path, func(current string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(source.path, current) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if !matcher.Match(rel) { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + entries = append(entries, sourceEntry{ + Kind: locationKindLocal, + Local: current, + Relative: rel, + Size: info.Size(), + ModTime: info.ModTime(), + }) + return nil + }) + if err != nil { + return nil, err + } + return entries, nil +} + +func collectS3Entries(client *simples3.S3, source location, recursive bool, matcher matcher, versionID string) ([]sourceEntry, error) { + prefixMode := recursive || source.key == "" + if prefixMode { + if versionID != "" { + return nil, usageErrorf("--version-id is only supported for a single source object") + } + prefix := s3TraversalPrefix(source, recursive) + entries := []sourceEntry{} + seq, finish := client.ListAll(simples3.ListInput{Bucket: source.bucket, Prefix: prefix}) + for object := range seq { + rel := strings.TrimPrefix(object.Key, prefix) + if rel == "" { + rel = path.Base(object.Key) + } + if !matcher.Match(rel) { + continue + } + entries = append(entries, sourceEntry{ + Kind: locationKindS3, + Bucket: source.bucket, + Key: object.Key, + Relative: rel, + Size: object.Size, + ModTime: parseS3ListTime(object.LastModified), + }) + } + if err := finish(); err != nil { + return nil, err + } + return entries, nil + } + + details, err := client.FileDetails(simples3.DetailsInput{Bucket: source.bucket, ObjectKey: source.key, VersionId: versionID}) + if err != nil { + return nil, err + } + base := path.Base(source.key) + if !matcher.Match(base) { + return nil, nil + } + size, _ := strconv.ParseInt(details.ContentLength, 10, 64) + return []sourceEntry{{ + Kind: locationKindS3, + Bucket: source.bucket, + Key: source.key, + Version: versionID, + Relative: base, + Size: size, + ModTime: parseHTTPTime(details.LastModified), + }}, nil +} + +func s3TraversalPrefix(source location, recursive bool) string { + if source.key == "" { + return "" + } + if recursive { + return strings.TrimSuffix(source.key, "/") + "/" + } + return source.s3Prefix() +} + +func validateRecursiveSource(source location, recursive bool) error { + if source.isS3() && !recursive && (source.key == "" || source.hasTrailing) { + return usageErrorf("%s looks like an S3 prefix; use --recursive", source.String()) + } + return nil +} + +func ensureSingleSourceEntry(source location, recursive bool, entries []sourceEntry) error { + if !recursive && len(entries) > 1 { + return usageErrorf("%s expands to multiple objects; use --recursive", source.String()) + } + return nil +} + +func parseS3ListTime(value string) time.Time { + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{} + } + return t +} + +func parseHTTPTime(value string) time.Time { + t, err := time.Parse(http.TimeFormat, value) + if err != nil { + return time.Time{} + } + return t +} + +func resolveTarget(entry sourceEntry, dest location, recursive bool) (targetRef, error) { + if dest.isS3() { + targetKey := dest.key + if recursive || dest.key == "" || dest.hasTrailing { + targetKey = joinS3Key(dest.key, entry.Relative) + } + return targetRef{Kind: locationKindS3, Bucket: dest.bucket, Key: targetKey}, nil + } + + targetPath := dest.path + if recursive { + targetPath = filepath.Join(dest.path, filepath.FromSlash(entry.Relative)) + return targetRef{Kind: locationKindLocal, Local: targetPath}, nil + } + if dest.hasTrailing || localPathIsDir(dest.path) { + targetPath = filepath.Join(dest.path, filepath.Base(filepath.FromSlash(entry.Relative))) + } + return targetRef{Kind: locationKindLocal, Local: targetPath}, nil +} + +func localPathIsDir(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} + +func entrySourceString(entry sourceEntry) string { + if entry.Kind == locationKindLocal { + return entry.Local + } + if entry.Key == "" { + return "s3://" + entry.Bucket + } + return "s3://" + entry.Bucket + "/" + entry.Key +} + +func targetString(target targetRef) string { + if target.Kind == locationKindLocal { + return target.Local + } + if target.Key == "" { + return "s3://" + target.Bucket + } + return "s3://" + target.Bucket + "/" + target.Key +} + +func detectContentTypeFromPath(path string) string { + contentType := mime.TypeByExtension(filepath.Ext(path)) + if contentType == "" { + return "application/octet-stream" + } + return contentType +} + +func copyEntry(client *simples3.S3, entry sourceEntry, target targetRef, options copyOptions, rt *runtime) error { + if options.DryRun { + return nil + } + switch { + case entry.Kind == locationKindLocal && target.Kind == locationKindS3: + return uploadLocalToS3(client, entry, target, options, rt) + case entry.Kind == locationKindS3 && target.Kind == locationKindLocal: + return downloadS3ToLocal(client, entry, target) + case entry.Kind == locationKindS3 && target.Kind == locationKindS3: + return copyS3ToS3(client, entry, target, options) + default: + return fmt.Errorf("unsupported transfer: %s -> %s", entrySourceString(entry), targetString(target)) + } +} + +func uploadLocalToS3(client *simples3.S3, entry sourceEntry, target targetRef, options copyOptions, rt *runtime) error { + file, err := os.Open(entry.Local) + if err != nil { + return err + } + defer file.Close() + + if entry.Size >= multipartThreshold { + if options.EmitProgress { + printLine(rt.stderr, "uploading %s (%d bytes, multipart)", entry.Local, entry.Size) + } + _, err = client.FileUploadMultipart(simples3.MultipartUploadInput{ + Bucket: target.Bucket, + ObjectKey: target.Key, + Body: file, + ContentType: detectContentTypeFromPath(entry.Local), + ACL: options.ACL, + PartSize: multipartPartSize, + Concurrency: 4, + ServerSideEncryption: options.SSE, + SSEKMSKeyId: options.SSEKMS, + }) + return err + } + + if options.EmitProgress { + printLine(rt.stderr, "uploading %s (%d bytes)", entry.Local, entry.Size) + } + _, err = client.FilePut(simples3.UploadInput{ + Bucket: target.Bucket, + ObjectKey: target.Key, + ContentType: detectContentTypeFromPath(entry.Local), + Body: file, + ACL: options.ACL, + ServerSideEncryption: options.SSE, + SSEKMSKeyId: options.SSEKMS, + }) + return err +} + +func downloadS3ToLocal(client *simples3.S3, entry sourceEntry, target targetRef) error { + body, err := client.FileDownload(simples3.DownloadInput{Bucket: entry.Bucket, ObjectKey: entry.Key, VersionId: entry.Version}) + if err != nil { + return err + } + defer body.Close() + + if err := os.MkdirAll(filepath.Dir(target.Local), 0o755); err != nil { + return err + } + tempFile, err := os.CreateTemp(filepath.Dir(target.Local), ".simples3-download-*") + if err != nil { + return err + } + tempName := tempFile.Name() + defer os.Remove(tempName) + + if _, err := io.Copy(tempFile, body); err != nil { + tempFile.Close() + return err + } + if err := tempFile.Close(); err != nil { + return err + } + if err := os.Rename(tempName, target.Local); err != nil { + return err + } + if !entry.ModTime.IsZero() { + _ = os.Chtimes(target.Local, time.Now(), entry.ModTime) + } + return nil +} + +func copyS3ToS3(client *simples3.S3, entry sourceEntry, target targetRef, options copyOptions) error { + _, err := client.CopyObject(simples3.CopyObjectInput{ + SourceBucket: entry.Bucket, + SourceKey: entry.Key, + DestBucket: target.Bucket, + DestKey: target.Key, + ServerSideEncryption: options.SSE, + SSEKMSKeyId: options.SSEKMS, + MetadataDirective: "COPY", + ContentType: "", + CustomMetadata: nil, + }) + return err +} + +func deleteSourceEntry(client *simples3.S3, entry sourceEntry, root location) error { + if entry.Kind == locationKindLocal { + if err := os.Remove(entry.Local); err != nil { + return err + } + if root.isLocal() && root.path != entry.Local { + removeEmptyParents(filepath.Dir(entry.Local), root.path) + } + return nil + } + return client.FileDelete(simples3.DeleteInput{Bucket: entry.Bucket, ObjectKey: entry.Key, VersionId: entry.Version}) +} + +func deleteTargetRef(client *simples3.S3, target targetRef) error { + if target.Kind == locationKindLocal { + return os.Remove(target.Local) + } + return client.FileDelete(simples3.DeleteInput{Bucket: target.Bucket, ObjectKey: target.Key}) +} + +func removeEmptyParents(startDir, stopDir string) { + current := startDir + stopDir = filepath.Clean(stopDir) + for { + if current == "." || current == string(filepath.Separator) { + return + } + if err := os.Remove(current); err != nil { + return + } + if filepath.Clean(current) == stopDir { + return + } + next := filepath.Dir(current) + if next == current { + return + } + current = next + } +} + +func fetchTargetMeta(client *simples3.S3, target targetRef) (targetMeta, error) { + if target.Kind == locationKindLocal { + info, err := os.Stat(target.Local) + if err != nil { + if os.IsNotExist(err) { + return targetMeta{}, nil + } + return targetMeta{}, err + } + if info.IsDir() { + return targetMeta{}, fmt.Errorf("destination %s is a directory", target.Local) + } + return targetMeta{Exists: true, Size: info.Size(), ModTime: info.ModTime()}, nil + } + + details, err := client.FileDetails(simples3.DetailsInput{Bucket: target.Bucket, ObjectKey: target.Key}) + if err != nil { + if strings.Contains(err.Error(), "404") || strings.Contains(strings.ToLower(err.Error()), "not found") { + return targetMeta{}, nil + } + return targetMeta{}, err + } + size, _ := strconv.ParseInt(details.ContentLength, 10, 64) + return targetMeta{Exists: true, Size: size, ModTime: parseHTTPTime(details.LastModified)}, nil +} + +func needsSync(entry sourceEntry, target targetRef, meta targetMeta) bool { + if !meta.Exists { + return true + } + if entry.Size != meta.Size { + return true + } + if !entry.ModTime.IsZero() && !meta.ModTime.IsZero() { + return !sameModTime(entry.ModTime, meta.ModTime) + } + return false +} + +func sameModTime(a, b time.Time) bool { + return a.UTC().Truncate(time.Second).Equal(b.UTC().Truncate(time.Second)) +} + +func collectTargetEntries(client *simples3.S3, target location, recursive bool, matcher matcher) ([]sourceEntry, error) { + if target.isLocal() { + if _, err := os.Stat(target.path); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + } + return collectSourceEntries(client, target, recursive, matcher, "") +} + +func chunkKeys(entries []sourceEntry, size int) [][]sourceEntry { + if len(entries) == 0 { + return nil + } + chunks := make([][]sourceEntry, 0, (len(entries)+size-1)/size) + for start := 0; start < len(entries); start += size { + end := start + size + if end > len(entries) { + end = len(entries) + } + chunks = append(chunks, entries[start:end]) + } + return chunks +} + +type deleteBatch struct { + entries []sourceEntry +} + +func buildDeleteResult(entry sourceEntry, dryRun bool) operationResult { + result := operationResult{Action: "remove", Source: entrySourceString(entry), Status: "deleted", Size: entry.Size, DryRun: dryRun} + if dryRun { + result.Status = "would-delete" + } + return result +} + +func executeDeleteBatches(client *simples3.S3, entries []sourceEntry, bucket string, options executionOptions) []operationResult { + if len(entries) == 0 { + return []operationResult{} + } + if options.DryRun { + results := make([]operationResult, len(entries)) + for i, entry := range entries { + results[i] = buildDeleteResult(entry, true) + } + return results + } + + batches := makeDeleteBatches(entries) + concurrency := options.Concurrency + if concurrency < 1 { + concurrency = 1 + } + if concurrency == 1 { + results := make([]operationResult, 0, len(entries)) + for _, batch := range batches { + batchResults, failed := runDeleteBatch(client, batch, bucket, options.Retries) + results = append(results, batchResults...) + if failed && !options.ContinueOnError { + break + } + } + return results + } + + type batchResult struct { + index int + results []operationResult + failed bool + } + results := make([][]operationResult, len(batches)) + started := make([]bool, len(batches)) + resultCh := make(chan batchResult, concurrency) + var wg sync.WaitGroup + stopLaunching := false + next := 0 + inFlight := 0 + launch := func(index int) { + started[index] = true + inFlight++ + wg.Add(1) + go func() { + defer wg.Done() + batchResults, failed := runDeleteBatch(client, batches[index], bucket, options.Retries) + resultCh <- batchResult{index: index, results: batchResults, failed: failed} + }() + } + for next < len(batches) && inFlight < concurrency { + launch(next) + next++ + } + for inFlight > 0 { + completed := <-resultCh + results[completed.index] = completed.results + inFlight-- + if completed.failed && !options.ContinueOnError { + stopLaunching = true + } + for !stopLaunching && next < len(batches) && inFlight < concurrency { + launch(next) + next++ + } + } + wg.Wait() + + flattened := make([]operationResult, 0, len(entries)) + for i := 0; i < next; i++ { + if started[i] { + flattened = append(flattened, results[i]...) + } + } + return flattened +} + +func makeDeleteBatches(entries []sourceEntry) []deleteBatch { + chunks := chunkKeys(entries, 1000) + batches := make([]deleteBatch, 0, len(chunks)) + for _, chunk := range chunks { + batches = append(batches, deleteBatch{entries: chunk}) + } + return batches +} + +func runDeleteBatch(client *simples3.S3, batch deleteBatch, bucket string, retries int) ([]operationResult, bool) { + results := make([]operationResult, len(batch.entries)) + keys := make([]string, 0, len(batch.entries)) + for i, entry := range batch.entries { + results[i] = buildDeleteResult(entry, false) + keys = append(keys, entry.Key) + } + + var output simples3.DeleteObjectsOutput + if err := retryOperation(retries, func() error { + var err error + output, err = client.DeleteObjects(simples3.DeleteObjectsInput{Bucket: bucket, Objects: keys, Quiet: true}) + return err + }); err != nil { + for i := range results { + results[i].Status = "failed" + results[i].Error = err.Error() + } + return results, true + } + + errorsByKey := make(map[string]simples3.DeleteError, len(output.Errors)) + for _, deleteErr := range output.Errors { + errorsByKey[deleteErr.Key] = deleteErr + } + failed := false + for i, entry := range batch.entries { + if deleteErr, ok := errorsByKey[entry.Key]; ok { + results[i].Status = "failed" + results[i].Error = formatDeleteError(deleteErr) + failed = true + } + } + return results, failed +} + +func formatDeleteError(deleteErr simples3.DeleteError) string { + switch { + case deleteErr.Code != "" && deleteErr.Message != "": + return deleteErr.Code + ": " + deleteErr.Message + case deleteErr.Message != "": + return deleteErr.Message + case deleteErr.Code != "": + return deleteErr.Code + default: + return "delete failed" + } +} + +func executePlannedOperations(ops []plannedOperation, options executionOptions) []operationResult { + if len(ops) == 0 { + return []operationResult{} + } + if options.DryRun { + results := make([]operationResult, len(ops)) + for i, op := range ops { + results[i] = op.result + } + return results + } + concurrency := options.Concurrency + if concurrency < 1 { + concurrency = 1 + } + if concurrency == 1 { + results := make([]operationResult, 0, len(ops)) + for _, op := range ops { + result := runPlannedOperation(op, options) + results = append(results, result) + if result.Error != "" && !options.ContinueOnError { + break + } + } + return results + } + + type workResult struct { + index int + result operationResult + } + results := make([]operationResult, len(ops)) + started := make([]bool, len(ops)) + resultCh := make(chan workResult, concurrency) + var wg sync.WaitGroup + stopLaunching := false + next := 0 + inFlight := 0 + launch := func(index int) { + started[index] = true + inFlight++ + wg.Add(1) + go func() { + defer wg.Done() + resultCh <- workResult{index: index, result: runPlannedOperation(ops[index], options)} + }() + } + for next < len(ops) && inFlight < concurrency { + launch(next) + next++ + } + for inFlight > 0 { + completed := <-resultCh + results[completed.index] = completed.result + inFlight-- + if completed.result.Error != "" && !options.ContinueOnError { + stopLaunching = true + } + for !stopLaunching && next < len(ops) && inFlight < concurrency { + launch(next) + next++ + } + } + wg.Wait() + + attempted := make([]operationResult, 0, next) + for i := 0; i < next; i++ { + if started[i] { + attempted = append(attempted, results[i]) + } + } + return attempted +} + +func runPlannedOperation(op plannedOperation, options executionOptions) operationResult { + result := op.result + if op.run == nil { + return result + } + if err := retryOperation(options.Retries, op.run); err != nil { + result.Status = "failed" + result.Error = err.Error() + } + return result +} + +func retryOperation(retries int, fn func() error) error { + attempts := retries + 1 + if attempts < 1 { + attempts = 1 + } + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + err := fn() + if err == nil { + return nil + } + lastErr = err + if attempt == attempts || !isRetriableError(err) { + break + } + time.Sleep(time.Duration(attempt*attempt) * 100 * time.Millisecond) + } + return lastErr +} + +func isRetriableError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + for _, token := range []string{"timeout", "temporary", "connection reset", "connection refused", "unexpected eof", "no such host", "503", "502", "500", "504", "slow down", "internalerror"} { + if strings.Contains(message, token) { + return true + } + } + return false +} + +func operationsPartialFailure(command string, results []operationResult) error { + if !hasFailures(results) { + return nil + } + failed := summarizeOperations(results).Failed + return partialFailureError(fmt.Sprintf("%s completed with %d failed operation(s)", command, failed), buildOperationsData(results)) +} diff --git a/testenv_test.go b/testenv_test.go new file mode 100644 index 0000000..4351c61 --- /dev/null +++ b/testenv_test.go @@ -0,0 +1,81 @@ +package simples3 + +import ( + "fmt" + "os" + "strings" + "testing" +) + +func TestMain(m *testing.M) { + normalizeTestEnv() + if err := ensureTestBucket(); err != nil { + fmt.Fprintf(os.Stderr, "failed to bootstrap test bucket: %v\n", err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func normalizeTestEnv() { + aliasEnv("AWS_S3_ACCESS_KEY", "AWS_ACCESS_KEY_ID") + aliasEnv("AWS_S3_SECRET_KEY", "AWS_SECRET_ACCESS_KEY") + aliasEnv("AWS_S3_REGION", "AWS_REGION", "AWS_DEFAULT_REGION") + aliasEnv("AWS_REGION", "AWS_S3_REGION", "AWS_DEFAULT_REGION") + aliasEnv("AWS_DEFAULT_REGION", "AWS_REGION", "AWS_S3_REGION") + + if os.Getenv("AWS_S3_REGION") == "" && os.Getenv("AWS_S3_ENDPOINT") != "" { + _ = os.Setenv("AWS_S3_REGION", "us-east-1") + } + if os.Getenv("AWS_REGION") == "" && os.Getenv("AWS_S3_REGION") != "" { + _ = os.Setenv("AWS_REGION", os.Getenv("AWS_S3_REGION")) + } + if os.Getenv("AWS_DEFAULT_REGION") == "" && os.Getenv("AWS_S3_REGION") != "" { + _ = os.Setenv("AWS_DEFAULT_REGION", os.Getenv("AWS_S3_REGION")) + } +} + +func aliasEnv(target string, sources ...string) { + if os.Getenv(target) != "" { + return + } + for _, source := range sources { + if value := os.Getenv(source); value != "" { + _ = os.Setenv(target, value) + return + } + } +} + +func ensureTestBucket() error { + bucket := os.Getenv("AWS_S3_BUCKET") + accessKey := os.Getenv("AWS_S3_ACCESS_KEY") + secretKey := os.Getenv("AWS_S3_SECRET_KEY") + region := os.Getenv("AWS_S3_REGION") + if bucket == "" || accessKey == "" || secretKey == "" { + return nil + } + if region == "" { + region = "us-east-1" + } + + s3 := New(region, accessKey, secretKey) + if endpoint := os.Getenv("AWS_S3_ENDPOINT"); endpoint != "" { + s3.SetEndpoint(endpoint) + } + + _, err := s3.CreateBucket(CreateBucketInput{Bucket: bucket, Region: region}) + if err == nil || bucketAlreadyExistsError(err) { + return nil + } + return err +} + +func bucketAlreadyExistsError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "bucketalreadyownedbyyou") || + strings.Contains(message, "bucket already owned by you") || + strings.Contains(message, "bucket already exists") +}