From 9fb41f06a97a23ac9d88453373b2a7a194eac8ec Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 13:30:49 -0800 Subject: [PATCH 01/65] add listentries --- pkg/remote/awsconn/awsconn.go | 23 +++++-- pkg/remote/fileshare/fileshare.go | 14 ++-- pkg/remote/fileshare/s3fs/s3fs.go | 106 +++++++++++++++++++++++------- 3 files changed, 109 insertions(+), 34 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index ff0deaedaf..c03d3082b3 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -124,13 +124,24 @@ func ParseProfiles() map[string]struct{} { } func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) { - output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) - if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - return nil, fmt.Errorf("error listing buckets: %v", apiErr) + var err error + var output *s3.ListBucketsOutput + var buckets []types.Bucket + bucketPaginator := s3.NewListBucketsPaginator(client, &s3.ListBucketsInput{}) + for bucketPaginator.HasMorePages() { + output, err = bucketPaginator.NextPage(ctx) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { + fmt.Println("You don't have permission to list buckets for this account.") + err = apiErr + } else { + log.Printf("Couldn't list buckets for your account. Here's why: %v\n", err) + } + break + } else { + buckets = append(buckets, output.Buckets...) } - return nil, fmt.Errorf("error listing buckets: %v", err) } return output.Buckets, nil } diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 9473db55a2..6b2efa6769 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -5,8 +5,10 @@ import ( "fmt" "log" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" @@ -28,12 +30,12 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { - // config, err := awsconn.GetConfig(ctx, connection) - // if err != nil { - // log.Printf("error getting aws config: %v", err) - // return nil, nil - // } - return nil, nil + config, err := awsconn.GetConfig(ctx, connection) + if err != nil { + log.Printf("error getting aws config: %v", err) + return nil, nil + } + return s3fs.NewS3Client(config), conn } else if conntype == connparse.ConnectionTypeWave { return wavefs.NewWaveClient(), conn } else if conntype == connparse.ConnectionTypeWsh { diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index b406615d4d..37c5a085ee 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -7,9 +7,11 @@ import ( "context" "errors" "log" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -42,31 +44,24 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, return wshutil.SendErrCh[iochantypes.Packet](errors.ErrUnsupported) } -func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) - go func() { - defer close(ch) - list, err := c.ListEntries(ctx, conn, opts) - if err != nil { - ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) - return - } - if list == nil { - ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{}} - return - } - for i := 0; i < len(list); i += wshrpc.DirChunkSize { - ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+wshrpc.DirChunkSize, len(list))]}} +func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + var entries []*wshrpc.FileInfo + rtnCh := c.ListEntriesStream(ctx, conn, opts) + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error } - }() - return ch + resp := respUnion.Response + entries = append(entries, resp.FileInfo...) + } + return entries, nil } -func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { - if conn.Path == "" || conn.Path == "/" { +func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + if conn.Host == "" || conn.Host == "/" { buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { - return nil, err + return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](err) } var entries []*wshrpc.FileInfo for _, bucket := range buckets { @@ -78,9 +73,76 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o }) } } - return entries, nil + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 1) + defer close(rtn) + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} + return rtn + } else { + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) + go func() { + defer close(rtn) + var err error + var output *s3.ListObjectsV2Output + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(conn.Host), + Prefix: aws.String(conn.Path), + } + objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) + for objectPaginator.HasMorePages() { + output, err = objectPaginator.NextPage(ctx) + if err != nil { + var noBucket *types.NoSuchBucket + if errors.As(err, &noBucket) { + log.Printf("Bucket %s does not exist.\n", conn.Host) + err = noBucket + } + rtn <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) + break + } else { + entryMap := make(map[string]*wshrpc.FileInfo, len(output.Contents)) + for _, obj := range output.Contents { + if obj.Key != nil { + name := strings.TrimPrefix(*obj.Key, conn.Path) + if strings.Count(name, "/") > 1 { + if entryMap[name] == nil { + name = strings.SplitN(name, "/", 2)[0] + entryMap[name] = &wshrpc.FileInfo{ + Name: name + "/", // add trailing slash to indicate directory + IsDir: true, + Dir: conn.Path, + Size: -1, + } + } + continue + } + size := int64(0) + if obj.Size != nil { + size = *obj.Size + } + entryMap[name] = &wshrpc.FileInfo{ + Name: name, + IsDir: false, + Dir: conn.Path, + Size: size, + } + } + } + entries := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + for _, entry := range entryMap { + entries = append(entries, entry) + if len(entries) == wshrpc.DirChunkSize { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} + entries = make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + } + } + if len(entries) > 0 { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} + } + } + } + }() + return rtn } - return nil, nil } func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { From 77915129d1ba9750479801c152c22124f6102203 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 15:16:17 -0800 Subject: [PATCH 02/65] list entries implemented --- pkg/remote/awsconn/awsconn.go | 67 +++++++++--------- pkg/remote/fileshare/s3fs/s3fs.go | 111 +++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 49 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index c03d3082b3..5c84532b7f 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -17,9 +17,9 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/wavetermdev/waveterm/pkg/waveobj" - "github.com/wavetermdev/waveterm/pkg/wconfig" "gopkg.in/ini.v1" ) @@ -44,24 +44,27 @@ func GetConfig(ctx context.Context, profile string) (*aws.Config, error) { } profile = connMatch[1] log.Printf("GetConfig: profile=%s", profile) - profiles, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ProfilesFile) - if len(cerrs) > 0 { - return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) - } - if profiles[profile] != nil { - configfilepath, _ := getTempFileFromConfig(profiles, ProfileConfigKey, profile) - credentialsfilepath, _ := getTempFileFromConfig(profiles, ProfileCredentialsKey, profile) - if configfilepath != "" { - log.Printf("configfilepath: %s", configfilepath) - optfns = append(optfns, config.WithSharedConfigFiles([]string{configfilepath})) - tempfiles[profile+"_config"] = configfilepath - } - if credentialsfilepath != "" { - log.Printf("credentialsfilepath: %s", credentialsfilepath) - optfns = append(optfns, config.WithSharedCredentialsFiles([]string{credentialsfilepath})) - tempfiles[profile+"_credentials"] = credentialsfilepath - } - } + + // TODO: Reimplement generic profile support + // profiles, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ProfilesFile) + // if len(cerrs) > 0 { + // return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) + // } + // if profiles[profile] != nil { + // configfilepath, _ := getTempFileFromConfig(profiles, ProfileConfigKey, profile) + // credentialsfilepath, _ := getTempFileFromConfig(profiles, ProfileCredentialsKey, profile) + // if configfilepath != "" { + // log.Printf("configfilepath: %s", configfilepath) + // optfns = append(optfns, config.WithSharedConfigFiles([]string{configfilepath})) + // tempfiles[profile+"_config"] = configfilepath + // } + // if credentialsfilepath != "" { + // log.Printf("credentialsfilepath: %s", credentialsfilepath) + // optfns = append(optfns, config.WithSharedCredentialsFiles([]string{credentialsfilepath})) + // tempfiles[profile+"_credentials"] = credentialsfilepath + // } + // } + optfns = append(optfns, config.WithRegion("us-west-2")) trimmedProfile := strings.TrimPrefix(profile, ProfilePrefix) optfns = append(optfns, config.WithSharedConfigProfile(trimmedProfile)) } @@ -112,10 +115,7 @@ func ParseProfiles() map[string]struct{} { f, err = ini.Load(fname) if err != nil { log.Printf("error reading aws credentials file: %v", err) - if profiles == nil { - profiles = make(map[string]struct{}) - } - return profiles + return nil } for _, v := range f.Sections() { profiles[ProfilePrefix+v.Name()] = struct{}{} @@ -131,17 +131,20 @@ func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) for bucketPaginator.HasMorePages() { output, err = bucketPaginator.NextPage(ctx) if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { - fmt.Println("You don't have permission to list buckets for this account.") - err = apiErr - } else { - log.Printf("Couldn't list buckets for your account. Here's why: %v\n", err) - } - break + CheckAccessDeniedErr(&err) + return nil, fmt.Errorf("error listing buckets: %v", err) } else { buckets = append(buckets, output.Buckets...) } } - return output.Buckets, nil + return buckets, nil +} + +func CheckAccessDeniedErr(err *error) bool { + var apiErr smithy.APIError + if err != nil && errors.As(*err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { + *err = apiErr + return true + } + return false } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 37c5a085ee..1071714b64 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -6,7 +6,9 @@ package s3fs import ( "context" "errors" + "fmt" "log" + "regexp" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -37,7 +39,22 @@ func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wsh } func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { - return wshutil.SendErrCh[wshrpc.FileData](errors.ErrUnsupported) + if conn.Host == "" || conn.Host == "/" { + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) + defer close(rtn) + entries, err := c.ListEntries(ctx, conn, nil) + if err != nil { + rtn <- wshutil.RespErr[wshrpc.FileData](err) + return rtn + } + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entries}} + return rtn + } else { + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) + defer close(rtn) + rtn <- wshutil.RespErr[wshrpc.FileData](errors.ErrUnsupported) + return rtn + } } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { @@ -48,6 +65,7 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o var entries []*wshrpc.FileInfo rtnCh := c.ListEntriesStream(ctx, conn, opts) for respUnion := range rtnCh { + log.Printf("respUnion: %v", respUnion) if respUnion.Error != nil { return nil, respUnion.Error } @@ -57,20 +75,33 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return entries, nil } +var slashRe = regexp.MustCompile(`/`) + func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + numToFetch := wshrpc.MaxDirSize + if opts != nil && opts.Limit > 0 { + numToFetch = min(opts.Limit, wshrpc.MaxDirSize) + } + numFetched := 0 if conn.Host == "" || conn.Host == "/" { buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { + log.Printf("error listing buckets: %v", err) return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](err) } var entries []*wshrpc.FileInfo for _, bucket := range buckets { - log.Printf("bucket: %v", *bucket.Name) + if numFetched >= numToFetch { + break + } if bucket.Name != nil { entries = append(entries, &wshrpc.FileInfo{ - Path: *bucket.Name, - IsDir: true, + Path: fmt.Sprintf("%s://%s/", conn.Scheme, *bucket.Name), // add trailing slash to indicate directory + Name: *bucket.Name, + ModTime: bucket.CreationDate.UnixMilli(), + IsDir: true, }) + numFetched++ } } rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 1) @@ -79,6 +110,8 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect return rtn } else { rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) + // keep track of "directories" that have been used to avoid duplicates between pages + prevUsedDirKeys := make(map[string]any) go func() { defer close(rtn) var err error @@ -88,11 +121,32 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Prefix: aws.String(conn.Path), } objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) + var parentPath string + hostAndPath := conn.GetPathWithHost() + slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) + if slashIndices != nil && len(slashIndices) > 0 { + if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] + } else if len(slashIndices) > 1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] + } + } + + if parentPath != "" { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: []*wshrpc.FileInfo{ + { + Path: fmt.Sprintf("%s://%s", conn.Scheme, parentPath), + Name: "..", + IsDir: true, + Size: -1, + }, + }}} + } for objectPaginator.HasMorePages() { output, err = objectPaginator.NextPage(ctx) if err != nil { var noBucket *types.NoSuchBucket - if errors.As(err, &noBucket) { + if !awsconn.CheckAccessDeniedErr(&err) && errors.As(err, &noBucket) { log.Printf("Bucket %s does not exist.\n", conn.Host) err = noBucket } @@ -101,30 +155,50 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect } else { entryMap := make(map[string]*wshrpc.FileInfo, len(output.Contents)) for _, obj := range output.Contents { - if obj.Key != nil { + if numFetched >= numToFetch { + break + } + lastModTime := int64(0) + if obj.LastModified != nil { + lastModTime = obj.LastModified.UnixMilli() + } + if obj.Key != nil && len(*obj.Key) > len(conn.Path) { name := strings.TrimPrefix(*obj.Key, conn.Path) - if strings.Count(name, "/") > 1 { + if strings.Count(name, "/") > 0 { + name = strings.SplitN(name, "/", 2)[0] + name = name + "/" // add trailing slash to indicate directory if entryMap[name] == nil { - name = strings.SplitN(name, "/", 2)[0] - entryMap[name] = &wshrpc.FileInfo{ - Name: name + "/", // add trailing slash to indicate directory - IsDir: true, - Dir: conn.Path, - Size: -1, + if _, ok := prevUsedDirKeys[name]; !ok { + entryMap[name] = &wshrpc.FileInfo{ + Path: conn.GetFullURI() + name, + Name: name, + IsDir: true, + Dir: conn.Path, + ModTime: lastModTime, + Size: -1, + } + prevUsedDirKeys[name] = struct{}{} + numFetched++ } + } else if entryMap[name].ModTime < lastModTime { + entryMap[name].ModTime = lastModTime } continue } + size := int64(0) if obj.Size != nil { size = *obj.Size } entryMap[name] = &wshrpc.FileInfo{ - Name: name, - IsDir: false, - Dir: conn.Path, - Size: size, + Name: name, + IsDir: false, + Dir: conn.Path, + Path: conn.GetFullURI() + name, + ModTime: lastModTime, + Size: size, } + numFetched++ } } entries := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) @@ -139,6 +213,9 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} } } + if numFetched >= numToFetch { + return + } } }() return rtn From 1d804fa7eafcc8448e42716c5faab9f7e0339803 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 16:14:47 -0800 Subject: [PATCH 03/65] more impls --- pkg/remote/fileshare/s3fs/s3fs.go | 310 +++++++++++++++++++++++++++--- 1 file changed, 285 insertions(+), 25 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 1071714b64..590c5f747a 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -4,9 +4,12 @@ package s3fs import ( + "bytes" "context" + "encoding/base64" "errors" "fmt" + "io" "log" "regexp" "strings" @@ -14,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -35,11 +39,54 @@ func NewS3Client(config *aws.Config) *S3Client { } func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { - return nil, errors.ErrUnsupported + rtnCh := c.ReadStream(ctx, conn, data) + var fileData *wshrpc.FileData + firstPk := true + isDir := false + var fileBuf bytes.Buffer + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error + } + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if resp.Info == nil { + return nil, fmt.Errorf("stream file protocol error, first pk fileinfo is empty") + } + fileData = &resp + if fileData.Info.IsDir { + isDir = true + } + continue + } + if isDir { + if len(resp.Entries) == 0 { + continue + } + fileData.Entries = append(fileData.Entries, resp.Entries...) + } else { + if resp.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + _, err := io.Copy(&fileBuf, decoder) + if err != nil { + return nil, fmt.Errorf("stream file, failed to decode base64 data: %w", err) + } + } + } + if !isDir { + fileData.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) + } + return fileData, nil } func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { - if conn.Host == "" || conn.Host == "/" { + bucket := conn.Host + objectKey := conn.Path + if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) defer close(rtn) entries, err := c.ListEntries(ctx, conn, nil) @@ -50,9 +97,60 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entries}} return rtn } else { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) - defer close(rtn) - rtn <- wshutil.RespErr[wshrpc.FileData](errors.ErrUnsupported) + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) + go func() { + defer close(rtn) + result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &objectKey, + }) + if err != nil { + var noKey *types.NoSuchKey + if errors.As(err, &noKey) { + log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectKey, bucket) + err = noKey + } else { + log.Printf("Couldn't get object %v:%v. Here's why: %v\n", bucket, objectKey, err) + } + rtn <- wshutil.RespErr[wshrpc.FileData](err) + return + } + size := int64(0) + if result.ContentLength != nil { + size = *result.ContentLength + } + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Info: &wshrpc.FileInfo{ + Name: objectKey, + IsDir: false, + Size: size, + ModTime: result.LastModified.UnixMilli(), + Path: conn.GetFullURI(), + }}} + if size == 0 { + return + } + defer result.Body.Close() + for { + select { + case <-ctx.Done(): + return + default: + buf := make([]byte, wshrpc.FileChunkSize) + n, err := result.Body.Read(buf) + if err != nil { + if err.Error() == "EOF" { + break + } + rtn <- wshutil.RespErr[wshrpc.FileData](err) + return + } + if n == 0 { + break + } + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: base64.StdEncoding.EncodeToString(buf[:n])}} + } + } + }() return rtn } } @@ -121,21 +219,11 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Prefix: aws.String(conn.Path), } objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) - var parentPath string - hostAndPath := conn.GetPathWithHost() - slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) - if slashIndices != nil && len(slashIndices) > 0 { - if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] - } else if len(slashIndices) > 1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] - } - } - + parentPath := getParentPathUri(conn) if parentPath != "" { rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: []*wshrpc.FileInfo{ { - Path: fmt.Sprintf("%s://%s", conn.Scheme, parentPath), + Path: parentPath, Name: "..", IsDir: true, Size: -1, @@ -223,23 +311,112 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect } func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - return nil, errors.ErrUnsupported + bucketName := conn.Host + objectKey := conn.Path + if bucketName == "" || bucketName == "/" { + return &wshrpc.FileInfo{ + Name: "/", + IsDir: true, + Size: -1, + ModTime: 0, + }, nil + } + if objectKey == "" || objectKey == "/" { + _, err := c.client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }) + exists := true + if err != nil { + var apiError smithy.APIError + if errors.As(err, &apiError) { + switch apiError.(type) { + case *types.NotFound: + log.Printf("Bucket %v is available.\n", bucketName) + exists = false + err = nil + default: + log.Printf("Either you don't have access to bucket %v or another error occurred. "+ + "Here's what happened: %v\n", bucketName, err) + } + } + } else { + log.Printf("Bucket %v exists and you already own it.", bucketName) + } + + if exists { + return &wshrpc.FileInfo{ + Name: bucketName, + IsDir: true, + Size: -1, + ModTime: 0, + }, nil + } else { + return nil, fmt.Errorf("bucket %v does not exist", bucketName) + } + } + result, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + if err != nil { + var noKey *types.NoSuchKey + if errors.As(err, &noKey) { + log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectKey, bucketName) + err = noKey + } else { + log.Printf("Couldn't get object %v:%v. Here's why: %v\n", bucketName, objectKey, err) + } + return nil, err + } + size := int64(0) + if result.ContentLength != nil { + size = *result.ContentLength + } + lastModified := int64(0) + if result.LastModified != nil { + lastModified = result.LastModified.UnixMilli() + } + return &wshrpc.FileInfo{ + Name: objectKey, + Path: conn.GetFullURI(), + Dir: getParentPathUri(conn), + IsDir: false, + Size: size, + ModTime: lastModified, + }, nil } func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - return errors.ErrUnsupported + if data.At != nil { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("file data offset and size not supported")) + } + bucket := conn.Host + objectKey := conn.Path + if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket and object key must be specified")) + } + _, err := c.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + Body: bytes.NewReader([]byte(data.Data64)), + }) + return err } func (c S3Client) AppendFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - return errors.ErrUnsupported + return errors.Join(errors.ErrUnsupported, fmt.Errorf("append file not supported")) } func (c S3Client) Mkdir(ctx context.Context, conn *connparse.Connection) error { - return errors.ErrUnsupported + return errors.Join(errors.ErrUnsupported, fmt.Errorf("mkdir not supported")) } func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return errors.ErrUnsupported + err := c.CopyInternal(ctx, srcConn, destConn, opts) + if err != nil { + return err + } + return c.Delete(ctx, srcConn, true) } func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { @@ -247,17 +424,100 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return errors.ErrUnsupported + srcBucket := srcConn.Host + srcKey := srcConn.Path + destBucket := destConn.Host + destKey := destConn.Path + if srcBucket == "" || srcBucket == "/" || srcKey == "" || srcKey == "/" || destBucket == "" || destBucket == "/" || destKey == "" || destKey == "/" { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("source and destination bucket and object key must be specified")) + } + _, err := c.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, srcKey)), + }) + return err } func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { - return errors.ErrUnsupported + bucket := conn.Host + objectKey := conn.Path + if bucket == "" || bucket == "/" { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket must be specified")) + } + if objectKey == "" || objectKey == "/" { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("object key must be specified")) + } + if recursive { + entries, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(objectKey), + }) + if err != nil { + return err + } + if len(entries.Contents) == 0 { + return nil + } + objects := make([]types.ObjectIdentifier, 0, len(entries.Contents)) + for _, obj := range entries.Contents { + objects = append(objects, types.ObjectIdentifier{Key: obj.Key}) + } + _, err = c.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objects, + }, + }) + return err + } + _, err := c.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + return err } func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { - return "", errors.ErrUnsupported + var joinParts []string + if conn.Host == "" || conn.Host == "/" { + if conn.Path == "" || conn.Path == "/" { + joinParts = parts + } else { + joinParts = append([]string{conn.Path}, parts...) + } + } else if conn.Path == "" || conn.Path == "/" { + joinParts = append([]string{conn.Host}, parts...) + } else { + joinParts = append([]string{conn.Host, conn.Path}, parts...) + } + + return fmt.Sprintf("%s://%s", conn.Scheme, strings.Join(joinParts, "/")), nil } func (c S3Client) GetConnectionType() string { return connparse.ConnectionTypeS3 } + +func getParentPathUri(conn *connparse.Connection) string { + parentPath := getParentPath(conn) + if parentPath == "" { + return "" + } + return fmt.Sprintf("%s://%s", conn.Scheme, parentPath) +} + +func getParentPath(conn *connparse.Connection) string { + var parentPath string + hostAndPath := conn.GetPathWithHost() + slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) + if slashIndices != nil && len(slashIndices) > 0 { + if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] + } else if len(slashIndices) > 1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] + } + } + return parentPath + +} From 91a936c7f455504624d0b0e0f11ceed8d4060ac3 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 18:10:55 -0800 Subject: [PATCH 04/65] cat is working! --- cmd/wsh/cmd/wshcmd-file-util.go | 42 ++------ cmd/wsh/cmd/wshcmd-file.go | 33 ++++-- frontend/app/store/wshclientapi.ts | 10 ++ frontend/types/gotypes.d.ts | 6 ++ pkg/remote/fileshare/fileshare.go | 8 ++ pkg/remote/fileshare/fstype/fstype.go | 2 + pkg/remote/fileshare/s3fs/s3fs.go | 145 +++++++++++++------------- pkg/remote/fileshare/wavefs/wavefs.go | 7 ++ pkg/remote/fileshare/wshfs/wshfs.go | 50 ++------- pkg/util/fileutil/fileutil.go | 127 ++++++++++++++++++++-- pkg/wshrpc/wshclient/wshclient.go | 11 ++ pkg/wshrpc/wshrpctypes.go | 9 ++ pkg/wshrpc/wshserver/wshserver.go | 8 ++ 13 files changed, 286 insertions(+), 172 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 432cc1b1fd..fdb65e3c4a 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -4,6 +4,7 @@ package cmd import ( + "context" "encoding/base64" "fmt" "io" @@ -27,7 +28,7 @@ func convertNotFoundErr(err error) error { return err } -func ensureFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { +func ensureFile(fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { @@ -56,7 +57,7 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { return fmt.Errorf("initializing file with empty write: %w", err) } - const chunkSize = 32 * 1024 // 32KB chunks + const chunkSize = wshrpc.FileChunkSize // 32KB chunks buf := make([]byte, chunkSize) totalWritten := int64(0) @@ -89,40 +90,9 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { return nil } -func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { - const chunkSize = 32 * 1024 // 32KB chunks - for offset := int64(0); offset < size; offset += chunkSize { - // Calculate the length of this chunk - length := chunkSize - if offset+int64(length) > size { - length = int(size - offset) - } - - // Set up the ReadAt request - fileData.At = &wshrpc.FileDataAt{ - Offset: offset, - Size: length, - } - - // Read the chunk - data, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: int64(fileTimeout)}) - if err != nil { - return fmt.Errorf("reading chunk at offset %d: %w", offset, err) - } - - // Decode and write the chunk - chunk, err := base64.StdEncoding.DecodeString(data.Data64) - if err != nil { - return fmt.Errorf("decoding chunk at offset %d: %w", offset, err) - } - - _, err = writer.Write(chunk) - if err != nil { - return fmt.Errorf("writing chunk at offset %d: %w", offset, err) - } - } - - return nil +func streamReadFromFile(ctx context.Context, fileData wshrpc.FileData, size int64, writer io.Writer) error { + ch := wshclient.FileReadStreamCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + return fileutil.ReadFileStreamToWriter(ctx, ch, writer) } type fileListResult struct { diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 62e60474ee..d3d4fece4c 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -212,7 +212,7 @@ func fileCatRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting file info: %w", err) } - err = streamReadFromFile(fileData, info.Size, os.Stdout) + err = streamReadFromFile(cmd.Context(), fileData, info.Size, os.Stdout) if err != nil { return fmt.Errorf("reading file: %w", err) } @@ -294,14 +294,31 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { Info: &wshrpc.FileInfo{ Path: path}} - _, err = ensureFile(path, fileData) + capability, err := wshclient.FileShareCapabilityCommand(RpcClient, fileData.Info.Path, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) if err != nil { - return err + return fmt.Errorf("getting fileshare capability: %w", err) } - - err = streamWriteToFile(fileData, WrappedStdin) - if err != nil { - return fmt.Errorf("writing file: %w", err) + if capability.CanAppend { + err = streamWriteToFile(fileData, WrappedStdin) + if err != nil { + return fmt.Errorf("writing file: %w", err) + } + } else { + buf := make([]byte, MaxFileSize) + n, err := WrappedStdin.Read(buf) + if err != nil && err != io.EOF { + return fmt.Errorf("reading input: %w", err) + } + if int64(n) == MaxFileSize { + if _, err := WrappedStdin.Read(make([]byte, 1)); err != io.EOF { + return fmt.Errorf("input exceeds maximum file size of %d bytes", MaxFileSize) + } + } + fileData.Data64 = base64.StdEncoding.EncodeToString(buf[:n]) + err = wshclient.FileWriteCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + if err != nil { + return fmt.Errorf("writing file: %w", err) + } } return nil @@ -316,7 +333,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { Info: &wshrpc.FileInfo{ Path: path}} - info, err := ensureFile(path, fileData) + info, err := ensureFile(fileData) if err != nil { return err } diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 9c614af4e7..1d6f914b3d 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -202,6 +202,16 @@ class RpcApiType { return client.wshRpcCall("fileread", data, opts); } + // command "filereadstream" [responsestream] + FileReadStreamCommand(client: WshClient, data: FileData, opts?: RpcOpts): AsyncGenerator { + return client.wshRpcStream("filereadstream", data, opts); + } + + // command "filesharecapability" [call] + FileShareCapabilityCommand(client: WshClient, data: string, opts?: RpcOpts): Promise { + return client.wshRpcCall("filesharecapability", data, opts); + } + // command "filestreamtar" [responsestream] FileStreamTarCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("filestreamtar", data, opts); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 400e86935d..b4ddc12145 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -441,6 +441,12 @@ declare global { append?: boolean; }; + // wshrpc.FileShareCapability + type FileShareCapability = { + CanAppend: boolean; + CanMkdir: boolean; + }; + // wconfig.FullConfigType type FullConfigType = { settings: SettingsType; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 6b2efa6769..b4c0bc55e8 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -169,3 +169,11 @@ func Append(ctx context.Context, data wshrpc.FileData) error { } return client.AppendFile(ctx, conn, data) } + +func GetCapability(ctx context.Context, path string) (wshrpc.FileShareCapability, error) { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return wshrpc.FileShareCapability{}, fmt.Errorf(ErrorParsingConnection, path) + } + return client.GetCapability(), nil +} diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 3c3d6fceb3..5ca82ccd4c 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -42,4 +42,6 @@ type FileShareClient interface { Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) // GetConnectionType returns the type of connection for the fileshare GetConnectionType() string + // GetCapability returns the capability of the fileshare + GetCapability() wshrpc.FileShareCapability } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 590c5f747a..7e3b293917 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -6,10 +6,10 @@ package s3fs import ( "bytes" "context" - "encoding/base64" "errors" "fmt" "io" + "io/fs" "log" "regexp" "strings" @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" @@ -40,71 +41,52 @@ func NewS3Client(config *aws.Config) *S3Client { func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { rtnCh := c.ReadStream(ctx, conn, data) - var fileData *wshrpc.FileData - firstPk := true - isDir := false - var fileBuf bytes.Buffer - for respUnion := range rtnCh { - if respUnion.Error != nil { - return nil, respUnion.Error - } - resp := respUnion.Response - if firstPk { - firstPk = false - // first packet has the fileinfo - if resp.Info == nil { - return nil, fmt.Errorf("stream file protocol error, first pk fileinfo is empty") - } - fileData = &resp - if fileData.Info.IsDir { - isDir = true - } - continue - } - if isDir { - if len(resp.Entries) == 0 { - continue - } - fileData.Entries = append(fileData.Entries, resp.Entries...) - } else { - if resp.Data64 == "" { - continue - } - decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) - _, err := io.Copy(&fileBuf, decoder) - if err != nil { - return nil, fmt.Errorf("stream file, failed to decode base64 data: %w", err) - } - } - } - if !isDir { - fileData.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) - } - return fileData, nil + return fileutil.ReadStreamToFileData(ctx, rtnCh) } func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { bucket := conn.Host objectKey := conn.Path - if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) + go func() { defer close(rtn) - entries, err := c.ListEntries(ctx, conn, nil) - if err != nil { - rtn <- wshutil.RespErr[wshrpc.FileData](err) - return rtn - } - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entries}} - return rtn - } else { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) - go func() { - defer close(rtn) - result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &objectKey, - }) + if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { + entries, err := c.ListEntries(ctx, conn, nil) if err != nil { + rtn <- wshutil.RespErr[wshrpc.FileData](err) + return + } + entryBuf := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + for _, entry := range entries { + entryBuf = append(entryBuf, entry) + if len(entryBuf) == wshrpc.DirChunkSize { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entryBuf}} + entryBuf = make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + } + } + if len(entryBuf) > 0 { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entryBuf}} + } + return + } else { + var result *s3.GetObjectOutput + var err error + if data.At != nil { + log.Printf("reading %v with offset %d and size %d", conn.GetFullURI(), data.At.Offset, data.At.Size) + result, err = c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", data.At.Offset, data.At.Offset+int64(data.At.Size)-1)), + }) + } else { + log.Printf("reading %v", conn.GetFullURI()) + result, err = c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + } + if err != nil { + log.Printf("error getting object %v:%v: %v", bucket, objectKey, err) var noKey *types.NoSuchKey if errors.As(err, &noKey) { log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectKey, bucket) @@ -119,40 +101,49 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da if result.ContentLength != nil { size = *result.ContentLength } - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Info: &wshrpc.FileInfo{ + finfo := &wshrpc.FileInfo{ Name: objectKey, IsDir: false, Size: size, ModTime: result.LastModified.UnixMilli(), Path: conn.GetFullURI(), - }}} + } + log.Printf("file info: %v", finfo) + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Info: finfo}} if size == 0 { + log.Printf("no data to read") return } defer result.Body.Close() + bytesRemaining := size for { + log.Printf("bytes remaining: %d", bytesRemaining) select { case <-ctx.Done(): + log.Printf("context done") + rtn <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) return default: - buf := make([]byte, wshrpc.FileChunkSize) + buf := make([]byte, min(bytesRemaining, wshrpc.FileChunkSize)) n, err := result.Body.Read(buf) - if err != nil { - if err.Error() == "EOF" { - break - } + if err != nil && !errors.Is(err, io.EOF) { rtn <- wshutil.RespErr[wshrpc.FileData](err) return } + log.Printf("read %d bytes", n) if n == 0 { break } - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: base64.StdEncoding.EncodeToString(buf[:n])}} + bytesRemaining -= int64(n) + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: string(buf)}} + if bytesRemaining == 0 || errors.Is(err, io.EOF) { + return + } } } - }() - return rtn - } + } + }() + return rtn } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { @@ -331,7 +322,6 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc if errors.As(err, &apiError) { switch apiError.(type) { case *types.NotFound: - log.Printf("Bucket %v is available.\n", bucketName) exists = false err = nil default: @@ -360,9 +350,9 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc }) if err != nil { var noKey *types.NoSuchKey - if errors.As(err, &noKey) { - log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectKey, bucketName) - err = noKey + var notFound *types.NotFound + if errors.As(err, &noKey) || errors.As(err, ¬Found) { + err = fs.ErrNotExist } else { log.Printf("Couldn't get object %v:%v. Here's why: %v\n", bucketName, objectKey, err) } @@ -499,6 +489,13 @@ func (c S3Client) GetConnectionType() string { return connparse.ConnectionTypeS3 } +func (c S3Client) GetCapability() wshrpc.FileShareCapability { + return wshrpc.FileShareCapability{ + CanAppend: false, + CanMkdir: false, + } +} + func getParentPathUri(conn *connparse.Connection) string { parentPath := getParentPath(conn) if parentPath == "" { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 63cbe36a1d..aa05f49439 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -548,6 +548,13 @@ func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts return newPath, nil } +func (c WaveClient) GetCapability() wshrpc.FileShareCapability { + return wshrpc.FileShareCapability{ + CanAppend: true, + CanMkdir: false, + } +} + func cleanPath(path string) (string, error) { if path == "" { return "", fmt.Errorf("path is empty") diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 61816ea576..1401730bff 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -4,14 +4,12 @@ package wshfs import ( - "bytes" "context" - "encoding/base64" "fmt" - "io" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -35,47 +33,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { rtnCh := c.ReadStream(ctx, conn, data) - var fileData *wshrpc.FileData - firstPk := true - isDir := false - var fileBuf bytes.Buffer - for respUnion := range rtnCh { - if respUnion.Error != nil { - return nil, respUnion.Error - } - resp := respUnion.Response - if firstPk { - firstPk = false - // first packet has the fileinfo - if resp.Info == nil { - return nil, fmt.Errorf("stream file protocol error, first pk fileinfo is empty") - } - fileData = &resp - if fileData.Info.IsDir { - isDir = true - } - continue - } - if isDir { - if len(resp.Entries) == 0 { - continue - } - fileData.Entries = append(fileData.Entries, resp.Entries...) - } else { - if resp.Data64 == "" { - continue - } - decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) - _, err := io.Copy(&fileBuf, decoder) - if err != nil { - return nil, fmt.Errorf("stream file, failed to decode base64 data: %w", err) - } - } - } - if !isDir { - fileData.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) - } - return fileData, nil + return fileutil.ReadStreamToFileData(ctx, rtnCh) } func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { @@ -190,3 +148,7 @@ func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts . func (c WshClient) GetConnectionType() string { return connparse.ConnectionTypeWsh } + +func (c WshClient) GetCapability() wshrpc.FileShareCapability { + return wshrpc.FileShareCapability{CanAppend: true, CanMkdir: true} +} diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 18bb538d64..1f7ac20a05 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -4,6 +4,10 @@ package fileutil import ( + "bytes" + "context" + "encoding/base64" + "fmt" "io" "io/fs" "mime" @@ -202,14 +206,117 @@ var _ fs.FileInfo = FsFileInfo{} // ToFsFileInfo converts wshrpc.FileInfo to FsFileInfo. // It panics if fi is nil. func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { - if fi == nil { - panic("ToFsFileInfo: nil FileInfo") - } - return FsFileInfo{ - NameInternal: fi.Name, - ModeInternal: fi.Mode, - SizeInternal: fi.Size, - ModTimeInternal: fi.ModTime, - IsDirInternal: fi.IsDir, - } + if fi == nil { + panic("ToFsFileInfo: nil FileInfo") + } + return FsFileInfo{ + NameInternal: fi.Name, + ModeInternal: fi.Mode, + SizeInternal: fi.Size, + ModTimeInternal: fi.ModTime, + IsDirInternal: fi.IsDir, + } +} + +func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { + var fileData *wshrpc.FileData + firstPk := true + isDir := false + drain := true + defer func() { + if drain { + go func() { + for range readCh { + } + }() + } + }() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %v", ctx.Err()) + case respUnion, ok := <-readCh: + if !ok { + drain = false + return nil + } + if respUnion.Error != nil { + return respUnion.Error + } + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if resp.Info == nil { + return fmt.Errorf("stream file protocol error, first pk fileinfo is empty") + } + fileData = &resp + if fileData.Info.IsDir { + isDir = true + } + continue + } + if isDir { + if len(resp.Entries) == 0 { + continue + } + if resp.Data64 != "" { + return fmt.Errorf("stream file protocol error, directory entry has data") + } + if err := dirCallback(resp.Entries); err != nil { + return err + } + } else { + if resp.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + if err := fileCallback(decoder); err != nil { + return err + } + } + } + } +} + +func ReadStreamToFileData(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData]) (*wshrpc.FileData, error) { + var fileData *wshrpc.FileData + var dataBuf bytes.Buffer + var entries []*wshrpc.FileInfo + err := ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + fileData = &wshrpc.FileData{ + Info: &finfo, + } + }, func(entries []*wshrpc.FileInfo) error { + entries = append(entries, entries...) + return nil + }, func(data io.Reader) error { + if _, err := io.Copy(&dataBuf, data); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + if fileData == nil { + return nil, fmt.Errorf("stream file protocol error, no file info") + } + if !fileData.Info.IsDir { + fileData.Data64 = base64.StdEncoding.EncodeToString(dataBuf.Bytes()) + } else { + fileData.Entries = entries + } + return fileData, nil +} + +func ReadFileStreamToWriter(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], writer io.Writer) error { + return ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + }, func(entries []*wshrpc.FileInfo) error { + return nil + }, func(data io.Reader) error { + _, err := io.Copy(writer, data) + return err + }) } diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 599cd79c51..6e48596dfe 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -248,6 +248,17 @@ func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp return resp, err } +// command "filereadstream", wshserver.FileReadStreamCommand +func FileReadStreamCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + return sendRpcRequestResponseStreamHelper[wshrpc.FileData](w, "filereadstream", data, opts) +} + +// command "filesharecapability", wshserver.FileShareCapabilityCommand +func FileShareCapabilityCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) (wshrpc.FileShareCapability, error) { + resp, err := sendRpcRequestCallHelper[wshrpc.FileShareCapability](w, "filesharecapability", data, opts) + return resp, err +} + // command "filestreamtar", wshserver.FileStreamTarCommand func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { return sendRpcRequestResponseStreamHelper[iochantypes.Packet](w, "filestreamtar", data, opts) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 06de27f5cc..1946c76f4d 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -64,9 +64,11 @@ const ( Command_DeleteBlock = "deleteblock" Command_FileWrite = "filewrite" Command_FileRead = "fileread" + Command_FileReadStream = "filereadstream" Command_FileMove = "filemove" Command_FileCopy = "filecopy" Command_FileStreamTar = "filestreamtar" + Command_FileShareCapability = "filesharecapability" Command_EventPublish = "eventpublish" Command_EventRecv = "eventrecv" Command_EventSub = "eventsub" @@ -157,12 +159,14 @@ type WshRpcInterface interface { FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error FileReadCommand(ctx context.Context, data FileData) (*FileData, error) + FileReadStreamCommand(ctx context.Context, data FileData) <-chan RespOrErrorUnion[FileData] FileStreamTarCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[iochantypes.Packet] FileMoveCommand(ctx context.Context, data CommandFileCopyData) error FileCopyCommand(ctx context.Context, data CommandFileCopyData) error FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) FileListStreamCommand(ctx context.Context, data FileListData) <-chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] + FileShareCapabilityCommand(ctx context.Context, path string) (FileShareCapability, error) EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error EventUnsubCommand(ctx context.Context, data string) error @@ -717,3 +721,8 @@ type ConnExtData struct { ConnName string `json:"connname"` LogBlockId string `json:"logblockid,omitempty"` } + +type FileShareCapability struct { + CanAppend bool + CanMkdir bool +} diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index ff403cd739..938703da29 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -332,6 +332,10 @@ func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) return fileshare.Read(ctx, data) } +func (ws *WshServer) FileReadStreamCommand(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + return fileshare.ReadStream(ctx, data) +} + func (ws *WshServer) FileCopyCommand(ctx context.Context, data wshrpc.CommandFileCopyData) error { return fileshare.Copy(ctx, data) } @@ -373,6 +377,10 @@ func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.Com return nil } +func (ws *WshServer) FileShareCapabilityCommand(ctx context.Context, path string) (wshrpc.FileShareCapability, error) { + return fileshare.GetCapability(ctx, path) +} + func (ws *WshServer) DeleteSubBlockCommand(ctx context.Context, data wshrpc.CommandDeleteBlockData) error { err := wcore.DeleteBlock(ctx, data.BlockId, false) if err != nil { From eeeedf658cc93f373add2f1234c7dde4a5ae5a62 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 18:13:23 -0800 Subject: [PATCH 05/65] save --- pkg/remote/fileshare/s3fs/s3fs.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 7e3b293917..12adee3c53 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -154,7 +154,6 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o var entries []*wshrpc.FileInfo rtnCh := c.ListEntriesStream(ctx, conn, opts) for respUnion := range rtnCh { - log.Printf("respUnion: %v", respUnion) if respUnion.Error != nil { return nil, respUnion.Error } @@ -175,7 +174,6 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect if conn.Host == "" || conn.Host == "/" { buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { - log.Printf("error listing buckets: %v", err) return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](err) } var entries []*wshrpc.FileInfo @@ -226,7 +224,6 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect if err != nil { var noBucket *types.NoSuchBucket if !awsconn.CheckAccessDeniedErr(&err) && errors.As(err, &noBucket) { - log.Printf("Bucket %s does not exist.\n", conn.Host) err = noBucket } rtn <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) @@ -325,12 +322,8 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc exists = false err = nil default: - log.Printf("Either you don't have access to bucket %v or another error occurred. "+ - "Here's what happened: %v\n", bucketName, err) } } - } else { - log.Printf("Bucket %v exists and you already own it.", bucketName) } if exists { From b2ce9318508628e9efdb0dd392e425c6a8025156 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 18:41:06 -0800 Subject: [PATCH 06/65] fix "directory" handling for ls and rm --- cmd/wsh/cmd/wshcmd-file.go | 12 ------------ pkg/remote/fileshare/s3fs/s3fs.go | 22 +++++++++++++++------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index d3d4fece4c..ab2d1ea112 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -264,18 +264,6 @@ func fileRmRun(cmd *cobra.Command, args []string) error { if err != nil { return err } - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} - - _, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) - err = convertNotFoundErr(err) - if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", path) - } - if err != nil { - return fmt.Errorf("getting file info: %w", err) - } err = wshclient.FileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: path, Recursive: recursive}, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) if err != nil { diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 12adee3c53..37ff8d648b 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -166,12 +166,17 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o var slashRe = regexp.MustCompile(`/`) func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + bucket := conn.Host + objectKeyPrefix := conn.Path + if objectKeyPrefix != "" && !strings.HasSuffix(objectKeyPrefix, "/") { + objectKeyPrefix = objectKeyPrefix + "/" + } numToFetch := wshrpc.MaxDirSize if opts != nil && opts.Limit > 0 { numToFetch = min(opts.Limit, wshrpc.MaxDirSize) } numFetched := 0 - if conn.Host == "" || conn.Host == "/" { + if bucket == "" || bucket == "/" { buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](err) @@ -204,8 +209,8 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect var err error var output *s3.ListObjectsV2Output input := &s3.ListObjectsV2Input{ - Bucket: aws.String(conn.Host), - Prefix: aws.String(conn.Path), + Bucket: aws.String(bucket), + Prefix: aws.String(objectKeyPrefix), } objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) parentPath := getParentPathUri(conn) @@ -238,8 +243,8 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect if obj.LastModified != nil { lastModTime = obj.LastModified.UnixMilli() } - if obj.Key != nil && len(*obj.Key) > len(conn.Path) { - name := strings.TrimPrefix(*obj.Key, conn.Path) + if obj.Key != nil && len(*obj.Key) > len(objectKeyPrefix) { + name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) if strings.Count(name, "/") > 0 { name = strings.SplitN(name, "/", 2)[0] name = name + "/" // add trailing slash to indicate directory @@ -249,7 +254,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Path: conn.GetFullURI() + name, Name: name, IsDir: true, - Dir: conn.Path, + Dir: objectKeyPrefix, ModTime: lastModTime, Size: -1, } @@ -269,7 +274,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect entryMap[name] = &wshrpc.FileInfo{ Name: name, IsDir: false, - Dir: conn.Path, + Dir: objectKeyPrefix, Path: conn.GetFullURI() + name, ModTime: lastModTime, Size: size, @@ -432,6 +437,9 @@ func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recurs return errors.Join(errors.ErrUnsupported, fmt.Errorf("object key must be specified")) } if recursive { + if !strings.HasSuffix(objectKey, "/") { + objectKey = objectKey + "/" + } entries, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(objectKey), From cd54d9b4af185d572d63963e079a05c5243e1674 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 19:51:01 -0800 Subject: [PATCH 07/65] save tar copy work --- pkg/remote/fileshare/fstype/fstype.go | 5 + pkg/remote/fileshare/s3fs/s3fs.go | 164 +++++++++++++++++++++++++- pkg/remote/fileshare/wavefs/wavefs.go | 6 +- pkg/remote/fileshare/wshfs/wshfs.go | 10 +- 4 files changed, 171 insertions(+), 14 deletions(-) diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 5ca82ccd4c..dd5a83cb42 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -5,12 +5,17 @@ package fstype import ( "context" + "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) +const ( + DefaultTimeout = 30 * time.Second +) + type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 37ff8d648b..b5b4e86d7c 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -13,6 +13,8 @@ import ( "log" "regexp" "strings" + "sync" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -23,6 +25,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/tarcopy" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -147,7 +150,164 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { - return wshutil.SendErrCh[iochantypes.Packet](errors.ErrUnsupported) + + bucket := conn.Host + if bucket == "" || bucket == "/" { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("bucket must be specified")) + } + + objectPrefix := conn.Path + + wholeBucket := objectPrefix == "" || objectPrefix == "/" + singleFile := false + includeDir := false + if !wholeBucket { + finfo, err := c.Stat(ctx, conn) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return wshutil.SendErrCh[iochantypes.Packet](err) + } + if finfo != nil && !finfo.IsDir { + singleFile = true + } else if strings.HasSuffix(objectPrefix, "/") { + includeDir = true + } + } + + timeout := fstype.DefaultTimeout + + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) * time.Millisecond + } + + readerCtx, cancel := context.WithTimeout(context.Background(), timeout) + + pathPrefix := conn.Path + if includeDir || singleFile { + pathPrefix = getParentPath(conn) + } + + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, pathPrefix) + + go func() { + defer func() { + tarClose() + cancel() + }() + + if singleFile { + result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(conn.Path), + }) + + if err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + + defer result.Body.Close() + + finfo := &wshrpc.FileInfo{ + Name: conn.Path, + IsDir: false, + Size: *result.ContentLength, + ModTime: result.LastModified.UnixMilli(), + } + + if err := writeHeader(fileutil.ToFsFileInfo(finfo), conn.Path); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + + if _, err := io.Copy(fileWriter, result.Body); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + } else { + var err error + var output *s3.ListObjectsV2Output + var input *s3.ListObjectsV2Input + if wholeBucket { + input = &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + } + } else { + objectPrefix := conn.Path + if !strings.HasSuffix(objectPrefix, "/") { + objectPrefix = objectPrefix + "/" + } + input = &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(objectPrefix), + } + } + objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) + for objectPaginator.HasMorePages() { + output, err = objectPaginator.NextPage(ctx) + if err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + nextObjs := make(map[int]struct { + finfo *wshrpc.FileInfo + result *s3.GetObjectOutput + }, len(output.Contents)) + errs := make([]error, 0) + wg := sync.WaitGroup{} + defer func() { + for _, obj := range nextObjs { + if obj.result != nil { + obj.result.Body.Close() + } + } + }() + getObjectAndFileInfo := func(obj *types.Object, index int) { + defer wg.Done() + result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: obj.Key, + }) + if err != nil { + errs = append(errs, err) + return + } + finfo := &wshrpc.FileInfo{ + Name: *obj.Key, + IsDir: false, + Size: *obj.Size, + ModTime: result.LastModified.UnixMilli(), + } + nextObjs[index] = struct { + finfo *wshrpc.FileInfo + result *s3.GetObjectOutput + }{ + finfo: finfo, + result: result, + } + } + for _, obj := range output.Contents { + wg.Add(1) + go getObjectAndFileInfo(&obj, len(nextObjs)) + } + wg.Wait() + if len(errs) > 0 { + rtn <- wshutil.RespErr[iochantypes.Packet](errors.Join(errs...)) + return + } + for _, obj := range nextObjs { + if err := writeHeader(fileutil.ToFsFileInfo(obj.finfo), obj.finfo.Name); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + if _, err := io.Copy(fileWriter, obj.result.Body); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } + } + } + } + }() + } func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { @@ -312,6 +472,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc IsDir: true, Size: -1, ModTime: 0, + Path: fmt.Sprintf("%s://", conn.Scheme), }, nil } if objectKey == "" || objectKey == "/" { @@ -517,5 +678,4 @@ func getParentPath(conn *connparse.Connection) string { } } return parentPath - } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index aa05f49439..97ca492570 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -29,10 +29,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - DefaultTimeout = 30 * time.Second -) - type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -116,7 +112,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio pathPrefix := getPathPrefix(conn) schemeAndHost := conn.GetSchemeAndHost() + "/" - timeout := DefaultTimeout + timeout := fstype.DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 1401730bff..51330dc90b 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -16,10 +16,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - ThirtySeconds = 30 * 1000 -) - // This needs to be set by whoever initializes the client, either main-server or wshcmd-connserver var RpcClient *wshutil.WshRpc @@ -48,7 +44,7 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { timeout := opts.Timeout if timeout == 0 { - timeout = ThirtySeconds + timeout = fstype.DefaultTimeout.Milliseconds() } return wshclient.RemoteTarStreamCommand(RpcClient, wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host), Timeout: timeout}) } @@ -113,7 +109,7 @@ func (c WshClient) MoveInternal(ctx context.Context, srcConn, destConn *connpars } timeout := opts.Timeout if timeout == 0 { - timeout = ThirtySeconds + timeout = fstype.DefaultTimeout.Milliseconds() } return wshclient.RemoteFileMoveCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } @@ -128,7 +124,7 @@ func (c WshClient) CopyInternal(ctx context.Context, srcConn, destConn *connpars } timeout := opts.Timeout if timeout == 0 { - timeout = ThirtySeconds + timeout = fstype.DefaultTimeout.Milliseconds() } return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } From bafd175c39634e216f1240062edc675620241601 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 1 Feb 2025 19:51:17 -0800 Subject: [PATCH 08/65] save --- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index b5b4e86d7c..bbe44f222a 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -307,7 +307,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } }() - + return rtn } func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { From 8cb908fba5064fa1aa49c7b23024d2505c047106 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 2 Feb 2025 15:45:49 -0800 Subject: [PATCH 09/65] copyremote initial impl, fix for wavefs --- pkg/remote/fileshare/s3fs/s3fs.go | 81 ++++++++++++++++++++++++++- pkg/remote/fileshare/wavefs/wavefs.go | 24 +++++--- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index bbe44f222a..050a1458e2 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -4,6 +4,7 @@ package s3fs import ( + "archive/tar" "bytes" "context" "errors" @@ -11,6 +12,7 @@ import ( "io" "io/fs" "log" + "path" "regexp" "strings" "sync" @@ -569,7 +571,51 @@ func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse } func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { - return errors.ErrUnsupported + + destBucket := destConn.Host + destKey := destConn.Path + overwrite := opts != nil && opts.Overwrite + merge := opts != nil && opts.Merge + if destBucket == "" || destBucket == "/" { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("destination bucket must be specified")) + } + entries, err := c.ListEntries(ctx, destConn, nil) + if err != nil { + return err + } + if len(entries) > 1 && !merge { + return errors.Join(errors.ErrUnsupported, fmt.Errorf("more than one entry in destination, use merge option to copy to existing directory")) + } + destPrefix := getPathPrefix(destConn) + readCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { + log.Printf("copying %v", next.Name) + if next.Typeflag == tar.TypeDir { + return nil + } + fileName, err := cleanPath(path.Join(destPrefix, next.Name)) + if !overwrite { + for _, entry := range entries { + if entry.Name == fileName { + return fmt.Errorf("destination already exists: %v", fileName) + } + } + } + _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destKey + next.Name), + Body: reader, + ContentLength: aws.Int64(next.Size), + }) + return err + }) + if err != nil { + cancel(err) + return err + } + return nil } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { @@ -679,3 +725,36 @@ func getParentPath(conn *connparse.Connection) string { } return parentPath } + +func getPathPrefix(conn *connparse.Connection) string { + fullUri := conn.GetFullURI() + pathPrefix := fullUri + lastSlash := strings.LastIndex(fullUri, "/") + if lastSlash > 10 && lastSlash < len(fullUri)-1 { + pathPrefix = fullUri[:lastSlash+1] + } + return pathPrefix +} + +func cleanPath(path string) (string, error) { + if path == "" { + return "", fmt.Errorf("path is empty") + } + if strings.HasPrefix(path, "/") { + path = path[1:] + } + if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { + return "", fmt.Errorf("s3 path cannot start with ~, ., or ..") + } + var newParts []string + for _, part := range strings.Split(path, "/") { + if part == ".." { + if len(newParts) > 0 { + newParts = newParts[:len(newParts)-1] + } + } else if part != "." { + newParts = append(newParts, part) + } + } + return strings.Join(newParts, "/"), nil +} diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 97ca492570..6dd5df9263 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -443,12 +443,21 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } + overwrite := opts != nil && opts.Overwrite + merge := opts != nil && opts.Merge destPrefix := getPathPrefix(destConn) destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) + entries, err := c.ListEntries(ctx, srcConn, nil) + if err != nil { + return fmt.Errorf("error listing blockfiles: %w", err) + } + if len(entries) > 1 && !merge { + return fmt.Errorf("more than one entry at destination prefix, use merge flag to copy") + } readCtx, cancel := context.WithCancelCause(ctx) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { if next.Typeflag == tar.TypeDir { return nil } @@ -456,14 +465,11 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse if err != nil { return fmt.Errorf("error cleaning path: %w", err) } - _, err = filestore.WFS.Stat(ctx, zoneId, fileName) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("error getting blockfile info: %w", err) - } - err := filestore.WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOpts{}) - if err != nil { - return fmt.Errorf("error making blockfile: %w", err) + if !overwrite { + for _, entry := range entries { + if entry.Name == fileName { + return fmt.Errorf("destination already exists: %v", fileName) + } } } log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) From 506fd725dd0ba3dffa94a3cc514bcc2c1917cb5c Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 3 Feb 2025 17:14:58 -0800 Subject: [PATCH 10/65] save work --- pkg/remote/fileshare/pathtree/pathtree.go | 93 +++++++++ .../fileshare/pathtree/pathtree_test.go | 117 +++++++++++ pkg/remote/fileshare/s3fs/s3fs.go | 191 +++++++++++------- pkg/wshrpc/wshremote/wshremote.go | 4 +- 4 files changed, 327 insertions(+), 78 deletions(-) create mode 100644 pkg/remote/fileshare/pathtree/pathtree.go create mode 100644 pkg/remote/fileshare/pathtree/pathtree_test.go diff --git a/pkg/remote/fileshare/pathtree/pathtree.go b/pkg/remote/fileshare/pathtree/pathtree.go new file mode 100644 index 0000000000..3ec93ebe1a --- /dev/null +++ b/pkg/remote/fileshare/pathtree/pathtree.go @@ -0,0 +1,93 @@ +package pathtree + +import ( + "log" + "strings" +) + +type WalkFunc func(path string, isLeaf bool) error + +type Tree struct { + Root *Node + RootPath string + nodes map[string]*Node + delimiter string +} + +type Node struct { + Children map[string]*Node +} + +func (n *Node) Walk(curPath string, walkFunc WalkFunc, delimiter string) error { + if err := walkFunc(curPath, len(n.Children) == 0); err != nil { + return err + } + for name, child := range n.Children { + if err := child.Walk(curPath+delimiter+name, walkFunc, delimiter); err != nil { + return err + } + } + return nil +} + +func NewTree(path string, delimiter string) *Tree { + if !strings.HasSuffix(path, delimiter) { + path += delimiter + } + return &Tree{ + Root: &Node{ + Children: make(map[string]*Node), + }, + nodes: make(map[string]*Node), + RootPath: path, + delimiter: delimiter, + } +} + +func (t *Tree) Add(path string) { + relativePath := strings.TrimPrefix(path, t.RootPath) + log.Printf("relativePath: %s", relativePath) + + // If the path is not a child of the root path, ignore it + if relativePath == path { + return + } + + // If the path is already in the tree, ignore it + if t.nodes[relativePath] != nil { + return + } + + components := strings.Split(relativePath, t.delimiter) + + // Quick check to see if the parent path is already in the tree, in which case we can skip the loop + if len(components) > 1 { + parentPath := strings.Join(components[:len(components)-1], t.delimiter) + log.Printf("parentPath: %s", parentPath) + if t.nodes[parentPath] != nil { + lastPathComponent := components[len(components)-1] + t.nodes[parentPath].Children[lastPathComponent] = &Node{ + Children: make(map[string]*Node), + } + t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] + return + } + } + + currentNode := t.Root + for i, component := range components { + if _, ok := currentNode.Children[component]; !ok { + currentNode.Children[component] = &Node{ + Children: make(map[string]*Node), + } + curPath := strings.Join(components[:i+1], t.delimiter) + log.Printf("curPath: %s", curPath) + t.nodes[curPath] = currentNode.Children[component] + } + currentNode = currentNode.Children[component] + } +} + +func (t *Tree) Walk(walkFunc WalkFunc) error { + return t.Root.Walk(strings.TrimSuffix(t.RootPath, t.delimiter), walkFunc, t.delimiter) +} diff --git a/pkg/remote/fileshare/pathtree/pathtree_test.go b/pkg/remote/fileshare/pathtree/pathtree_test.go new file mode 100644 index 0000000000..5589b93230 --- /dev/null +++ b/pkg/remote/fileshare/pathtree/pathtree_test.go @@ -0,0 +1,117 @@ +package pathtree_test + +import ( + "errors" + "log" + "testing" + + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" +) + +func TestAdd(t *testing.T) { + t.Parallel() + + tree := initializeTree() + + // Check that the tree has the expected structure + if len(tree.Root.Children) != 3 { + t.Errorf("expected 3 children, got %d", len(tree.Root.Children)) + } + + if len(tree.Root.Children["a"].Children) != 3 { + t.Errorf("expected 3 children, got %d", len(tree.Root.Children["a"].Children)) + } + + if len(tree.Root.Children["b"].Children) != 1 { + t.Errorf("expected 1 child, got %d", len(tree.Root.Children["b"].Children)) + } + + if len(tree.Root.Children["b"].Children["g"].Children) != 1 { + t.Errorf("expected 1 child, got %d", len(tree.Root.Children["b"].Children["g"].Children)) + } + + if len(tree.Root.Children["b"].Children["g"].Children["h"].Children) != 0 { + t.Errorf("expected 0 children, got %d", len(tree.Root.Children["b"].Children["g"].Children["h"].Children)) + } + + if len(tree.Root.Children["c"].Children) != 0 { + t.Errorf("expected 0 children, got %d", len(tree.Root.Children["c"].Children)) + } + + // Check that adding the same path again does not change the tree + tree.Add("root/a/d") + if len(tree.Root.Children["a"].Children) != 3 { + t.Errorf("expected 3 children, got %d", len(tree.Root.Children["a"].Children)) + } + + // Check that adding a path that is not a child of the root path does not change the tree + tree.Add("etc/passwd") + if len(tree.Root.Children) != 3 { + t.Errorf("expected 3 children, got %d", len(tree.Root.Children)) + } +} + +func TestWalk(t *testing.T) { + t.Parallel() + + tree := initializeTree() + + // Check that the tree traverses all nodes and identifies leaf nodes correctly + pathMap := make(map[string]bool) + err := tree.Walk(func(path string, isLeaf bool) error { + pathMap[path] = isLeaf + return nil + }) + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + expectedPathMap := map[string]bool{ + "root/": false, + "root/a": false, + "root/a/d": true, + "root/a/e": true, + "root/a/f": true, + "root/b": false, + "root/b/g": false, + "root/b/g/h": true, + "root/c": true, + } + + log.Printf("pathMap: %v", pathMap) + + for path, isLeaf := range expectedPathMap { + if pathMap[path] != isLeaf { + if isLeaf { + t.Errorf("expected %s to be a leaf", path) + } else { + t.Errorf("expected %s to not be a leaf", path) + } + } + } + + expectedError := errors.New("test error") + + // Check that the walk function returns an error if it is returned by the walk function + err = tree.Walk(func(path string, isLeaf bool) error { + return expectedError + }) + if err != expectedError { + t.Errorf("expected error %v, got %v", expectedError, err) + } +} + +func initializeTree() *pathtree.Tree { + tree := pathtree.NewTree("root/", "/") + tree.Add("root/a") + tree.Add("root/b") + tree.Add("root/c") + tree.Add("root/a/d") + tree.Add("root/a/e") + tree.Add("root/a/f") + tree.Add("root/b/g") + tree.Add("root/b/g/h") + log.Printf("tree: %v", tree) + return tree +} diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 050a1458e2..0912b32350 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -7,6 +7,7 @@ import ( "archive/tar" "bytes" "context" + "encoding/base64" "errors" "fmt" "io" @@ -25,6 +26,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/tarcopy" @@ -161,34 +163,40 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, objectPrefix := conn.Path wholeBucket := objectPrefix == "" || objectPrefix == "/" - singleFile := false - includeDir := false + var singleFileObj *s3.GetObjectOutput + var err error if !wholeBucket { - finfo, err := c.Stat(ctx, conn) - if err != nil && !errors.Is(err, fs.ErrNotExist) { + singleFileObj, err = c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectPrefix), + }) + if err != nil { + var noKey *types.NoSuchKey + if errors.As(err, &noKey) { + log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectPrefix, bucket) + return wshutil.SendErrCh[iochantypes.Packet](noKey) + } return wshutil.SendErrCh[iochantypes.Packet](err) } - if finfo != nil && !finfo.IsDir { - singleFile = true - } else if strings.HasSuffix(objectPrefix, "/") { - includeDir = true - } } + singleFile := singleFileObj != nil + includeDir := singleFileObj == nil && objectPrefix != "" && !strings.HasSuffix(objectPrefix, "/") timeout := fstype.DefaultTimeout - if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } - readerCtx, cancel := context.WithTimeout(context.Background(), timeout) - pathPrefix := conn.Path - if includeDir || singleFile { - pathPrefix = getParentPath(conn) + tarPathPrefix := conn.Path + if singleFile || includeDir { + tarPathPrefix = path.Dir(objectPrefix) + if tarPathPrefix != "" && !strings.HasSuffix(tarPathPrefix, "/") { + tarPathPrefix = tarPathPrefix + "/" + } } - rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, pathPrefix) + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, tarPathPrefix) go func() { defer func() { @@ -196,6 +204,34 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, cancel() }() + writeFileAndHeader := func(objOutput *s3.GetObjectOutput, objKey string) error { + modTime := int64(0) + if objOutput != nil && objOutput.LastModified != nil { + modTime = objOutput.LastModified.UnixMilli() + } + size := int64(-1) + if objOutput != nil && objOutput.ContentLength != nil { + size = *objOutput.ContentLength + } + finfo := &wshrpc.FileInfo{ + Name: objKey, + IsDir: objOutput == nil, + Size: size, + ModTime: modTime, + Mode: 0644, + } + if err := writeHeader(fileutil.ToFsFileInfo(finfo), objKey); err != nil { + return err + } + if objOutput != nil { + base64Reader := base64.NewDecoder(base64.StdEncoding, objOutput.Body) + if _, err := io.Copy(fileWriter, base64Reader); err != nil { + return err + } + } + return nil + } + if singleFile { result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), @@ -209,25 +245,11 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, defer result.Body.Close() - finfo := &wshrpc.FileInfo{ - Name: conn.Path, - IsDir: false, - Size: *result.ContentLength, - ModTime: result.LastModified.UnixMilli(), - } - - if err := writeHeader(fileutil.ToFsFileInfo(finfo), conn.Path); err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return - } - - if _, err := io.Copy(fileWriter, result.Body); err != nil { + if err := writeFileAndHeader(result, conn.Path); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](err) return } } else { - var err error - var output *s3.ListObjectsV2Output var input *s3.ListObjectsV2Input if wholeBucket { input = &s3.ListObjectsV2Input{ @@ -243,6 +265,23 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, Prefix: aws.String(objectPrefix), } } + + // Make sure that the tree and outputMap are thread-safe + treeMutex := sync.Mutex{} + tree := pathtree.NewTree(tarPathPrefix, "/") + outputMap := make(map[string]*s3.GetObjectOutput) + + defer func() { + for _, obj := range outputMap { + if obj != nil { + obj.Body.Close() + } + } + }() + + // Fetch all the matching objects concurrently + var output *s3.ListObjectsV2Output + wg := sync.WaitGroup{} objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) for objectPaginator.HasMorePages() { output, err = objectPaginator.NextPage(ctx) @@ -250,20 +289,8 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, rtn <- wshutil.RespErr[iochantypes.Packet](err) return } - nextObjs := make(map[int]struct { - finfo *wshrpc.FileInfo - result *s3.GetObjectOutput - }, len(output.Contents)) errs := make([]error, 0) - wg := sync.WaitGroup{} - defer func() { - for _, obj := range nextObjs { - if obj.result != nil { - obj.result.Body.Close() - } - } - }() - getObjectAndFileInfo := func(obj *types.Object, index int) { + getObjectAndFileInfo := func(obj *types.Object) { defer wg.Done() result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), @@ -273,40 +300,28 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, errs = append(errs, err) return } - finfo := &wshrpc.FileInfo{ - Name: *obj.Key, - IsDir: false, - Size: *obj.Size, - ModTime: result.LastModified.UnixMilli(), - } - nextObjs[index] = struct { - finfo *wshrpc.FileInfo - result *s3.GetObjectOutput - }{ - finfo: finfo, - result: result, - } + treeMutex.Lock() + defer treeMutex.Unlock() + outputMap[*obj.Key] = result + tree.Add(*obj.Key) } for _, obj := range output.Contents { wg.Add(1) - go getObjectAndFileInfo(&obj, len(nextObjs)) + go getObjectAndFileInfo(&obj) } - wg.Wait() if len(errs) > 0 { rtn <- wshutil.RespErr[iochantypes.Packet](errors.Join(errs...)) return } - for _, obj := range nextObjs { - if err := writeHeader(fileutil.ToFsFileInfo(obj.finfo), obj.finfo.Name); err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return - } - if _, err := io.Copy(fileWriter, obj.result.Body); err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return - } - } } + + wg.Wait() + + // Walk the tree and write the tar entries + tree.Walk(func(path string, _ bool) error { + mapEntry := outputMap[path] + return writeFileAndHeader(mapEntry, path) + }) } }() return rtn @@ -573,20 +588,43 @@ func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { destBucket := destConn.Host - destKey := destConn.Path overwrite := opts != nil && opts.Overwrite merge := opts != nil && opts.Merge if destBucket == "" || destBucket == "/" { - return errors.Join(errors.ErrUnsupported, fmt.Errorf("destination bucket must be specified")) + return fmt.Errorf("destination bucket must be specified") } - entries, err := c.ListEntries(ctx, destConn, nil) + + var entries []*wshrpc.FileInfo + _, err := c.Stat(ctx, destConn) if err != nil { - return err + if errors.Is(err, fs.ErrNotExist) { + entries, err = c.ListEntries(ctx, destConn, nil) + if err != nil { + return err + } + if len(entries) > 0 { + if overwrite { + err := c.Delete(ctx, destConn, true) + if err != nil { + return err + } + } else if !merge { + return fmt.Errorf("more than one entry exists at prefix, neither force nor merge specified") + } + } + } else { + return err + } + } else if !overwrite { + return fmt.Errorf("destination already exists, use force to overwrite: %v", destConn.GetFullURI()) } - if len(entries) > 1 && !merge { - return errors.Join(errors.ErrUnsupported, fmt.Errorf("more than one entry in destination, use merge option to copy to existing directory")) + + destPrefix := destConn.Path + // Make sure destPrefix has a trailing slash if the destination is a "directory" + if destPrefix != "" && entries != nil && !strings.HasSuffix(destPrefix, "/") { + destPrefix = destPrefix + "/" } - destPrefix := getPathPrefix(destConn) + readCtx, cancel := context.WithCancelCause(ctx) defer cancel(nil) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) @@ -596,6 +634,7 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C return nil } fileName, err := cleanPath(path.Join(destPrefix, next.Name)) + log.Printf("cleaned path: %v", fileName) if !overwrite { for _, entry := range entries { if entry.Name == fileName { @@ -605,7 +644,7 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(destBucket), - Key: aws.String(destKey + next.Name), + Key: aws.String(fileName), Body: reader, ContentLength: aws.Int64(next.Size), }) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index f324f52fec..1fd44b4980 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -389,7 +389,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if destinfo.IsDir() { if !finfo.IsDir() { if !overwrite { - return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + return fmt.Errorf("cannot create directory %q, file exists at path, force not specified", nextPath) } else { err := os.Remove(nextPath) if err != nil { @@ -397,7 +397,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } } else if !merge && !overwrite { - return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) + return fmt.Errorf("cannot create directory %q, directory exists at path, neither force nor merge specified", nextPath) } else if overwrite { err := os.RemoveAll(nextPath) if err != nil { From 14c51cf234c7fd624a3d02689cd8d86382b7831d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 3 Feb 2025 17:18:39 -0800 Subject: [PATCH 11/65] check dir walk error --- pkg/remote/fileshare/s3fs/s3fs.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 0912b32350..82329cc12c 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -318,10 +318,13 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, wg.Wait() // Walk the tree and write the tar entries - tree.Walk(func(path string, _ bool) error { + if err := tree.Walk(func(path string, _ bool) error { mapEntry := outputMap[path] return writeFileAndHeader(mapEntry, path) - }) + }); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return + } } }() return rtn From 09f3bce0afa0d8d1139778e4d44cd413694b73a7 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 3 Feb 2025 18:00:14 -0800 Subject: [PATCH 12/65] save --- cmd/wsh/cmd/wshcmd-file.go | 2 +- pkg/remote/fileshare/s3fs/s3fs.go | 60 +++++++++++++-------------- pkg/remote/fileshare/wavefs/wavefs.go | 6 +-- pkg/util/fileutil/fileutil.go | 2 +- pkg/util/tarcopy/tarcopy.go | 11 +++-- pkg/wshrpc/wshremote/wshremote.go | 16 +++++-- 6 files changed, 53 insertions(+), 44 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index ab2d1ea112..1402020c47 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -31,7 +31,7 @@ const ( WaveFileScheme = "wavefile" WaveFilePrefix = "wavefile://" - DefaultFileTimeout = 5000 + DefaultFileTimeout = int64(30) * 1000 TimeoutYear = int64(365) * 24 * 60 * 60 * 1000 UriHelpText = ` diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 82329cc12c..29f1fdc660 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -128,7 +128,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da select { case <-ctx.Done(): log.Printf("context done") - rtn <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + rtn <- wshutil.RespErr[wshrpc.FileData](context.Cause(ctx)) return default: buf := make([]byte, min(bytesRemaining, wshrpc.FileChunkSize)) @@ -163,24 +163,28 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, objectPrefix := conn.Path wholeBucket := objectPrefix == "" || objectPrefix == "/" - var singleFileObj *s3.GetObjectOutput + var singleFileResult *s3.GetObjectOutput + defer func() { + if singleFileResult != nil { + singleFileResult.Body.Close() + } + }() var err error if !wholeBucket { - singleFileObj, err = c.client.GetObject(ctx, &s3.GetObjectInput{ + singleFileResult, err = c.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(objectPrefix), }) if err != nil { var noKey *types.NoSuchKey - if errors.As(err, &noKey) { - log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectPrefix, bucket) - return wshutil.SendErrCh[iochantypes.Packet](noKey) + var notFound *types.NotFound + if !errors.As(err, &noKey) && !errors.As(err, ¬Found) { + return wshutil.SendErrCh[iochantypes.Packet](err) } - return wshutil.SendErrCh[iochantypes.Packet](err) } } - singleFile := singleFileObj != nil - includeDir := singleFileObj == nil && objectPrefix != "" && !strings.HasSuffix(objectPrefix, "/") + singleFile := singleFileResult != nil + includeDir := singleFileResult == nil && objectPrefix != "" && !strings.HasSuffix(objectPrefix, "/") timeout := fstype.DefaultTimeout if opts.Timeout > 0 { @@ -190,10 +194,8 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, tarPathPrefix := conn.Path if singleFile || includeDir { - tarPathPrefix = path.Dir(objectPrefix) - if tarPathPrefix != "" && !strings.HasSuffix(tarPathPrefix, "/") { - tarPathPrefix = tarPathPrefix + "/" - } + tarPathPrefix = getParentPathString(tarPathPrefix) + log.Printf("objectPrefix: %v; tarPathPrefix: %v", objectPrefix, tarPathPrefix) } rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, tarPathPrefix) @@ -205,11 +207,12 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, }() writeFileAndHeader := func(objOutput *s3.GetObjectOutput, objKey string) error { + log.Printf("writing file %v", objKey) modTime := int64(0) if objOutput != nil && objOutput.LastModified != nil { modTime = objOutput.LastModified.UnixMilli() } - size := int64(-1) + size := int64(0) if objOutput != nil && objOutput.ContentLength != nil { size = *objOutput.ContentLength } @@ -229,23 +232,12 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, return err } } + log.Printf("wrote file %v", objKey) return nil } if singleFile { - result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(conn.Path), - }) - - if err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return - } - - defer result.Body.Close() - - if err := writeFileAndHeader(result, conn.Path); err != nil { + if err := writeFileAndHeader(singleFileResult, conn.Path); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](err) return } @@ -400,7 +392,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Path: parentPath, Name: "..", IsDir: true, - Size: -1, + Size: 0, }, }}} } @@ -436,7 +428,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect IsDir: true, Dir: objectKeyPrefix, ModTime: lastModTime, - Size: -1, + Size: 0, } prevUsedDirKeys[name] = struct{}{} numFetched++ @@ -490,7 +482,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: "/", IsDir: true, - Size: -1, + Size: 0, ModTime: 0, Path: fmt.Sprintf("%s://", conn.Scheme), }, nil @@ -516,7 +508,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: bucketName, IsDir: true, - Size: -1, + Size: 0, ModTime: 0, }, nil } else { @@ -755,8 +747,12 @@ func getParentPathUri(conn *connparse.Connection) string { } func getParentPath(conn *connparse.Connection) string { - var parentPath string hostAndPath := conn.GetPathWithHost() + return getParentPathString(hostAndPath) +} + +func getParentPathString(hostAndPath string) string { + parentPath := "" slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) if slashIndices != nil && len(slashIndices) > 0 { if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 6dd5df9263..4aba8562f7 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -50,7 +50,7 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, if !rtnData.Info.IsDir { for i := 0; i < dataLen; i += wshrpc.FileChunkSize { if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + ch <- wshutil.RespErr[wshrpc.FileData](context.Cause(ctx)) return } dataEnd := min(i+wshrpc.FileChunkSize, dataLen) @@ -59,7 +59,7 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, } else { for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + ch <- wshutil.RespErr[wshrpc.FileData](context.Cause(ctx)) return } ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} @@ -126,7 +126,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio }() for _, file := range list { if readerCtx.Err() != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](readerCtx.Err()) + rtn <- wshutil.RespErr[iochantypes.Packet](context.Cause(readerCtx)) return } file.Mode = 0644 diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 1f7ac20a05..15a47c40fc 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -235,7 +235,7 @@ func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[w for { select { case <-ctx.Done(): - return fmt.Errorf("context cancelled: %v", ctx.Err()) + return fmt.Errorf("context cancelled: %v", context.Cause(ctx)) case respUnion, ok := <-readCh: if !ok { drain = false diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 825ac2b31a..a2cad5e4de 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -42,17 +42,22 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) - return rtnChan, func(fi fs.FileInfo, file string) error { + return rtnChan, func(fi fs.FileInfo, path string) error { + log.Printf("path: %s\n", path) + log.Printf("fi: %v\n", fi) // generate tar header - header, err := tar.FileInfoHeader(fi, file) + header, err := tar.FileInfoHeader(fi, path) if err != nil { return err } + log.Printf("header: %v\n", header) + log.Printf("isDir: %v\n", fi.IsDir()) - header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) + header.Name = filepath.Clean(strings.TrimPrefix(path, pathPrefix)) if err := validatePath(header.Name); err != nil { return err } + log.Printf("header.Name: %s\n", header.Name) // write header if err := tarWriter.WriteHeader(header); err != nil { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 1fd44b4980..724ffafac6 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -376,6 +376,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } numFiles++ finfo := next.FileInfo() + log.Printf("copying file %q\n", next.Name) nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -388,9 +389,11 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if destinfo != nil { if destinfo.IsDir() { if !finfo.IsDir() { + log.Println("dest is dir, src is file") if !overwrite { return fmt.Errorf("cannot create directory %q, file exists at path, force not specified", nextPath) } else { + log.Printf("removing file %q\n", nextPath) err := os.Remove(nextPath) if err != nil { return fmt.Errorf("cannot remove file %q: %w", nextPath, err) @@ -399,6 +402,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } else if !merge && !overwrite { return fmt.Errorf("cannot create directory %q, directory exists at path, neither force nor merge specified", nextPath) } else if overwrite { + log.Printf("removing directory %q\n", nextPath) err := os.RemoveAll(nextPath) if err != nil { return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) @@ -406,9 +410,11 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } else { if finfo.IsDir() { + log.Println("dest is file, src is dir") if !overwrite { return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) } else { + log.Printf("removing directory %q\n", nextPath) err := os.RemoveAll(nextPath) if err != nil { return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) @@ -420,11 +426,13 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } else { if finfo.IsDir() { + log.Printf("creating directory %q\n", nextPath) err := os.MkdirAll(nextPath, finfo.Mode()) if err != nil { return fmt.Errorf("cannot create directory %q: %w", nextPath, err) } } else { + log.Printf("creating file %q\n", nextPath) err := os.MkdirAll(filepath.Dir(nextPath), 0755) if err != nil { return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) @@ -658,19 +666,19 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C } destPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(destConn.Path)) destinfo, err := os.Stat(destPathCleaned) - if err == nil { + if err == nil && destinfo != nil { if !destinfo.IsDir() { if !overwrite { - return fmt.Errorf("destination %q already exists, use overwrite option", destUri) + return fmt.Errorf("destination %q already exists, use overwrite option", destPathCleaned) } else { err := os.Remove(destPathCleaned) if err != nil { - return fmt.Errorf("cannot remove file %q: %w", destUri, err) + return fmt.Errorf("cannot remove file %q: %w", destPathCleaned, err) } } } } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat destination %q: %w", destUri, err) + return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) } srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { From 92bb98edf040a624bc0512c546fb648c1ece37cf Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 12:47:04 -0800 Subject: [PATCH 13/65] add dir mode --- pkg/remote/fileshare/pathtree/pathtree.go | 60 ++++++++++++------- .../fileshare/pathtree/pathtree_test.go | 37 +++++------- pkg/remote/fileshare/s3fs/s3fs.go | 47 +++++++++++---- pkg/util/fileutil/fileutil.go | 6 +- pkg/util/tarcopy/tarcopy.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 9 +-- 6 files changed, 99 insertions(+), 62 deletions(-) diff --git a/pkg/remote/fileshare/pathtree/pathtree.go b/pkg/remote/fileshare/pathtree/pathtree.go index 3ec93ebe1a..fd59af170a 100644 --- a/pkg/remote/fileshare/pathtree/pathtree.go +++ b/pkg/remote/fileshare/pathtree/pathtree.go @@ -5,7 +5,7 @@ import ( "strings" ) -type WalkFunc func(path string, isLeaf bool) error +type WalkFunc func(path string, numChildren int) error type Tree struct { Root *Node @@ -19,7 +19,7 @@ type Node struct { } func (n *Node) Walk(curPath string, walkFunc WalkFunc, delimiter string) error { - if err := walkFunc(curPath, len(n.Children) == 0); err != nil { + if err := walkFunc(curPath, len(n.Children)); err != nil { return err } for name, child := range n.Children { @@ -31,7 +31,7 @@ func (n *Node) Walk(curPath string, walkFunc WalkFunc, delimiter string) error { } func NewTree(path string, delimiter string) *Tree { - if !strings.HasSuffix(path, delimiter) { + if path != "" && !strings.HasSuffix(path, delimiter) { path += delimiter } return &Tree{ @@ -45,13 +45,20 @@ func NewTree(path string, delimiter string) *Tree { } func (t *Tree) Add(path string) { - relativePath := strings.TrimPrefix(path, t.RootPath) - log.Printf("relativePath: %s", relativePath) + log.Printf("tree.Add: path: %s", path) + var relativePath string + if t.RootPath == "" { + relativePath = path + } else { + relativePath = strings.TrimPrefix(path, t.RootPath) + + // If the path is not a child of the root path, ignore it + if relativePath == path { + return + } - // If the path is not a child of the root path, ignore it - if relativePath == path { - return } + log.Printf("relativePath: %s", relativePath) // If the path is already in the tree, ignore it if t.nodes[relativePath] != nil { @@ -59,24 +66,27 @@ func (t *Tree) Add(path string) { } components := strings.Split(relativePath, t.delimiter) + log.Printf("components: %v", components) - // Quick check to see if the parent path is already in the tree, in which case we can skip the loop - if len(components) > 1 { - parentPath := strings.Join(components[:len(components)-1], t.delimiter) - log.Printf("parentPath: %s", parentPath) - if t.nodes[parentPath] != nil { - lastPathComponent := components[len(components)-1] - t.nodes[parentPath].Children[lastPathComponent] = &Node{ - Children: make(map[string]*Node), - } - t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] - return - } - } + // // Quick check to see if the parent path is already in the tree, in which case we can skip the loop + // if len(components) > 1 { + // parentPath := strings.Join(components[:len(components)-1], t.delimiter) + // log.Printf("parentPath: %s", parentPath) + // if t.nodes[parentPath] != nil { + // lastPathComponent := components[len(components)-1] + // t.nodes[parentPath].Children[lastPathComponent] = &Node{ + // Children: make(map[string]*Node), + // } + // t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] + // return + // } + // } currentNode := t.Root for i, component := range components { + log.Printf("component: %s", component) if _, ok := currentNode.Children[component]; !ok { + log.Printf("Adding component: %s", component) currentNode.Children[component] = &Node{ Children: make(map[string]*Node), } @@ -89,5 +99,11 @@ func (t *Tree) Add(path string) { } func (t *Tree) Walk(walkFunc WalkFunc) error { - return t.Root.Walk(strings.TrimSuffix(t.RootPath, t.delimiter), walkFunc, t.delimiter) + log.Printf("RootPath: %s", t.RootPath) + for key, child := range t.Root.Children { + if err := child.Walk(t.RootPath+key, walkFunc, t.delimiter); err != nil { + return err + } + } + return nil } diff --git a/pkg/remote/fileshare/pathtree/pathtree_test.go b/pkg/remote/fileshare/pathtree/pathtree_test.go index 5589b93230..efaa25578e 100644 --- a/pkg/remote/fileshare/pathtree/pathtree_test.go +++ b/pkg/remote/fileshare/pathtree/pathtree_test.go @@ -57,9 +57,9 @@ func TestWalk(t *testing.T) { tree := initializeTree() // Check that the tree traverses all nodes and identifies leaf nodes correctly - pathMap := make(map[string]bool) - err := tree.Walk(func(path string, isLeaf bool) error { - pathMap[path] = isLeaf + pathMap := make(map[string]int) + err := tree.Walk(func(path string, numChildren int) error { + pathMap[path] = numChildren return nil }) @@ -67,34 +67,29 @@ func TestWalk(t *testing.T) { t.Errorf("unexpected error: %v", err) } - expectedPathMap := map[string]bool{ - "root/": false, - "root/a": false, - "root/a/d": true, - "root/a/e": true, - "root/a/f": true, - "root/b": false, - "root/b/g": false, - "root/b/g/h": true, - "root/c": true, + expectedPathMap := map[string]int{ + "root/a": 3, + "root/a/d": 0, + "root/a/e": 0, + "root/a/f": 0, + "root/b": 1, + "root/b/g": 1, + "root/b/g/h": 0, + "root/c": 0, } log.Printf("pathMap: %v", pathMap) - for path, isLeaf := range expectedPathMap { - if pathMap[path] != isLeaf { - if isLeaf { - t.Errorf("expected %s to be a leaf", path) - } else { - t.Errorf("expected %s to not be a leaf", path) - } + for path, numChildren := range expectedPathMap { + if pathMap[path] != numChildren { + t.Errorf("expected %d children for path %s, got %d", numChildren, path, pathMap[path]) } } expectedError := errors.New("test error") // Check that the walk function returns an error if it is returned by the walk function - err = tree.Walk(func(path string, isLeaf bool) error { + err = tree.Walk(func(path string, numChildren int) error { return expectedError }) if err != expectedError { diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 29f1fdc660..25d8017180 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -13,6 +13,7 @@ import ( "io" "io/fs" "log" + "os" "path" "regexp" "strings" @@ -34,6 +35,11 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + FileMode os.FileMode = 0644 + DirMode os.FileMode = 0755 | os.ModeDir +) + type S3Client struct { client *s3.Client } @@ -206,27 +212,38 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, cancel() }() - writeFileAndHeader := func(objOutput *s3.GetObjectOutput, objKey string) error { + writeFileAndHeader := func(objOutput *s3.GetObjectOutput, objKey string, numChildren int) error { log.Printf("writing file %v", objKey) - modTime := int64(0) - if objOutput != nil && objOutput.LastModified != nil { - modTime = objOutput.LastModified.UnixMilli() - } + modTime := int64(time.Now().Unix()) size := int64(0) - if objOutput != nil && objOutput.ContentLength != nil { - size = *objOutput.ContentLength + isDir := objOutput == nil + if isDir { + size = int64(numChildren) + } else { + if objOutput.ContentLength != nil { + size = *objOutput.ContentLength + } + if objOutput.LastModified != nil { + modTime = objOutput.LastModified.UnixMilli() + } + } + + mode := FileMode + if isDir { + mode = DirMode } + finfo := &wshrpc.FileInfo{ Name: objKey, - IsDir: objOutput == nil, + IsDir: isDir, Size: size, ModTime: modTime, - Mode: 0644, + Mode: mode, } if err := writeHeader(fileutil.ToFsFileInfo(finfo), objKey); err != nil { return err } - if objOutput != nil { + if !isDir { base64Reader := base64.NewDecoder(base64.StdEncoding, objOutput.Body) if _, err := io.Copy(fileWriter, base64Reader); err != nil { return err @@ -237,7 +254,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } if singleFile { - if err := writeFileAndHeader(singleFileResult, conn.Path); err != nil { + if err := writeFileAndHeader(singleFileResult, conn.Path, 0); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](err) return } @@ -309,10 +326,14 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, wg.Wait() + // log.Printf("outputMap: %v", outputMap) + // Walk the tree and write the tar entries - if err := tree.Walk(func(path string, _ bool) error { + if err := tree.Walk(func(path string, numChildren int) error { mapEntry := outputMap[path] - return writeFileAndHeader(mapEntry, path) + // log.Printf("path: %v", path) + // log.Printf("mapEntry: %v", mapEntry) + return writeFileAndHeader(mapEntry, path, numChildren) }); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](err) return diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 15a47c40fc..8c879fb2af 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/fs" + "log" "mime" "net/http" "os" @@ -209,13 +210,16 @@ func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { if fi == nil { panic("ToFsFileInfo: nil FileInfo") } - return FsFileInfo{ + fsFileInfo := FsFileInfo{ NameInternal: fi.Name, ModeInternal: fi.Mode, SizeInternal: fi.Size, ModTimeInternal: fi.ModTime, IsDirInternal: fi.IsDir, } + + log.Printf("fi: %v; fsFileInfo: %v\n", fi, fsFileInfo) + return fsFileInfo } func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index a2cad5e4de..2c88d31aff 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -51,7 +51,7 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. return err } log.Printf("header: %v\n", header) - log.Printf("isDir: %v\n", fi.IsDir()) + log.Printf("isDir: %v; headerIsDir: %v\n", fi.IsDir(), header.Typeflag) header.Name = filepath.Clean(strings.TrimPrefix(path, pathPrefix)) if err := validatePath(header.Name); err != nil { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 724ffafac6..b5541a629d 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -376,19 +376,20 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } numFiles++ finfo := next.FileInfo() - log.Printf("copying file %q\n", next.Name) + log.Printf("copying file %v\n", finfo) + srcIsDir := finfo.IsDir() nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } - if !finfo.IsDir() { + if !srcIsDir { totalBytes += finfo.Size() } if destinfo != nil { if destinfo.IsDir() { - if !finfo.IsDir() { + if !srcIsDir { log.Println("dest is dir, src is file") if !overwrite { return fmt.Errorf("cannot create directory %q, file exists at path, force not specified", nextPath) @@ -409,7 +410,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } } else { - if finfo.IsDir() { + if srcIsDir { log.Println("dest is file, src is dir") if !overwrite { return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) From 24ac975da0973d48ea4190ea02b6fbbfd9abeb9a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 16:46:52 -0800 Subject: [PATCH 14/65] lots of fixes --- cmd/wsh/cmd/wshcmd-file-util.go | 12 +- cmd/wsh/cmd/wshcmd-file.go | 31 ++--- pkg/remote/connparse/connparse.go | 16 +-- pkg/remote/fileshare/s3fs/s3fs.go | 180 ++++++++++++++++-------------- pkg/util/fileutil/fileutil.go | 3 - pkg/util/iochan/iochan.go | 2 + pkg/util/tarcopy/tarcopy.go | 39 +++---- pkg/util/utilfn/utilfn.go | 18 +++ pkg/wshrpc/wshremote/wshremote.go | 1 + 9 files changed, 156 insertions(+), 146 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index fdb65e3c4a..98970cf858 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -29,14 +29,14 @@ func convertNotFoundErr(err error) error { } func ensureFile(fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { - info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - err = wshclient.FileCreateCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err = wshclient.FileCreateCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return nil, fmt.Errorf("creating file: %w", err) } - info, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + info, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return nil, fmt.Errorf("getting file info: %w", err) } @@ -52,7 +52,7 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { // First truncate the file with an empty write emptyWrite := fileData emptyWrite.Data64 = "" - err := wshclient.FileWriteCommand(RpcClient, emptyWrite, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err := wshclient.FileWriteCommand(RpcClient, emptyWrite, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("initializing file with empty write: %w", err) } @@ -90,8 +90,8 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { return nil } -func streamReadFromFile(ctx context.Context, fileData wshrpc.FileData, size int64, writer io.Writer) error { - ch := wshclient.FileReadStreamCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) +func streamReadFromFile(ctx context.Context, fileData wshrpc.FileData, writer io.Writer) error { + ch := wshclient.FileReadStreamCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) return fileutil.ReadFileStreamToWriter(ctx, ch, writer) } diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 1402020c47..6b03df845c 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -31,8 +31,7 @@ const ( WaveFileScheme = "wavefile" WaveFilePrefix = "wavefile://" - DefaultFileTimeout = int64(30) * 1000 - TimeoutYear = int64(365) * 24 * 60 * 60 * 1000 + TimeoutYear = int64(365) * 24 * 60 * 60 * 1000 UriHelpText = ` @@ -83,12 +82,12 @@ Wave Terminal is capable of managing files from remote SSH hosts, S3-compatible systems, and the internal Wave filesystem. Files are addressed via URIs, which vary depending on the storage system.` + UriHelpText} -var fileTimeout int +var fileTimeout int64 func init() { rootCmd.AddCommand(fileCmd) - fileCmd.PersistentFlags().IntVarP(&fileTimeout, "timeout", "t", 15000, "timeout in milliseconds for long operations") + fileCmd.PersistentFlags().Int64VarP(&fileTimeout, "timeout", "t", 15000, "timeout in milliseconds for long operations") fileListCmd.Flags().BoolP("recursive", "r", false, "list subdirectories recursively") fileListCmd.Flags().BoolP("long", "l", false, "use long listing format") @@ -202,17 +201,7 @@ func fileCatRun(cmd *cobra.Command, args []string) error { Info: &wshrpc.FileInfo{ Path: path}} - // Get file info first to check existence and get size - info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) - err = convertNotFoundErr(err) - if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", path) - } - if err != nil { - return fmt.Errorf("getting file info: %w", err) - } - - err = streamReadFromFile(cmd.Context(), fileData, info.Size, os.Stdout) + err = streamReadFromFile(cmd.Context(), fileData, os.Stdout) if err != nil { return fmt.Errorf("reading file: %w", err) } @@ -229,7 +218,7 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { Info: &wshrpc.FileInfo{ Path: path}} - info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return fmt.Errorf("%s: no such file", path) @@ -265,7 +254,7 @@ func fileRmRun(cmd *cobra.Command, args []string) error { return err } - err = wshclient.FileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: path, Recursive: recursive}, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err = wshclient.FileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: path, Recursive: recursive}, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("removing file: %w", err) } @@ -282,7 +271,7 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { Info: &wshrpc.FileInfo{ Path: path}} - capability, err := wshclient.FileShareCapabilityCommand(RpcClient, fileData.Info.Path, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + capability, err := wshclient.FileShareCapabilityCommand(RpcClient, fileData.Info.Path, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("getting fileshare capability: %w", err) } @@ -303,7 +292,7 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { } } fileData.Data64 = base64.StdEncoding.EncodeToString(buf[:n]) - err = wshclient.FileWriteCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err = wshclient.FileWriteCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("writing file: %w", err) } @@ -350,7 +339,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { if buf.Len() >= 8192 { // 8KB batch size fileData.Data64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: int64(fileTimeout)}) + err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("appending to file: %w", err) } @@ -361,7 +350,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { if buf.Len() > 0 { fileData.Data64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: int64(fileTimeout)}) + err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("appending to file: %w", err) } diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 9951065617..fff13161ba 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -47,6 +47,9 @@ func (c *Connection) GetPathWithHost() string { if c.Host == "" { return "" } + if c.Path == "" { + return c.Host + } if strings.HasPrefix(c.Path, "/") { return c.Host + c.Path } @@ -107,16 +110,13 @@ func ParseURI(uri string) (*Connection, error) { parseGenericPath := func() { split = strings.SplitN(rest, "/", 2) host = split[0] - if len(split) > 1 { + if len(split) > 1 && split[1] != "" { remotePath = split[1] + } else if strings.HasSuffix(rest, "/") { + // preserve trailing slash + remotePath = "/" } else { - split = strings.SplitN(rest, "/", 2) - host = split[0] - if len(split) > 1 { - remotePath = split[1] - } else { - remotePath = "/" - } + remotePath = "" } } parseWshPath := func() { diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 25d8017180..de6b2072a2 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,7 +5,6 @@ package s3fs import ( "archive/tar" - "bytes" "context" "encoding/base64" "errors" @@ -160,17 +159,20 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { + recursive := opts != nil && opts.Recursive bucket := conn.Host if bucket == "" || bucket == "/" { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("bucket must be specified")) } - objectPrefix := conn.Path + // whether the operation is on the whole bucket + wholeBucket := conn.Path == "" || conn.Path == "/" - wholeBucket := objectPrefix == "" || objectPrefix == "/" + // get the object if it's a single file operation var singleFileResult *s3.GetObjectOutput defer func() { + // in case we error out before the object gets copied, make sure to close it if singleFileResult != nil { singleFileResult.Body.Close() } @@ -179,9 +181,10 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, if !wholeBucket { singleFileResult, err = c.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucket), - Key: aws.String(objectPrefix), + Key: aws.String(conn.Path), // does not care if the path has a prefixed slash }) if err != nil { + // if the object doesn't exist, we can assume the prefix is a directory and continue var noKey *types.NoSuchKey var notFound *types.NotFound if !errors.As(err, &noKey) && !errors.As(err, ¬Found) { @@ -189,8 +192,15 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } } + + // whether the operation is on a single file singleFile := singleFileResult != nil - includeDir := singleFileResult == nil && objectPrefix != "" && !strings.HasSuffix(objectPrefix, "/") + if !singleFile && !recursive { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("recursive must be set to true for non-single file operations")) + } + + // whether to include the directory itself in the tar + includeDir := (wholeBucket && conn.Path == "") || (singleFileResult == nil && conn.Path != "" && !strings.HasSuffix(conn.Path, "/")) timeout := fstype.DefaultTimeout if opts.Timeout > 0 { @@ -198,10 +208,19 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } readerCtx, cancel := context.WithTimeout(context.Background(), timeout) + // the prefix that should be removed from the tar paths tarPathPrefix := conn.Path - if singleFile || includeDir { + + if wholeBucket { + // we treat the bucket name as the root directory. If we're not including the directory itself, we need to remove the bucket name from the tar paths + if includeDir { + tarPathPrefix = "" + } else { + tarPathPrefix = bucket + } + } else if singleFile || includeDir { + // if we're including the directory itself, we need to remove the last part of the path tarPathPrefix = getParentPathString(tarPathPrefix) - log.Printf("objectPrefix: %v; tarPathPrefix: %v", objectPrefix, tarPathPrefix) } rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, tarPathPrefix) @@ -212,55 +231,27 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, cancel() }() - writeFileAndHeader := func(objOutput *s3.GetObjectOutput, objKey string, numChildren int) error { - log.Printf("writing file %v", objKey) - modTime := int64(time.Now().Unix()) - size := int64(0) - isDir := objOutput == nil - if isDir { - size = int64(numChildren) - } else { - if objOutput.ContentLength != nil { - size = *objOutput.ContentLength - } - if objOutput.LastModified != nil { - modTime = objOutput.LastModified.UnixMilli() - } - } - - mode := FileMode - if isDir { - mode = DirMode + // below we get the objects concurrently so we need to store the results in a map + objMap := make(map[string]*s3.GetObjectOutput) + // close the objects when we're done + defer func() { + for key, obj := range objMap { + log.Printf("closing object %v", key) + obj.Body.Close() } + }() - finfo := &wshrpc.FileInfo{ - Name: objKey, - IsDir: isDir, - Size: size, - ModTime: modTime, - Mode: mode, - } - if err := writeHeader(fileutil.ToFsFileInfo(finfo), objKey); err != nil { - return err - } - if !isDir { - base64Reader := base64.NewDecoder(base64.StdEncoding, objOutput.Body) - if _, err := io.Copy(fileWriter, base64Reader); err != nil { - return err - } - } - log.Printf("wrote file %v", objKey) - return nil - } + // tree to keep track of the paths we've added and insert fake directories for subpaths + tree := pathtree.NewTree(tarPathPrefix, "/") if singleFile { - if err := writeFileAndHeader(singleFileResult, conn.Path, 0); err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return - } + objMap[conn.Path] = singleFileResult + tree.Add(conn.Path) } else { + // list the objects in the bucket and add them to a tree that we can then walk to write the tar entries var input *s3.ListObjectsV2Input if wholeBucket { + // get all the objects in the bucket input = &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), } @@ -275,22 +266,13 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } - // Make sure that the tree and outputMap are thread-safe - treeMutex := sync.Mutex{} - tree := pathtree.NewTree(tarPathPrefix, "/") - outputMap := make(map[string]*s3.GetObjectOutput) - - defer func() { - for _, obj := range outputMap { - if obj != nil { - obj.Body.Close() - } - } - }() + // mutex to protect the tree and objMap since we're fetching objects concurrently + treeMapMutex := sync.Mutex{} + // wait group to await the finished fetches + wg := sync.WaitGroup{} // Fetch all the matching objects concurrently var output *s3.ListObjectsV2Output - wg := sync.WaitGroup{} objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) for objectPaginator.HasMorePages() { output, err = objectPaginator.NextPage(ctx) @@ -309,10 +291,14 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, errs = append(errs, err) return } - treeMutex.Lock() - defer treeMutex.Unlock() - outputMap[*obj.Key] = result - tree.Add(*obj.Key) + path := *obj.Key + if wholeBucket { + path = bucket + "/" + path + } + treeMapMutex.Lock() + defer treeMapMutex.Unlock() + objMap[path] = result + tree.Add(path) } for _, obj := range output.Contents { wg.Add(1) @@ -323,21 +309,48 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, return } } - wg.Wait() + } - // log.Printf("outputMap: %v", outputMap) + // Walk the tree and write the tar entries + if err := tree.Walk(func(path string, numChildren int) error { + mapEntry, isFile := objMap[path] - // Walk the tree and write the tar entries - if err := tree.Walk(func(path string, numChildren int) error { - mapEntry := outputMap[path] - // log.Printf("path: %v", path) - // log.Printf("mapEntry: %v", mapEntry) - return writeFileAndHeader(mapEntry, path, numChildren) - }); err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) - return + // default vals assume entry is dir, since mapEntry might not exist + modTime := int64(time.Now().Unix()) + mode := DirMode + size := int64(numChildren) + + if isFile { + mode = FileMode + size = *mapEntry.ContentLength + if mapEntry.LastModified != nil { + modTime = mapEntry.LastModified.UnixMilli() + } } + + finfo := &wshrpc.FileInfo{ + Name: path, + IsDir: !isFile, + Size: size, + ModTime: modTime, + Mode: mode, + } + if err := writeHeader(fileutil.ToFsFileInfo(finfo), path); err != nil { + return err + } + if isFile { + if n, err := io.Copy(fileWriter, mapEntry.Body); err != nil { + return err + } else if n != size { + return fmt.Errorf("error copying %v; expected to read %d bytes, but read %d", path, size, n) + } + } + return nil + }); err != nil { + log.Printf("error walking tree: %v", err) + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return } }() return rtn @@ -536,7 +549,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return nil, fmt.Errorf("bucket %v does not exist", bucketName) } } - result, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ + result, err := c.client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ Bucket: aws.String(bucketName), Key: aws.String(objectKey), }) @@ -551,8 +564,8 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return nil, err } size := int64(0) - if result.ContentLength != nil { - size = *result.ContentLength + if result.ObjectSize != nil { + size = *result.ObjectSize } lastModified := int64(0) if result.LastModified != nil { @@ -578,9 +591,10 @@ func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket and object key must be specified")) } _, err := c.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(objectKey), - Body: bytes.NewReader([]byte(data.Data64)), + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + Body: base64.NewDecoder(base64.StdEncoding, strings.NewReader(data.Data64)), + ContentLength: aws.Int64(int64(base64.StdEncoding.DecodedLen(len(data.Data64)))), }) return err } diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 8c879fb2af..1911efe24a 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/fs" - "log" "mime" "net/http" "os" @@ -217,8 +216,6 @@ func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { ModTimeInternal: fi.ModTime, IsDirInternal: fi.IsDir, } - - log.Printf("fi: %v; fsFileInfo: %v\n", fi, fsFileInfo) return fsFileInfo } diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 98fb94a196..8177dae73f 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "log" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -22,6 +23,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func ch := make(chan wshrpc.RespOrErrorUnion[iochantypes.Packet], 32) go func() { defer func() { + log.Printf("Closing ReaderChan\n") close(ch) callback() }() diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 2c88d31aff..e987467110 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -18,6 +18,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -39,9 +40,14 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { - gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) + log.Printf("Closing pipe reader\n") + utilfn.GracefulClose(pipeReader, tarCopySrcName, pipeReaderName, maxRetries, retryDelay) }) + if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") { + pathPrefix += "/" + } + return rtnChan, func(fi fs.FileInfo, path string) error { log.Printf("path: %s\n", path) log.Printf("fi: %v\n", fi) @@ -50,14 +56,13 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. if err != nil { return err } - log.Printf("header: %v\n", header) - log.Printf("isDir: %v; headerIsDir: %v\n", fi.IsDir(), header.Typeflag) header.Name = filepath.Clean(strings.TrimPrefix(path, pathPrefix)) if err := validatePath(header.Name); err != nil { return err } - log.Printf("header.Name: %s\n", header.Name) + + log.Printf("header: %v\n", header) // write header if err := tarWriter.WriteHeader(header); err != nil { @@ -65,8 +70,9 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } return nil }, tarWriter, func() { - gracefulClose(tarWriter, tarCopySrcName, tarWriterName) - gracefulClose(pipeWriter, tarCopySrcName, pipeWriterName) + log.Printf("Closing tar writer\n") + utilfn.GracefulClose(tarWriter, tarCopySrcName, tarWriterName, maxRetries, retryDelay) + utilfn.GracefulClose(pipeWriter, tarCopySrcName, pipeWriterName, maxRetries, retryDelay) } } @@ -86,12 +92,12 @@ func validatePath(path string) error { func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { - gracefulClose(pipeWriter, tarCopyDestName, pipeWriterName) + utilfn.GracefulClose(pipeWriter, tarCopyDestName, pipeWriterName, maxRetries, retryDelay) cancel(nil) }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - if !gracefulClose(pipeReader, tarCopyDestName, pipeReaderName) { + if !utilfn.GracefulClose(pipeReader, tarCopyDestName, pipeReaderName, maxRetries, retryDelay) { // If the pipe reader cannot be closed, cancel the context. This should kill the writer goroutine. cancel(nil) } @@ -123,20 +129,3 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan } } } - -func gracefulClose(closer io.Closer, debugName string, closerName string) bool { - closed := false - for retries := 0; retries < maxRetries; retries++ { - if err := closer.Close(); err != nil { - log.Printf("%s: error closing %s: %v, trying again in %dms\n", debugName, closerName, err, retryDelay.Milliseconds()) - time.Sleep(retryDelay) - continue - } - closed = true - break - } - if !closed { - log.Printf("%s: unable to close %s after %d retries\n", debugName, closerName, maxRetries) - } - return closed -} diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index fb04dd61c8..0ea266dec2 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "log" "math" mathrand "math/rand" "os" @@ -948,3 +949,20 @@ func DumpGoRoutineStacks() { n := runtime.Stack(buf, true) os.Stdout.Write(buf[:n]) } + +func GracefulClose(closer io.Closer, debugName string, closerName string, maxRetries int, retryDelay time.Duration) bool { + closed := false + for retries := 0; retries < maxRetries; retries++ { + if err := closer.Close(); err != nil { + log.Printf("%s: error closing %s: %v, trying again in %dms\n", debugName, closerName, err, retryDelay.Milliseconds()) + time.Sleep(retryDelay) + continue + } + closed = true + break + } + if !closed { + log.Printf("%s: unable to close %s after %d retries\n", debugName, closerName, maxRetries) + } + return closed +} diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index b5541a629d..5d47eefbbc 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -291,6 +291,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } + defer data.Close() if _, err := io.Copy(fileWriter, data); err != nil { return err } From 4b610d3c3f366753397cfdba03feba7a79ac2112 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 16:58:35 -0800 Subject: [PATCH 15/65] add s3 trailing slash test --- pkg/remote/connparse/connparse_test.go | 158 +++++++++++++++---------- 1 file changed, 98 insertions(+), 60 deletions(-) diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index c530c8e768..704334c47b 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -17,20 +17,20 @@ func TestParseURI_WSHWithScheme(t *testing.T) { } expected := "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "user@localhost:8080" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "user@localhost:8080/path/to/file" pathWithHost := c.GetPathWithHost() if pathWithHost != expected { - t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + t.Fatalf("expected path with host to be \"%q\", got \"%q\"", expected, pathWithHost) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } if len(c.GetSchemeParts()) != 1 { t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) @@ -44,27 +44,27 @@ func TestParseURI_WSHWithScheme(t *testing.T) { } expected = "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "user@192.168.0.1:22" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "user@192.168.0.1:22/path/to/file" pathWithHost = c.GetPathWithHost() if pathWithHost != expected { - t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + t.Fatalf("expected path with host to be \"%q\", got \"%q\"", expected, pathWithHost) } expected = "wsh" if c.GetType() != expected { - t.Fatalf("expected conn type to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected conn type to be \"%q\", got \"%q\"", expected, c.Scheme) } if len(c.GetSchemeParts()) != 1 { t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) } got := c.GetFullURI() if got != cstr { - t.Fatalf("expected full URI to be %q, got %q", cstr, got) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", cstr, got) } } @@ -79,18 +79,18 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { } expected := "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } if c.Host != "conn" { - t.Fatalf("expected host to be empty, got %q", c.Host) + t.Fatalf("expected host to be empty, got \"%q\"", c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://conn/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } // Test with a complex remote path @@ -101,19 +101,19 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { } expected = "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "user@localhost:8080" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://user@localhost:8080/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } // Test with an IP address @@ -124,19 +124,19 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { } expected = "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "user@192.168.0.1:8080" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://user@192.168.0.1:8080/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -151,19 +151,19 @@ func TestParseURI_WSHCurrentPathShorthand(t *testing.T) { } expected := "~/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "current" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://current/~/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } // Test with a absolute path @@ -174,19 +174,19 @@ func TestParseURI_WSHCurrentPathShorthand(t *testing.T) { } expected = "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "current" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://current/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -198,19 +198,19 @@ func TestParseURI_WSHCurrentPath(t *testing.T) { } expected := "./Documents/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "current" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://current/./Documents/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -222,19 +222,19 @@ func TestParseURI_WSHCurrentPathWindows(t *testing.T) { } expected := ".\\Documents\\path\\to\\file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "current" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://current/.\\Documents\\path\\to\\file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -247,14 +247,14 @@ func TestParseURI_WSHLocalShorthand(t *testing.T) { } expected := "~/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } if c.Host != "local" { - t.Fatalf("expected host to be empty, got %q", c.Host) + t.Fatalf("expected host to be empty, got \"%q\"", c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } cstr = "wsh:///~/path/to/file" @@ -264,18 +264,18 @@ func TestParseURI_WSHLocalShorthand(t *testing.T) { } expected = "~/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } if c.Host != "local" { - t.Fatalf("expected host to be empty, got %q", c.Host) + t.Fatalf("expected host to be empty, got \"%q\"", c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://local/~/path/to/file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -290,19 +290,19 @@ func TestParseURI_WSHWSL(t *testing.T) { } expected := "/path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "wsl://Ubuntu" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://wsl://Ubuntu/path/to/file" if expected != c.GetFullURI() { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } t.Log("Testing with scheme") @@ -324,19 +324,19 @@ func TestParseUri_LocalWindowsAbsPath(t *testing.T) { } expected := "C:\\path\\to\\file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "local" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://local/C:\\path\\to\\file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -355,19 +355,19 @@ func TestParseURI_LocalWindowsRelativeShorthand(t *testing.T) { } expected := "~\\path\\to\\file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "local" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "wsh" if c.Scheme != expected { - t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + t.Fatalf("expected scheme to be \"%q\", got \"%q\"", expected, c.Scheme) } expected = "wsh://local/~\\path\\to\\file" if c.GetFullURI() != expected { - t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", expected, c.GetFullURI()) } } @@ -380,22 +380,60 @@ func TestParseURI_BasicS3(t *testing.T) { } expected := "path/to/file" if c.Path != expected { - t.Fatalf("expected path to be %q, got %q", expected, c.Path) + t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } expected = "bucket" if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) } expected = "bucket/path/to/file" pathWithHost := c.GetPathWithHost() if pathWithHost != expected { - t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + t.Fatalf("expected path with host to be \"%q\", got \"%q\"", expected, pathWithHost) } expected = "s3" if c.GetType() != expected { - t.Fatalf("expected conn type to be %q, got %q", expected, c.GetType()) + t.Fatalf("expected conn type to be \"%q\", got \"%q\"", expected, c.GetType()) } if len(c.GetSchemeParts()) != 2 { t.Fatalf("expected scheme parts to be 2, got %d", len(c.GetSchemeParts())) } } + +func TestParseURI_S3BucketOnly(t *testing.T) { + t.Parallel() + + testUri := func(cstr string, pathExpected string, pathWithHostExpected string) { + c, err := connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + if c.Path != pathExpected { + t.Fatalf("expected path to be \"%q\", got \"%q\"", pathExpected, c.Path) + } + expected := "bucket" + if c.Host != expected { + t.Fatalf("expected host to be \"%q\", got \"%q\"", expected, c.Host) + } + pathWithHost := c.GetPathWithHost() + if pathWithHost != pathWithHostExpected { + t.Fatalf("expected path with host to be \"%q\", got \"%q\"", expected, pathWithHost) + } + expected = "s3" + if c.GetType() != expected { + t.Fatalf("expected conn type to be \"%q\", got \"%q\"", expected, c.GetType()) + } + if len(c.GetSchemeParts()) != 2 { + t.Fatalf("expected scheme parts to be 2, got %d", len(c.GetSchemeParts())) + } + fullUri := c.GetFullURI() + if fullUri != cstr { + t.Fatalf("expected full URI to be \"%q\", got \"%q\"", cstr, fullUri) + } + } + + t.Log("Testing with no trailing slash") + testUri("profile:s3://bucket", "", "bucket") + t.Log("Testing with trailing slash") + testUri("profile:s3://bucket/", "/", "bucket/") +} From 5b7cf867116334078afd38f1bbf14578bafcda78 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 17:01:01 -0800 Subject: [PATCH 16/65] add back quick add for pathtree --- pkg/remote/fileshare/pathtree/pathtree.go | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/remote/fileshare/pathtree/pathtree.go b/pkg/remote/fileshare/pathtree/pathtree.go index fd59af170a..cb3335f286 100644 --- a/pkg/remote/fileshare/pathtree/pathtree.go +++ b/pkg/remote/fileshare/pathtree/pathtree.go @@ -68,19 +68,19 @@ func (t *Tree) Add(path string) { components := strings.Split(relativePath, t.delimiter) log.Printf("components: %v", components) - // // Quick check to see if the parent path is already in the tree, in which case we can skip the loop - // if len(components) > 1 { - // parentPath := strings.Join(components[:len(components)-1], t.delimiter) - // log.Printf("parentPath: %s", parentPath) - // if t.nodes[parentPath] != nil { - // lastPathComponent := components[len(components)-1] - // t.nodes[parentPath].Children[lastPathComponent] = &Node{ - // Children: make(map[string]*Node), - // } - // t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] - // return - // } - // } + // Quick check to see if the parent path is already in the tree, in which case we can skip the loop + if len(components) > 1 { + parentPath := strings.Join(components[:len(components)-1], t.delimiter) + log.Printf("parentPath: %s", parentPath) + if t.nodes[parentPath] != nil { + lastPathComponent := components[len(components)-1] + t.nodes[parentPath].Children[lastPathComponent] = &Node{ + Children: make(map[string]*Node), + } + t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] + return + } + } currentNode := t.Root for i, component := range components { From 39cd7ce40e79b103702743c08e10a0792f18d8ac Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 17:02:37 -0800 Subject: [PATCH 17/65] Update pkg/util/fileutil/fileutil.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/util/fileutil/fileutil.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 1911efe24a..0bbe23cd19 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -289,8 +289,8 @@ func ReadStreamToFileData(ctx context.Context, readCh <-chan wshrpc.RespOrErrorU fileData = &wshrpc.FileData{ Info: &finfo, } - }, func(entries []*wshrpc.FileInfo) error { - entries = append(entries, entries...) + }, func(fileEntries []*wshrpc.FileInfo) error { + entries = append(entries, fileEntries...) return nil }, func(data io.Reader) error { if _, err := io.Copy(&dataBuf, data); err != nil { From 297f400facc82b9dc5b0afc00c78539ed3bc32d4 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 5 Feb 2025 18:49:09 -0800 Subject: [PATCH 18/65] fix readstream for s3 --- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index de6b2072a2..dc02a760e0 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -147,7 +147,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da break } bytesRemaining -= int64(n) - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: string(buf)}} + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: base64.StdEncoding.EncodeToString(buf[:n])}} if bytesRemaining == 0 || errors.Is(err, io.EOF) { return } From 0ae9ecd70f9f8a56952158671bf9dfc245503613 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 7 Feb 2025 10:45:09 -0800 Subject: [PATCH 19/65] bad merge --- cmd/wsh/cmd/wshcmd-file.go | 1 - pkg/util/fileutil/fileutil.go | 107 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index cc11c384ee..8aae91b91a 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "fmt" "io" - "io/fs" "log" "os" "path" diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 4c894f190c..765eede825 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -4,6 +4,10 @@ package fileutil import ( + "bytes" + "context" + "encoding/base64" + "fmt" "io" "io/fs" "mime" @@ -237,3 +241,106 @@ func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { IsDirInternal: fi.IsDir, } } + +func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { + var fileData *wshrpc.FileData + firstPk := true + isDir := false + drain := true + defer func() { + if drain { + go func() { + for range readCh { + } + }() + } + }() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %v", context.Cause(ctx)) + case respUnion, ok := <-readCh: + if !ok { + drain = false + return nil + } + if respUnion.Error != nil { + return respUnion.Error + } + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if resp.Info == nil { + return fmt.Errorf("stream file protocol error, first pk fileinfo is empty") + } + fileData = &resp + if fileData.Info.IsDir { + isDir = true + } + continue + } + if isDir { + if len(resp.Entries) == 0 { + continue + } + if resp.Data64 != "" { + return fmt.Errorf("stream file protocol error, directory entry has data") + } + if err := dirCallback(resp.Entries); err != nil { + return err + } + } else { + if resp.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + if err := fileCallback(decoder); err != nil { + return err + } + } + } + } +} + +func ReadStreamToFileData(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData]) (*wshrpc.FileData, error) { + var fileData *wshrpc.FileData + var dataBuf bytes.Buffer + var entries []*wshrpc.FileInfo + err := ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + fileData = &wshrpc.FileData{ + Info: &finfo, + } + }, func(fileEntries []*wshrpc.FileInfo) error { + entries = append(entries, fileEntries...) + return nil + }, func(data io.Reader) error { + if _, err := io.Copy(&dataBuf, data); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + if fileData == nil { + return nil, fmt.Errorf("stream file protocol error, no file info") + } + if !fileData.Info.IsDir { + fileData.Data64 = base64.StdEncoding.EncodeToString(dataBuf.Bytes()) + } else { + fileData.Entries = entries + } + return fileData, nil +} + +func ReadFileStreamToWriter(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], writer io.Writer) error { + return ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + }, func(entries []*wshrpc.FileInfo) error { + return nil + }, func(data io.Reader) error { + _, err := io.Copy(writer, data) + return err + }) +} From dc951dd805094fbbcb23d9759cec03b86c304b60 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 7 Feb 2025 11:00:37 -0800 Subject: [PATCH 20/65] revert wshremote --- pkg/wshrpc/wshremote/wshremote.go | 76 ++++--------------------------- 1 file changed, 8 insertions(+), 68 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 58c8b8c5f0..711de2e26e 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -292,7 +292,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - defer data.Close() if _, err := io.Copy(fileWriter, data); err != nil { return err } @@ -480,70 +479,11 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } numFiles++ finfo := next.FileInfo() - nextPath := filepath.Join(destPathCleaned, next.Name) - destinfo, err = os.Stat(nextPath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat file %q: %w", nextPath, err) - } - if !finfo.IsDir() { - totalBytes += finfo.Size() - } - - if destinfo != nil { - if destinfo.IsDir() { - if !finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) - } else { - err := os.Remove(nextPath) - if err != nil { - return fmt.Errorf("cannot remove file %q: %w", nextPath, err) - } - } - } else if !merge && !overwrite { - return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) - } else if overwrite { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } - } - } else { - if finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) - } else { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } - } - } else if !overwrite { - return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) - } - } - } else { - if finfo.IsDir() { - err := os.MkdirAll(nextPath, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create directory %q: %w", nextPath, err) - } - } else { - err := os.MkdirAll(filepath.Dir(nextPath), 0755) - if err != nil { - return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) - } - file, err := os.OpenFile(nextPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create new file %q: %w", nextPath, err) - } - _, err = io.Copy(file, reader) - if err != nil { - return fmt.Errorf("cannot write file %q: %w", nextPath, err) - } - file.Close() - } + n, err := copyFileFunc(filepath.Join(destPathCleaned, next.Name), finfo, reader) + if err != nil { + return fmt.Errorf("cannot copy file %q: %w", next.Name, err) } + totalBytes += n return nil }) if err != nil { @@ -762,19 +702,19 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C } destPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(destConn.Path)) destinfo, err := os.Stat(destPathCleaned) - if err == nil && destinfo != nil { + if err == nil { if !destinfo.IsDir() { if !overwrite { - return fmt.Errorf("destination %q already exists, use overwrite option", destPathCleaned) + return fmt.Errorf("destination %q already exists, use overwrite option", destUri) } else { err := os.Remove(destPathCleaned) if err != nil { - return fmt.Errorf("cannot remove file %q: %w", destPathCleaned, err) + return fmt.Errorf("cannot remove file %q: %w", destUri, err) } } } } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) + return fmt.Errorf("cannot stat destination %q: %w", destUri, err) } srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { From c22a560d2d3ecaa345af3e938d830c3af58a093f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 7 Feb 2025 11:02:52 -0800 Subject: [PATCH 21/65] put back data close --- pkg/wshrpc/wshremote/wshremote.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 711de2e26e..f25d0debbd 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -292,6 +292,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } + defer data.Close() if _, err := io.Copy(fileWriter, data); err != nil { return err } From 264ff2d6ab5a5fd32a9b7084ec5fdf9164846360 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 7 Feb 2025 11:18:21 -0800 Subject: [PATCH 22/65] fix empty file info error with readfilestream --- pkg/util/fileutil/fileutil.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 765eede825..39a754e252 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -279,6 +279,7 @@ func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[w if fileData.Info.IsDir { isDir = true } + fileInfoCallback(*fileData.Info) continue } if isDir { From db84722dc4999dc35b02fbba562521b1008ab066 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 12:06:10 -0800 Subject: [PATCH 23/65] Remove recursive from cp, make it default --- cmd/wsh/cmd/wshcmd-file.go | 13 ++++--------- frontend/app/store/wshclientapi.ts | 4 ++-- frontend/types/gotypes.d.ts | 7 ------- pkg/remote/fileshare/fileshare.go | 12 ++++++++++-- pkg/remote/fileshare/wavefs/wavefs.go | 12 ++++++++++-- pkg/remote/fileshare/wshfs/wshfs.go | 4 ++-- pkg/wshrpc/wshclient/wshclient.go | 4 ++-- pkg/wshrpc/wshremote/wshremote.go | 20 +++++++++++--------- pkg/wshrpc/wshrpctypes.go | 14 ++++---------- 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 7c1c60ded0..0011c47527 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -103,7 +103,6 @@ func init() { fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") - fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") fileCmd.AddCommand(fileCpCmd) fileMvCmd.Flags().BoolP("recursive", "r", false, "move directories recursively") @@ -174,7 +173,7 @@ var fileAppendCmd = &cobra.Command{ var fileCpCmd = &cobra.Command{ Use: "cp [source-uri] [destination-uri]" + UriHelpText, Aliases: []string{"copy"}, - Short: "copy files between storage systems", + Short: "copy files between storage systems, recursively if needed", Long: "Copy files between different storage systems." + UriHelpText, Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt\n wsh file cp wsh://user@ec2/home/user/config.txt wavefile://client/config.txt", Args: cobra.ExactArgs(2), @@ -398,10 +397,6 @@ func getTargetPath(src, dst string) (string, error) { func fileCpRun(cmd *cobra.Command, args []string) error { src, dst := args[0], args[1] - recursive, err := cmd.Flags().GetBool("recursive") - if err != nil { - return err - } merge, err := cmd.Flags().GetBool("merge") if err != nil { return err @@ -419,9 +414,9 @@ func fileCpRun(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("unable to parse dest path: %w", err) } - log.Printf("Copying %s to %s; recursive: %v, merge: %v, force: %v", srcPath, destPath, recursive, merge, force) + log.Printf("Copying %s to %s; merge: %v, force: %v", srcPath, destPath, merge, force) rpcOpts := &wshrpc.RpcOpts{Timeout: TimeoutYear} - err = wshclient.FileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Recursive: recursive, Merge: merge, Overwrite: force, Timeout: TimeoutYear}}, rpcOpts) + err = wshclient.FileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Merge: merge, Overwrite: force, Timeout: TimeoutYear}}, rpcOpts) if err != nil { return fmt.Errorf("copying file: %w", err) } @@ -449,7 +444,7 @@ func fileMvRun(cmd *cobra.Command, args []string) error { } log.Printf("Moving %s to %s; recursive: %v, force: %v", srcPath, destPath, recursive, force) rpcOpts := &wshrpc.RpcOpts{Timeout: TimeoutYear} - err = wshclient.FileMoveCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Recursive: recursive, Overwrite: force, Timeout: TimeoutYear}}, rpcOpts) + err = wshclient.FileMoveCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Overwrite: force, Timeout: TimeoutYear, Recursive: recursive}}, rpcOpts) if err != nil { return fmt.Errorf("moving file: %w", err) } diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 1c6903104e..3c83bc98d1 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -263,7 +263,7 @@ class RpcApiType { } // command "remotefilecopy" [call] - RemoteFileCopyCommand(client: WshClient, data: CommandRemoteFileCopyData, opts?: RpcOpts): Promise { + RemoteFileCopyCommand(client: WshClient, data: CommandFileCopyData, opts?: RpcOpts): Promise { return client.wshRpcCall("remotefilecopy", data, opts); } @@ -283,7 +283,7 @@ class RpcApiType { } // command "remotefilemove" [call] - RemoteFileMoveCommand(client: WshClient, data: CommandRemoteFileCopyData, opts?: RpcOpts): Promise { + RemoteFileMoveCommand(client: WshClient, data: CommandFileCopyData, opts?: RpcOpts): Promise { return client.wshRpcCall("remotefilemove", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index a690a60cec..180fb42cec 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -202,13 +202,6 @@ declare global { message: string; }; - // wshrpc.CommandRemoteFileCopyData - type CommandRemoteFileCopyData = { - srcuri: string; - desturi: string; - opts?: FileCopyOpts; - }; - // wshrpc.CommandRemoteListEntriesData type CommandRemoteListEntriesData = { path: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 9473db55a2..10ca41f97e 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -118,11 +118,19 @@ func Move(ctx context.Context, data wshrpc.CommandFileCopyData) error { return fmt.Errorf("error creating fileshare client, could not parse destination connection %s", data.DestUri) } if srcConn.Host != destConn.Host { - err := destClient.CopyRemote(ctx, srcConn, destConn, srcClient, data.Opts) + finfo, err := srcClient.Stat(ctx, srcConn) + if err != nil { + return fmt.Errorf("cannot stat %q: %w", data.SrcUri, err) + } + recursive := data.Opts != nil && data.Opts.Recursive + if finfo.IsDir && data.Opts != nil && !recursive { + return fmt.Errorf("cannot move directory %q to %q without recursive flag", data.SrcUri, data.DestUri) + } + err = destClient.CopyRemote(ctx, srcConn, destConn, srcClient, data.Opts) if err != nil { return fmt.Errorf("cannot copy %q to %q: %w", data.SrcUri, data.DestUri, err) } - return srcClient.Delete(ctx, srcConn, data.Opts.Recursive) + return srcClient.Delete(ctx, srcConn, recursive) } else { return srcClient.MoveInternal(ctx, srcConn, destConn, data.Opts) } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 63cbe36a1d..dadb2f3636 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -393,11 +393,19 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar if srcConn.Host != destConn.Host { return fmt.Errorf("move internal, src and dest hosts do not match") } - err := c.CopyInternal(ctx, srcConn, destConn, opts) + finfo, err := c.Stat(ctx, srcConn) + if err != nil { + return fmt.Errorf("error getting file info: %w", err) + } + recursive := opts != nil && opts.Recursive + if finfo.IsDir && !recursive { + return fmt.Errorf("source is a directory, use recursive flag to move") + } + err = c.CopyInternal(ctx, srcConn, destConn, opts) if err != nil { return fmt.Errorf("error copying blockfile: %w", err) } - err = c.Delete(ctx, srcConn, opts.Recursive) + err = c.Delete(ctx, srcConn, recursive) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 61816ea576..424c589d20 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -157,7 +157,7 @@ func (c WshClient) MoveInternal(ctx context.Context, srcConn, destConn *connpars if timeout == 0 { timeout = ThirtySeconds } - return wshclient.RemoteFileMoveCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) + return wshclient.RemoteFileMoveCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } func (c WshClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, _ fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { @@ -172,7 +172,7 @@ func (c WshClient) CopyInternal(ctx context.Context, srcConn, destConn *connpars if timeout == 0 { timeout = ThirtySeconds } - return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) + return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 6fdbaf7473..5d2f140097 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -321,7 +321,7 @@ func RecordTEventCommand(w *wshutil.WshRpc, data telemetrydata.TEvent, opts *wsh } // command "remotefilecopy", wshserver.RemoteFileCopyCommand -func RemoteFileCopyCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteFileCopyData, opts *wshrpc.RpcOpts) error { +func RemoteFileCopyCommand(w *wshutil.WshRpc, data wshrpc.CommandFileCopyData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotefilecopy", data, opts) return err } @@ -345,7 +345,7 @@ func RemoteFileJoinCommand(w *wshutil.WshRpc, data []string, opts *wshrpc.RpcOpt } // command "remotefilemove", wshserver.RemoteFileMoveCommand -func RemoteFileMoveCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteFileCopyData, opts *wshrpc.RpcOpts) error { +func RemoteFileMoveCommand(w *wshutil.WshRpc, data wshrpc.CommandFileCopyData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotefilemove", data, opts) return err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 711de2e26e..f7aebd145e 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -240,7 +240,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if opts == nil { opts = &wshrpc.FileCopyOpts{} } - recursive := opts.Recursive logPrintfDev("RemoteTarStreamCommand: path=%s\n", path) path, err := wavebase.ExpandHomeDir(path) if err != nil { @@ -258,11 +257,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } else { pathPrefix = filepath.Dir(cleanedPath) + "/" } - if finfo.IsDir() { - if !recursive { - return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) - } - } timeout := DefaultTimeout if opts.Timeout > 0 { @@ -314,7 +308,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return rtn } -func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.CommandRemoteFileCopyData) error { +func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.CommandFileCopyData) error { log.Printf("RemoteFileCopyCommand: src=%s, dest=%s\n", data.SrcUri, data.DestUri) opts := data.Opts if opts == nil { @@ -689,12 +683,13 @@ func (impl *ServerImpl) RemoteFileTouchCommand(ctx context.Context, path string) return nil } -func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.CommandRemoteFileCopyData) error { +func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.CommandFileCopyData) error { logPrintfDev("RemoteFileCopyCommand: src=%s, dest=%s\n", data.SrcUri, data.DestUri) opts := data.Opts destUri := data.DestUri srcUri := data.SrcUri overwrite := opts != nil && opts.Overwrite + recursive := opts != nil && opts.Recursive destConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, destUri) if err != nil { @@ -722,7 +717,14 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C } if srcConn.Host == destConn.Host { srcPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(srcConn.Path)) - err := os.Rename(srcPathCleaned, destPathCleaned) + finfo, err := os.Stat(srcPathCleaned) + if err != nil { + return fmt.Errorf("cannot stat file %q: %w", srcPathCleaned, err) + } + if finfo.IsDir() && !recursive { + return fmt.Errorf("cannot move directory %q, recursive option not specified", srcUri) + } + err = os.Rename(srcPathCleaned, destPathCleaned) if err != nil { return fmt.Errorf("cannot move file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 2a74a9c63e..0cf70ae3f0 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -206,11 +206,11 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[FileData] RemoteTarStreamCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[iochantypes.Packet] - RemoteFileCopyCommand(ctx context.Context, data CommandRemoteFileCopyData) error + RemoteFileCopyCommand(ctx context.Context, data CommandFileCopyData) error RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error - RemoteFileMoveCommand(ctx context.Context, data CommandRemoteFileCopyData) error + RemoteFileMoveCommand(ctx context.Context, data CommandFileCopyData) error RemoteFileDeleteCommand(ctx context.Context, data CommandDeleteFileData) error RemoteWriteFileCommand(ctx context.Context, data FileData) error RemoteFileJoinCommand(ctx context.Context, paths []string) (*FileInfo, error) @@ -515,12 +515,6 @@ type CommandFileCopyData struct { Opts *FileCopyOpts `json:"opts,omitempty"` } -type CommandRemoteFileCopyData struct { - SrcUri string `json:"srcuri"` - DestUri string `json:"desturi"` - Opts *FileCopyOpts `json:"opts,omitempty"` -} - type CommandRemoteStreamTarData struct { Path string `json:"path"` Opts *FileCopyOpts `json:"opts,omitempty"` @@ -528,8 +522,8 @@ type CommandRemoteStreamTarData struct { type FileCopyOpts struct { Overwrite bool `json:"overwrite,omitempty"` - Recursive bool `json:"recursive,omitempty"` - Merge bool `json:"merge,omitempty"` + Recursive bool `json:"recursive,omitempty"` // only used for move, always true for copy + Merge bool `json:"merge,omitempty"` // only used for copy, always false for move Timeout int64 `json:"timeout,omitempty"` } From 5da7740266abbdae77c59950226b27a8802a5c96 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 19:19:40 -0800 Subject: [PATCH 24/65] save --- pkg/remote/fileshare/fstype/fstype.go | 5 ++ pkg/remote/fileshare/wavefs/wavefs.go | 60 ++++++++++++------------ pkg/util/tarcopy/tarcopy.go | 20 ++++++-- pkg/wshrpc/wshremote/wshremote.go | 66 +++++++++++++++------------ 4 files changed, 87 insertions(+), 64 deletions(-) diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 3c3d6fceb3..2e44e6b003 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -5,12 +5,17 @@ package fstype import ( "context" + "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) +const ( + DefaultTimeout = 30 * time.Second +) + type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index dadb2f3636..2cb58a8f0d 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -29,10 +29,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - DefaultTimeout = 30 * time.Second -) - type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -54,7 +50,7 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, if !rtnData.Info.IsDir { for i := 0; i < dataLen; i += wshrpc.FileChunkSize { if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + ch <- wshutil.RespErr[wshrpc.FileData](context.Cause(ctx)) return } dataEnd := min(i+wshrpc.FileChunkSize, dataLen) @@ -63,7 +59,7 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, } else { for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + ch <- wshutil.RespErr[wshrpc.FileData](context.Cause(ctx)) return } ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} @@ -116,7 +112,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio pathPrefix := getPathPrefix(conn) schemeAndHost := conn.GetSchemeAndHost() + "/" - timeout := DefaultTimeout + timeout := fstype.DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } @@ -130,12 +126,12 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio }() for _, file := range list { if readerCtx.Err() != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](readerCtx.Err()) + rtn <- wshutil.RespErr[iochantypes.Packet](context.Cause(readerCtx)) return } file.Mode = 0644 - if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path); err != nil { + if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path, file.Path == conn.Path); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error writing tar header: %w", err)) return } @@ -393,19 +389,11 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar if srcConn.Host != destConn.Host { return fmt.Errorf("move internal, src and dest hosts do not match") } - finfo, err := c.Stat(ctx, srcConn) - if err != nil { - return fmt.Errorf("error getting file info: %w", err) - } - recursive := opts != nil && opts.Recursive - if finfo.IsDir && !recursive { - return fmt.Errorf("source is a directory, use recursive flag to move") - } - err = c.CopyInternal(ctx, srcConn, destConn, opts) + err := c.CopyInternal(ctx, srcConn, destConn, opts) if err != nil { return fmt.Errorf("error copying blockfile: %w", err) } - err = c.Delete(ctx, srcConn, recursive) + err = c.Delete(ctx, srcConn, opts.Recursive) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) } @@ -455,31 +443,41 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } + overwrite := opts != nil && opts.Overwrite + merge := opts != nil && opts.Merge + destHasSlash := strings.HasSuffix(destConn.Path, "/") destPrefix := getPathPrefix(destConn) destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) + entries, err := c.ListEntries(ctx, srcConn, nil) + if err != nil { + return fmt.Errorf("error listing blockfiles: %w", err) + } + if len(entries) > 1 && !merge { + return fmt.Errorf("more than one entry at destination prefix, use merge flag to copy") + } readCtx, cancel := context.WithCancelCause(ctx) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { - if next.Typeflag == tar.TypeDir { + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error { + if next.IsDir() { return nil } - fileName, err := cleanPath(path.Join(destPrefix, next.Name)) + fileName, err := cleanPath(path.Join(destPrefix, next.Name())) + if singleFile && !destHasSlash { + fileName, err = cleanPath(destConn.Path) + } if err != nil { return fmt.Errorf("error cleaning path: %w", err) } - _, err = filestore.WFS.Stat(ctx, zoneId, fileName) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("error getting blockfile info: %w", err) - } - err := filestore.WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOpts{}) - if err != nil { - return fmt.Errorf("error making blockfile: %w", err) + if !overwrite { + for _, entry := range entries { + if entry.Name == fileName { + return fmt.Errorf("destination already exists: %v", fileName) + } } } log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) - dataBuf := make([]byte, next.Size) + dataBuf := make([]byte, next.Size()) _, err = reader.Read(dataBuf) if err != nil { if !errors.Is(err, io.EOF) { diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 06e008811c..355adb061a 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -29,26 +29,33 @@ const ( pipeReaderName = "pipe reader" pipeWriterName = "pipe writer" tarWriterName = "tar writer" + + // custom flag to indicate that the source is a single file, not a directory the contents of a directory + SingleFile = "singlefile" ) // TarCopySrc creates a tar stream writer and returns a channel to send the tar stream to. // writeHeader is a function that writes the tar header for the file. // writer is the tar writer to write the file data to. // close is a function that closes the tar writer and internal pipe writer. -func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { +func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string, singleFile bool) error, writer io.Writer, close func()) { pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) - return rtnChan, func(fi fs.FileInfo, file string) error { + return rtnChan, func(fi fs.FileInfo, file string, singleFile bool) error { // generate tar header header, err := tar.FileInfoHeader(fi, file) if err != nil { return err } + if singleFile { + header.PAXRecords[SingleFile] = "true" + } + header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) if err := validatePath(header.Name); err != nil { return err @@ -78,7 +85,7 @@ func validatePath(path string) error { // TarCopyDest reads a tar stream from a channel and writes the files to the destination. // readNext is a function that is called for each file in the tar stream to read the file data. It should return an error if the file cannot be read. // The function returns an error if the tar stream cannot be read. -func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { +func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { gracefulClose(pipeWriter, tarCopyDestName, pipeWriterName) @@ -110,7 +117,12 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan return err } } - err = readNext(next, tarReader) + + // Check for directory traversal + if strings.Contains(next.Name, "..") { + return nil + } + err = readNext(next.FileInfo(), tarReader, next.PAXRecords[SingleFile] == "true") if err != nil { return err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index f7aebd145e..a0bf9b188c 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -252,7 +252,8 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } var pathPrefix string - if finfo.IsDir() && strings.HasSuffix(cleanedPath, "/") { + singleFile := !finfo.IsDir() + if !singleFile && strings.HasSuffix(cleanedPath, "/") { pathPrefix = cleanedPath } else { pathPrefix = filepath.Dir(cleanedPath) + "/" @@ -277,7 +278,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - if err = writeHeader(info, path); err != nil { + if err = writeHeader(info, path, singleFile); err != nil { return err } // if not a dir, write file content @@ -294,10 +295,10 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } log.Printf("RemoteTarStreamCommand: starting\n") err = nil - if finfo.IsDir() { - err = filepath.Walk(path, walkFunc) + if singleFile { + err = walkFunc(cleanedPath, finfo, nil) } else { - err = walkFunc(path, finfo, nil) + err = filepath.Walk(cleanedPath, walkFunc) } if err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](err) @@ -325,19 +326,25 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } destPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(destConn.Path)) destinfo, err := os.Stat(destPathCleaned) - if err == nil { - if !destinfo.IsDir() { - if !overwrite { - return fmt.Errorf("destination %q already exists, use overwrite option", destPathCleaned) - } else { - err := os.Remove(destPathCleaned) - if err != nil { - return fmt.Errorf("cannot remove file %q: %w", destPathCleaned, err) - } + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) + } + } + + destExists := destinfo != nil + destIsDir := destExists && destinfo.IsDir() + destHasSlash := strings.HasSuffix(destUri, "/") + + if destExists && !destIsDir { + if !overwrite { + return fmt.Errorf("file already exists at destination %q, use overwrite option", destPathCleaned) + } else { + err := os.Remove(destPathCleaned) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", destPathCleaned, err) } } - } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) } srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { @@ -345,13 +352,13 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } copyFileFunc := func(path string, finfo fs.FileInfo, srcFile io.Reader) (int64, error) { - destinfo, err = os.Stat(path) + nextinfo, err := os.Stat(path) if err != nil && !errors.Is(err, fs.ErrNotExist) { return 0, fmt.Errorf("cannot stat file %q: %w", path, err) } - if destinfo != nil { - if destinfo.IsDir() { + if nextinfo != nil { + if nextinfo.IsDir() { if !finfo.IsDir() { // try to create file in directory path = filepath.Join(path, filepath.Base(finfo.Name())) @@ -464,18 +471,19 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C numFiles := 0 numSkipped := 0 totalBytes := int64(0) - err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { - // Check for directory traversal - if strings.Contains(next.Name, "..") { - log.Printf("skipping file with unsafe path: %q\n", next.Name) - numSkipped++ - return nil - } + + err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error { numFiles++ - finfo := next.FileInfo() - n, err := copyFileFunc(filepath.Join(destPathCleaned, next.Name), finfo, reader) + nextpath := filepath.Join(destPathCleaned, next.Name()) + if singleFile { + // custom flag to indicate that the source is a single file, not a directory the contents of a directory + if !destHasSlash { + nextpath = destPathCleaned + } + } + n, err := copyFileFunc(nextpath, next, reader) if err != nil { - return fmt.Errorf("cannot copy file %q: %w", next.Name, err) + return fmt.Errorf("cannot copy file %q: %w", next.Name(), err) } totalBytes += n return nil From abb8e0d955ba6bd1e39074105ee5f3011138ffc6 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 21:50:53 -0800 Subject: [PATCH 25/65] save --- pkg/remote/fileshare/wavefs/wavefs.go | 8 ++--- pkg/util/fileutil/fileutil.go | 4 +++ pkg/util/tarcopy/tarcopy.go | 8 +++-- pkg/wshrpc/wshremote/wshremote.go | 49 +++++++++++++++++++-------- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 2cb58a8f0d..181e5699c4 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -458,11 +458,11 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse } readCtx, cancel := context.WithCancelCause(ctx) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error { - if next.IsDir() { + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { + if next.Typeflag == tar.TypeDir { return nil } - fileName, err := cleanPath(path.Join(destPrefix, next.Name())) + fileName, err := cleanPath(path.Join(destPrefix, next.Name)) if singleFile && !destHasSlash { fileName, err = cleanPath(destConn.Path) } @@ -477,7 +477,7 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse } } log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) - dataBuf := make([]byte, next.Size()) + dataBuf := make([]byte, next.Size) _, err = reader.Read(dataBuf) if err != nil { if !errors.Is(err, io.EOF) { diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 4c894f190c..d7c8940db4 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -19,6 +19,7 @@ import ( ) func FixPath(path string) (string, error) { + origPath := path var err error if strings.HasPrefix(path, "~") { path = filepath.Join(wavebase.GetHomeDir(), path[1:]) @@ -28,6 +29,9 @@ func FixPath(path string) (string, error) { return "", err } } + if strings.HasSuffix(origPath, "/") && !strings.HasSuffix(path, "/") { + path += "/" + } return path, nil } diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 355adb061a..6d443bf81f 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -53,7 +53,7 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } if singleFile { - header.PAXRecords[SingleFile] = "true" + header.PAXRecords = map[string]string{SingleFile: "true"} } header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) @@ -61,6 +61,8 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. return err } + log.Printf("TarCopySrc: header name: %v\n", header.Name) + // write header if err := tarWriter.WriteHeader(header); err != nil { return err @@ -85,7 +87,7 @@ func validatePath(path string) error { // TarCopyDest reads a tar stream from a channel and writes the files to the destination. // readNext is a function that is called for each file in the tar stream to read the file data. It should return an error if the file cannot be read. // The function returns an error if the tar stream cannot be read. -func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error) error { +func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader, singleFile bool) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { gracefulClose(pipeWriter, tarCopyDestName, pipeWriterName) @@ -122,7 +124,7 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan if strings.Contains(next.Name, "..") { return nil } - err = readNext(next.FileInfo(), tarReader, next.PAXRecords[SingleFile] == "true") + err = readNext(next, tarReader, next.PAXRecords != nil && next.PAXRecords[SingleFile] == "true") if err != nil { return err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index a0bf9b188c..b64ee2aca8 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -18,6 +18,7 @@ import ( "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/suggestion" "github.com/wavetermdev/waveterm/pkg/util/fileutil" @@ -30,10 +31,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - DefaultTimeout = 30 * time.Second -) - type ServerImpl struct { LogWriter io.Writer } @@ -240,7 +237,8 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if opts == nil { opts = &wshrpc.FileCopyOpts{} } - logPrintfDev("RemoteTarStreamCommand: path=%s\n", path) + log.Printf("RemoteTarStreamCommand: path=%s\n", path) + srcHasSlash := strings.HasSuffix(path, "/") path, err := wavebase.ExpandHomeDir(path) if err != nil { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot expand path %q: %w", path, err)) @@ -253,13 +251,14 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. var pathPrefix string singleFile := !finfo.IsDir() - if !singleFile && strings.HasSuffix(cleanedPath, "/") { + if !singleFile && srcHasSlash { pathPrefix = cleanedPath } else { pathPrefix = filepath.Dir(cleanedPath) + "/" } + log.Printf("RemoteTarStreamCommand: path=%s, pathPrefix=%s\n", path, pathPrefix) - timeout := DefaultTimeout + timeout := fstype.DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } @@ -278,6 +277,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } + log.Printf("RemoteTarStreamCommand: path=%s\n", path) if err = writeHeader(info, path, singleFile); err != nil { return err } @@ -359,7 +359,9 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if nextinfo != nil { if nextinfo.IsDir() { + log.Printf("RemoteFileCopyCommand: nextinfo is dir, path=%s\n", path) if !finfo.IsDir() { + log.Printf("RemoteFileCopyCommand: finfo is file: %s\n", path) // try to create file in directory path = filepath.Join(path, filepath.Base(finfo.Name())) newdestinfo, err := os.Stat(path) @@ -391,13 +393,17 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return 0, fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", path) } } + } else { + log.Printf("RemoteFileCopyCommand: nextinfo is nil, path=%s\n", path) } if finfo.IsDir() { + log.Printf("RemoteFileCopyCommand: making dirs %s\n", path) err := os.MkdirAll(path, finfo.Mode()) if err != nil { return 0, fmt.Errorf("cannot create directory %q: %w", path, err) } + return 0, nil } else { err := os.MkdirAll(filepath.Dir(path), 0755) if err != nil { @@ -427,12 +433,19 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } if srcFileStat.IsDir() { + log.Print("RemoteFileCopyCommand: copying directory\n") + srcPathPrefix := filepath.Dir(srcPathCleaned) + if strings.HasSuffix(srcUri, "/") { + log.Printf("RemoteFileCopyCommand: src has slash, using %q as src path\n", srcPathCleaned) + srcPathPrefix = srcPathCleaned + } err = filepath.Walk(srcPathCleaned, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } srcFilePath := path - destFilePath := filepath.Join(destPathCleaned, strings.TrimPrefix(path, srcPathCleaned)) + destFilePath := filepath.Join(destPathCleaned, strings.TrimPrefix(path, srcPathPrefix)) + log.Printf("RemoteFileCopyCommand: copying %q to %q\n", srcFilePath, destFilePath) var file *os.File if !info.IsDir() { file, err = os.Open(srcFilePath) @@ -448,18 +461,24 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } } else { + log.Print("RemoteFileCopyCommand: copying single file\n") file, err := os.Open(srcPathCleaned) if err != nil { return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) } defer file.Close() - _, err = copyFileFunc(destPathCleaned, srcFileStat, file) + destFilePath := filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) + if destHasSlash { + log.Printf("RemoteFileCopyCommand: dest has slash, using %q as dest path\n", destPathCleaned) + destFilePath = destPathCleaned + } + _, err = copyFileFunc(destFilePath, srcFileStat, file) if err != nil { return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } } } else { - timeout := DefaultTimeout + timeout := fstype.DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } @@ -472,18 +491,20 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C numSkipped := 0 totalBytes := int64(0) - err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next fs.FileInfo, reader *tar.Reader, singleFile bool) error { + err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { numFiles++ - nextpath := filepath.Join(destPathCleaned, next.Name()) + nextpath := filepath.Join(destPathCleaned, next.Name) + log.Printf("RemoteFileCopyCommand: copying %q to %q\n", next.Name, nextpath) if singleFile { // custom flag to indicate that the source is a single file, not a directory the contents of a directory if !destHasSlash { nextpath = destPathCleaned } } - n, err := copyFileFunc(nextpath, next, reader) + finfo := next.FileInfo() + n, err := copyFileFunc(nextpath, finfo, reader) if err != nil { - return fmt.Errorf("cannot copy file %q: %w", next.Name(), err) + return fmt.Errorf("cannot copy file %q: %w", next.Name, err) } totalBytes += n return nil From e57f869a9e77ef01e9de546c1e436cbb7df508fb Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 22:29:12 -0800 Subject: [PATCH 26/65] save --- pkg/util/tarcopy/tarcopy.go | 9 ++++++--- pkg/wshutil/wshproxy.go | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 6d443bf81f..e17a1ac02b 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -45,9 +45,9 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) - return rtnChan, func(fi fs.FileInfo, file string, singleFile bool) error { + return rtnChan, func(fi fs.FileInfo, path string, singleFile bool) error { // generate tar header - header, err := tar.FileInfoHeader(fi, file) + header, err := tar.FileInfoHeader(fi, path) if err != nil { return err } @@ -56,7 +56,10 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. header.PAXRecords = map[string]string{SingleFile: "true"} } - header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) + header.Name = filepath.Clean(strings.TrimPrefix(path, pathPrefix)) + if header.Name == "." { + return nil + } if err := validatePath(header.Name); err != nil { return err } diff --git a/pkg/wshutil/wshproxy.go b/pkg/wshutil/wshproxy.go index 0bc5ae088d..6ad0a96193 100644 --- a/pkg/wshutil/wshproxy.go +++ b/pkg/wshutil/wshproxy.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/google/uuid" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/shellutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -248,6 +249,9 @@ func (p *WshRpcProxy) HandleAuthentication() (*wshrpc.RpcContext, error) { } func (p *WshRpcProxy) SendRpcMessage(msg []byte) { + defer func() { + panichandler.PanicHandler("WshRpcProxy.SendRpcMessage", recover()) + }() p.ToRemoteCh <- msg } From 20badde9308e5868d021e1fc668c95cb1532b204 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 22:49:42 -0800 Subject: [PATCH 27/65] fix preceding slashes in tar headers --- cmd/wsh/cmd/wshcmd-view.go | 1 + pkg/util/tarcopy/tarcopy.go | 23 ++++++++++++----------- pkg/wshrpc/wshremote/wshremote.go | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-view.go b/cmd/wsh/cmd/wshcmd-view.go index 97ee8ffdb6..a2f8f86394 100644 --- a/cmd/wsh/cmd/wshcmd-view.go +++ b/cmd/wsh/cmd/wshcmd-view.go @@ -19,6 +19,7 @@ var viewMagnified bool var viewCmd = &cobra.Command{ Use: "view {file|directory|URL}", + Aliases: []string{"preview", "open"}, Short: "preview/edit a file or directory", RunE: viewRun, PreRunE: preRunSetupRpcClient, diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index e17a1ac02b..0e3b3c2aa4 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -56,14 +56,17 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. header.PAXRecords = map[string]string{SingleFile: "true"} } - header.Name = filepath.Clean(strings.TrimPrefix(path, pathPrefix)) - if header.Name == "." { - return nil - } - if err := validatePath(header.Name); err != nil { + path, err = fixPath(path, pathPrefix) + if err != nil { return err } + // skip if path is empty, which means the file is the root directory + if path == "" { + return nil + } + header.Name = path + log.Printf("TarCopySrc: header name: %v\n", header.Name) // write header @@ -77,14 +80,12 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } } -func validatePath(path string) error { +func fixPath(path string, prefix string) (string, error) { + path = strings.TrimPrefix(strings.TrimPrefix(filepath.Clean(strings.TrimPrefix(path, prefix)), "/"), "\\") if strings.Contains(path, "..") { - return fmt.Errorf("invalid tar path containing directory traversal: %s", path) - } - if strings.HasPrefix(path, "/") { - return fmt.Errorf("invalid tar path starting with /: %s", path) + return "", fmt.Errorf("invalid tar path containing directory traversal: %s", path) } - return nil + return path, nil } // TarCopyDest reads a tar stream from a channel and writes the files to the destination. diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index b64ee2aca8..35698ca7e1 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -254,7 +254,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if !singleFile && srcHasSlash { pathPrefix = cleanedPath } else { - pathPrefix = filepath.Dir(cleanedPath) + "/" + pathPrefix = filepath.Dir(cleanedPath) } log.Printf("RemoteTarStreamCommand: path=%s, pathPrefix=%s\n", path, pathPrefix) From ad56f14d54d2eebd11ac8f81eddb02df7a7edfb8 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 22:52:35 -0800 Subject: [PATCH 28/65] handle case where singleFile flag is set twice --- pkg/util/tarcopy/tarcopy.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 0e3b3c2aa4..ca19cb9138 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -35,7 +35,7 @@ const ( ) // TarCopySrc creates a tar stream writer and returns a channel to send the tar stream to. -// writeHeader is a function that writes the tar header for the file. +// writeHeader is a function that writes the tar header for the file. If only a single file is being written, the singleFile flag should be set to true. // writer is the tar writer to write the file data to. // close is a function that closes the tar writer and internal pipe writer. func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string, singleFile bool) error, writer io.Writer, close func()) { @@ -45,6 +45,8 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) + singleFileFlagSet := false + return rtnChan, func(fi fs.FileInfo, path string, singleFile bool) error { // generate tar header header, err := tar.FileInfoHeader(fi, path) @@ -53,7 +55,12 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } if singleFile { + if singleFileFlagSet { + return errors.New("attempting to write multiple files to a single file tar stream") + } + header.PAXRecords = map[string]string{SingleFile: "true"} + singleFileFlagSet = true } path, err = fixPath(path, pathPrefix) @@ -89,7 +96,7 @@ func fixPath(path string, prefix string) (string, error) { } // TarCopyDest reads a tar stream from a channel and writes the files to the destination. -// readNext is a function that is called for each file in the tar stream to read the file data. It should return an error if the file cannot be read. +// readNext is a function that is called for each file in the tar stream to read the file data. If only a single file is being written from the tar src, the singleFile flag will be set in this callback. It should return an error if the file cannot be read. // The function returns an error if the tar stream cannot be read. func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader, singleFile bool) error) error { pipeReader, pipeWriter := io.Pipe() From b0ce1c2d18104a31570b51f07a3fd23127aeced6 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 22:53:04 -0800 Subject: [PATCH 29/65] update comment --- pkg/util/tarcopy/tarcopy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index ca19cb9138..aaa480d0b3 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -30,7 +30,7 @@ const ( pipeWriterName = "pipe writer" tarWriterName = "tar writer" - // custom flag to indicate that the source is a single file, not a directory the contents of a directory + // custom flag to indicate that the source is a single file SingleFile = "singlefile" ) From b6a67d85cd6e5a84d47c617bdf62f3f7ad7867e4 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sat, 8 Feb 2025 23:11:56 -0800 Subject: [PATCH 30/65] save --- pkg/remote/fileshare/s3fs/s3fs.go | 36 ++++++++---------- pkg/remote/fileshare/wavefs/wavefs.go | 54 ++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index dc02a760e0..dd32857420 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -336,7 +336,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, ModTime: modTime, Mode: mode, } - if err := writeHeader(fileutil.ToFsFileInfo(finfo), path); err != nil { + if err := writeHeader(fileutil.ToFsFileInfo(finfo), path, singleFile); err != nil { return err } if isFile { @@ -616,10 +616,12 @@ func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse } func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { - destBucket := destConn.Host overwrite := opts != nil && opts.Overwrite merge := opts != nil && opts.Merge + destHasSlash := strings.HasSuffix(destConn.Path, "/") + destPrefix := getPathPrefix(destConn) + destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") if destBucket == "" || destBucket == "/" { return fmt.Errorf("destination bucket must be specified") } @@ -632,39 +634,30 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C if err != nil { return err } - if len(entries) > 0 { - if overwrite { - err := c.Delete(ctx, destConn, true) - if err != nil { - return err - } - } else if !merge { - return fmt.Errorf("more than one entry exists at prefix, neither force nor merge specified") - } + if len(entries) > 0 && !merge { + return fmt.Errorf("more than one entry exists at prefix, merge not specified") } } else { return err } } else if !overwrite { - return fmt.Errorf("destination already exists, use force to overwrite: %v", destConn.GetFullURI()) - } - - destPrefix := destConn.Path - // Make sure destPrefix has a trailing slash if the destination is a "directory" - if destPrefix != "" && entries != nil && !strings.HasSuffix(destPrefix, "/") { - destPrefix = destPrefix + "/" + return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) } readCtx, cancel := context.WithCancelCause(ctx) defer cancel(nil) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { - log.Printf("copying %v", next.Name) + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { if next.Typeflag == tar.TypeDir { return nil } fileName, err := cleanPath(path.Join(destPrefix, next.Name)) - log.Printf("cleaned path: %v", fileName) + if singleFile && !destHasSlash { + fileName, err = cleanPath(destConn.Path) + } + if err != nil { + return fmt.Errorf("error cleaning path: %w", err) + } if !overwrite { for _, entry := range entries { if entry.Name == fileName { @@ -672,6 +665,7 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } } } + log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(destBucket), Key: aws.String(fileName), diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index c403efa57b..e54d88b94f 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -13,6 +13,7 @@ import ( "io/fs" "log" "path" + "path/filepath" "strings" "time" @@ -104,14 +105,37 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { log.Printf("ReadTarStream: conn: %v, opts: %v\n", conn, opts) - list, err := c.ListEntries(ctx, conn, nil) + path := conn.Path + srcHasSlash := strings.HasSuffix(path, "/") + cleanedPath, err := cleanPath(path) if err != nil { - return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error listing blockfiles: %w", err)) + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error cleaning path: %w", err)) } + finfo, err := c.Stat(ctx, conn) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error getting file info: %w", err)) + } + singleFile := finfo != nil && !finfo.IsDir pathPrefix := getPathPrefix(conn) + if !singleFile && srcHasSlash { + pathPrefix = cleanedPath + } else { + pathPrefix = filepath.Dir(cleanedPath) + } + schemeAndHost := conn.GetSchemeAndHost() + "/" + var entries []*wshrpc.FileInfo + if singleFile { + entries = []*wshrpc.FileInfo{finfo} + } else { + entries, err = c.ListEntries(ctx, conn, nil) + if err != nil { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error listing blockfiles: %w", err)) + } + } + timeout := fstype.DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond @@ -124,14 +148,14 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio tarClose() cancel() }() - for _, file := range list { + for _, file := range entries { if readerCtx.Err() != nil { rtn <- wshutil.RespErr[iochantypes.Packet](context.Cause(readerCtx)) return } file.Mode = 0644 - if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path, file.Path == conn.Path); err != nil { + if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path, singleFile); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error writing tar header: %w", err)) return } @@ -449,13 +473,25 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse destPrefix := getPathPrefix(destConn) destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) - entries, err := c.ListEntries(ctx, srcConn, nil) + + var entries []*wshrpc.FileInfo + _, err := c.Stat(ctx, destConn) if err != nil { - return fmt.Errorf("error listing blockfiles: %w", err) - } - if len(entries) > 1 && !merge { - return fmt.Errorf("more than one entry at destination prefix, use merge flag to copy") + if errors.Is(err, fs.ErrNotExist) { + entries, err = c.ListEntries(ctx, destConn, nil) + if err != nil { + return err + } + if len(entries) > 0 && !merge { + return fmt.Errorf("more than one entry exists at prefix, merge not specified") + } + } else { + return err + } + } else if !overwrite { + return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) } + readCtx, cancel := context.WithCancelCause(ctx) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { From f56e189034382d2269eb4c210569510b7136537d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 12:45:12 -0800 Subject: [PATCH 31/65] remove s3 cp recursive flag, unnecessary context cancel --- pkg/remote/fileshare/s3fs/s3fs.go | 5 ----- pkg/util/tarcopy/tarcopy.go | 1 - 2 files changed, 6 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index dd32857420..e43a994ff8 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -159,8 +159,6 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { - recursive := opts != nil && opts.Recursive - bucket := conn.Host if bucket == "" || bucket == "/" { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("bucket must be specified")) @@ -195,9 +193,6 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, // whether the operation is on a single file singleFile := singleFileResult != nil - if !singleFile && !recursive { - return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("recursive must be set to true for non-single file operations")) - } // whether to include the directory itself in the tar includeDir := (wholeBucket && conn.Path == "") || (singleFileResult == nil && conn.Path != "" && !strings.HasSuffix(conn.Path, "/")) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 44dc5136f8..25e7679b62 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -105,7 +105,6 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { utilfn.GracefulClose(pipeWriter, tarCopyDestName, pipeWriterName, maxRetries, retryDelay) - cancel(nil) }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { From 3c4aa68877dde7a2b46e115523c9253e76339a36 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 12:52:14 -0800 Subject: [PATCH 32/65] more gracefulclose --- pkg/remote/fileshare/s3fs/s3fs.go | 8 ++++---- pkg/util/tarcopy/tarcopy.go | 13 +++++-------- pkg/util/utilfn/utilfn.go | 7 ++++++- pkg/wshrpc/wshremote/wshremote.go | 16 ++++++++-------- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index e43a994ff8..07f8965e94 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -30,6 +30,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/tarcopy" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -126,7 +127,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da log.Printf("no data to read") return } - defer result.Body.Close() + defer utilfn.GracefulClose(result.Body, "s3fs", conn.GetFullURI()) bytesRemaining := size for { log.Printf("bytes remaining: %d", bytesRemaining) @@ -172,7 +173,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, defer func() { // in case we error out before the object gets copied, make sure to close it if singleFileResult != nil { - singleFileResult.Body.Close() + utilfn.GracefulClose(singleFileResult.Body, "s3fs", conn.Path) } }() var err error @@ -231,8 +232,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, // close the objects when we're done defer func() { for key, obj := range objMap { - log.Printf("closing object %v", key) - obj.Body.Close() + utilfn.GracefulClose(obj.Body, "s3fs", key) } }() diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 25e7679b62..8b014461f0 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -14,7 +14,6 @@ import ( "log" "path/filepath" "strings" - "time" "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" @@ -23,8 +22,6 @@ import ( ) const ( - maxRetries = 5 - retryDelay = 10 * time.Millisecond tarCopySrcName = "TarCopySrc" tarCopyDestName = "TarCopyDest" pipeReaderName = "pipe reader" @@ -44,7 +41,7 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { log.Printf("Closing pipe reader\n") - utilfn.GracefulClose(pipeReader, tarCopySrcName, pipeReaderName, maxRetries, retryDelay) + utilfn.GracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) singleFileFlagSet := false @@ -85,8 +82,8 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. return nil }, tarWriter, func() { log.Printf("Closing tar writer\n") - utilfn.GracefulClose(tarWriter, tarCopySrcName, tarWriterName, maxRetries, retryDelay) - utilfn.GracefulClose(pipeWriter, tarCopySrcName, pipeWriterName, maxRetries, retryDelay) + utilfn.GracefulClose(tarWriter, tarCopySrcName, tarWriterName) + utilfn.GracefulClose(pipeWriter, tarCopySrcName, pipeWriterName) } } @@ -104,11 +101,11 @@ func fixPath(path string, prefix string) (string, error) { func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader, singleFile bool) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { - utilfn.GracefulClose(pipeWriter, tarCopyDestName, pipeWriterName, maxRetries, retryDelay) + utilfn.GracefulClose(pipeWriter, tarCopyDestName, pipeWriterName) }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - if !utilfn.GracefulClose(pipeReader, tarCopyDestName, pipeReaderName, maxRetries, retryDelay) { + if !utilfn.GracefulClose(pipeReader, tarCopyDestName, pipeReaderName) { // If the pipe reader cannot be closed, cancel the context. This should kill the writer goroutine. cancel(nil) } diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index 7ac7e60cad..14d6204a52 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -1025,7 +1025,12 @@ func QuickHashString(s string) string { return base64.RawURLEncoding.EncodeToString(h.Sum(nil)) } -func GracefulClose(closer io.Closer, debugName string, closerName string, maxRetries int, retryDelay time.Duration) bool { +const ( + maxRetries = 5 + retryDelay = 10 * time.Millisecond +) + +func GracefulClose(closer io.Closer, debugName string, closerName string) bool { closed := false for retries := 0; retries < maxRetries; retries++ { if err := closer.Close(); err != nil { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 03e51b6348..112cea2e25 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -143,7 +143,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string if err != nil { return fmt.Errorf("cannot open file %q: %w", path, err) } - defer fd.Close() + defer utilfn.GracefulClose(fd, "remoteStreamFileRegular", path) var filePos int64 if !byteRange.All && byteRange.Start > 0 { _, err := fd.Seek(byteRange.Start, io.SeekStart) @@ -287,7 +287,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - defer data.Close() + defer utilfn.GracefulClose(data, "RemoteTarStreamCommand", path) if _, err := io.Copy(fileWriter, data); err != nil { return err } @@ -416,7 +416,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return 0, fmt.Errorf("cannot create new file %q: %w", path, err) } - defer file.Close() + defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", path) _, err = io.Copy(file, srcFile) if err != nil { return 0, fmt.Errorf("cannot write file %q: %w", path, err) @@ -453,7 +453,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot open file %q: %w", srcFilePath, err) } - defer file.Close() + defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcFilePath) } _, err = copyFileFunc(destFilePath, info, file) return err @@ -467,7 +467,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) } - defer file.Close() + defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcPathCleaned) destFilePath := filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) if destHasSlash { log.Printf("RemoteFileCopyCommand: dest has slash, using %q as dest path\n", destPathCleaned) @@ -626,7 +626,7 @@ func checkIsReadOnly(path string, fileInfo fs.FileInfo, exists bool) bool { if err != nil { return true } - fd.Close() + utilfn.GracefulClose(fd, "checkIsReadOnly", tmpFileName) os.Remove(tmpFileName) return false } @@ -635,7 +635,7 @@ func checkIsReadOnly(path string, fileInfo fs.FileInfo, exists bool) bool { if err != nil { return true } - file.Close() + utilfn.GracefulClose(file, "checkIsReadOnly", path) return false } @@ -831,7 +831,7 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD if err != nil { return fmt.Errorf("cannot open file %q: %w", path, err) } - defer file.Close() + defer utilfn.GracefulClose(file, "RemoteWriteFileCommand", path) if atOffset > 0 && !append { n, err = file.WriteAt(dataBytes[:n], atOffset) } else { From 4aa64c458d414b246bdacc75f44cf05e726798d9 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 13:44:38 -0800 Subject: [PATCH 33/65] add traversal error --- pkg/util/tarcopy/tarcopy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 8b014461f0..dcdbbe082b 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -133,7 +133,7 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan // Check for directory traversal if strings.Contains(next.Name, "..") { - return nil + return fmt.Errorf("invalid tar path containing directory traversal: %s", next.Name) } err = readNext(next, tarReader, next.PAXRecords != nil && next.PAXRecords[SingleFile] == "true") if err != nil { From 03ae872f2db07b5108f469bb29284f1dc75fed98 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 13:45:58 -0800 Subject: [PATCH 34/65] apply suggestion --- pkg/wshrpc/wshrpctypes.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 74338b5947..6a7ff9086c 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -752,7 +752,10 @@ type SuggestionType struct { UrlUrl string `json:"url:url,omitempty"` } +// FileShareCapability represents the capabilities of a file share type FileShareCapability struct { - CanAppend bool - CanMkdir bool + // CanAppend indicates whether the file share supports appending to files + CanAppend bool `json:"canappend"` + // CanMkdir indicates whether the file share supports creating directories + CanMkdir bool `json:"canmkdir"` } From 24b26ac913b7d56e5b1f383f3fdd4bb9b7d11c88 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 13:50:23 -0800 Subject: [PATCH 35/65] apply suggestion --- pkg/remote/fileshare/pathtree/pathtree.go | 48 +++++++++++++++-------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/pkg/remote/fileshare/pathtree/pathtree.go b/pkg/remote/fileshare/pathtree/pathtree.go index cb3335f286..cfc09a4bb1 100644 --- a/pkg/remote/fileshare/pathtree/pathtree.go +++ b/pkg/remote/fileshare/pathtree/pathtree.go @@ -46,6 +46,10 @@ func NewTree(path string, delimiter string) *Tree { func (t *Tree) Add(path string) { log.Printf("tree.Add: path: %s", path) + // Validate input + if path == "" { + return + } var relativePath string if t.RootPath == "" { relativePath = path @@ -58,7 +62,6 @@ func (t *Tree) Add(path string) { } } - log.Printf("relativePath: %s", relativePath) // If the path is already in the tree, ignore it if t.nodes[relativePath] != nil { @@ -66,32 +69,45 @@ func (t *Tree) Add(path string) { } components := strings.Split(relativePath, t.delimiter) - log.Printf("components: %v", components) + // Validate path components + for _, component := range components { + if component == "" || component == "." || component == ".." { + return // Skip invalid paths + } + } // Quick check to see if the parent path is already in the tree, in which case we can skip the loop - if len(components) > 1 { - parentPath := strings.Join(components[:len(components)-1], t.delimiter) - log.Printf("parentPath: %s", parentPath) - if t.nodes[parentPath] != nil { - lastPathComponent := components[len(components)-1] - t.nodes[parentPath].Children[lastPathComponent] = &Node{ - Children: make(map[string]*Node), - } - t.nodes[relativePath] = t.nodes[parentPath].Children[lastPathComponent] - return - } + if parent := t.tryAddToExistingParent(components); parent { + return + } + + t.addNewPath(components) +} + +func (t *Tree) tryAddToExistingParent(components []string) bool { + if len(components) <= 1 { + return false + } + parentPath := strings.Join(components[:len(components)-1], t.delimiter) + if t.nodes[parentPath] == nil { + return false } + lastPathComponent := components[len(components)-1] + t.nodes[parentPath].Children[lastPathComponent] = &Node{ + Children: make(map[string]*Node), + } + t.nodes[strings.Join(components, t.delimiter)] = t.nodes[parentPath].Children[lastPathComponent] + return true +} +func (t *Tree) addNewPath(components []string) { currentNode := t.Root for i, component := range components { - log.Printf("component: %s", component) if _, ok := currentNode.Children[component]; !ok { - log.Printf("Adding component: %s", component) currentNode.Children[component] = &Node{ Children: make(map[string]*Node), } curPath := strings.Join(components[:i+1], t.delimiter) - log.Printf("curPath: %s", curPath) t.nodes[curPath] = currentNode.Children[component] } currentNode = currentNode.Children[component] From 5a512f410eba0782a237b96d8c2c1205fc1fefc0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 13:51:58 -0800 Subject: [PATCH 36/65] apply suggestion --- pkg/remote/fileshare/pathtree/pathtree.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/remote/fileshare/pathtree/pathtree.go b/pkg/remote/fileshare/pathtree/pathtree.go index cfc09a4bb1..5d4918fbae 100644 --- a/pkg/remote/fileshare/pathtree/pathtree.go +++ b/pkg/remote/fileshare/pathtree/pathtree.go @@ -31,6 +31,9 @@ func (n *Node) Walk(curPath string, walkFunc WalkFunc, delimiter string) error { } func NewTree(path string, delimiter string) *Tree { + if len(delimiter) > 1 { + log.Printf("Warning: multi-character delimiter '%s' may cause unexpected behavior", delimiter) + } if path != "" && !strings.HasSuffix(path, delimiter) { path += delimiter } From 0e45087f38197765c748a324bccce34bfb3035b7 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Sun, 9 Feb 2025 13:57:14 -0800 Subject: [PATCH 37/65] add dirmode to wavefs --- pkg/remote/fileshare/wavefs/wavefs.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index e54d88b94f..205fe33f4e 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -12,6 +12,7 @@ import ( "io" "io/fs" "log" + "os" "path" "path/filepath" "strings" @@ -30,6 +31,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + DirMode os.FileMode = 0755 | os.ModeDir +) + type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -252,6 +257,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, Size: 0, IsDir: true, SupportsMkdir: false, + Mode: DirMode, }) } fileList = filteredList From 3840be3de4dcd9b7648f98c62b906adf28b4fd64 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 10 Feb 2025 16:16:21 -0800 Subject: [PATCH 38/65] remove logs, fix trailing slash in normalized filepath in dir preview --- frontend/app/view/preview/directorypreview.tsx | 7 +------ frontend/app/view/preview/preview.tsx | 2 +- frontend/types/gotypes.d.ts | 4 ++-- pkg/wshrpc/wshremote/wshremote.go | 11 ----------- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index a5c600760e..83046a6987 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -739,16 +739,11 @@ const TableRow = React.forwardRef(function ( absParent: dirPath, uri: formatRemoteUri(row.getValue("path") as string), }; - const [{ isDragging }, drag, dragPreview] = useDrag( + const [_, drag] = useDrag( () => ({ type: "FILE_ITEM", canDrag: true, item: () => dragItem, - collect: (monitor) => { - return { - isDragging: monitor.isDragging(), - }; - }, }), [dragItem] ); diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 01675288a5..f2a5edf54e 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -386,7 +386,7 @@ export class PreviewModel implements ViewModel { return null; } if (fileInfo.isdir) { - return fileInfo.dir + "/"; + return fileInfo.dir; } return fileInfo.dir + "/" + fileInfo.name; }); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 8c37a42e04..13cf9adafa 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -453,8 +453,8 @@ declare global { // wshrpc.FileShareCapability type FileShareCapability = { - CanAppend: boolean; - CanMkdir: boolean; + canappend: boolean; + canmkdir: boolean; }; // wconfig.FullConfigType diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 112cea2e25..098b81e57f 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -256,7 +256,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } else { pathPrefix = filepath.Dir(cleanedPath) } - log.Printf("RemoteTarStreamCommand: path=%s, pathPrefix=%s\n", path, pathPrefix) timeout := fstype.DefaultTimeout if opts.Timeout > 0 { @@ -277,7 +276,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - log.Printf("RemoteTarStreamCommand: path=%s\n", path) if err = writeHeader(info, path, singleFile); err != nil { return err } @@ -360,9 +358,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if nextinfo != nil { if nextinfo.IsDir() { - log.Printf("RemoteFileCopyCommand: nextinfo is dir, path=%s\n", path) if !finfo.IsDir() { - log.Printf("RemoteFileCopyCommand: finfo is file: %s\n", path) // try to create file in directory path = filepath.Join(path, filepath.Base(finfo.Name())) newdestinfo, err := os.Stat(path) @@ -394,8 +390,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return 0, fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", path) } } - } else { - log.Printf("RemoteFileCopyCommand: nextinfo is nil, path=%s\n", path) } if finfo.IsDir() { @@ -434,10 +428,8 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } if srcFileStat.IsDir() { - log.Print("RemoteFileCopyCommand: copying directory\n") srcPathPrefix := filepath.Dir(srcPathCleaned) if strings.HasSuffix(srcUri, "/") { - log.Printf("RemoteFileCopyCommand: src has slash, using %q as src path\n", srcPathCleaned) srcPathPrefix = srcPathCleaned } err = filepath.Walk(srcPathCleaned, func(path string, info fs.FileInfo, err error) error { @@ -446,7 +438,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } srcFilePath := path destFilePath := filepath.Join(destPathCleaned, strings.TrimPrefix(path, srcPathPrefix)) - log.Printf("RemoteFileCopyCommand: copying %q to %q\n", srcFilePath, destFilePath) var file *os.File if !info.IsDir() { file, err = os.Open(srcFilePath) @@ -462,7 +453,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } } else { - log.Print("RemoteFileCopyCommand: copying single file\n") file, err := os.Open(srcPathCleaned) if err != nil { return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) @@ -470,7 +460,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcPathCleaned) destFilePath := filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) if destHasSlash { - log.Printf("RemoteFileCopyCommand: dest has slash, using %q as dest path\n", destPathCleaned) destFilePath = destPathCleaned } _, err = copyFileFunc(destFilePath, srcFileStat, file) From ff62039d0b17902f794bdc497020c66ca57a07d0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 10 Feb 2025 22:16:23 -0800 Subject: [PATCH 39/65] save --- frontend/app/block/blockframe.tsx | 3 +- frontend/app/modals/conntypeahead.tsx | 14 +-- frontend/app/store/global.ts | 11 +++ frontend/app/store/wshclientapi.ts | 5 ++ frontend/app/view/preview/preview.tsx | 45 +++++++--- pkg/remote/connparse/connparse_test.go | 6 +- pkg/remote/fileshare/fileshare.go | 1 + pkg/remote/fileshare/s3fs/s3fs.go | 119 +++++++++++++------------ pkg/remote/fileshare/wavefs/wavefs.go | 2 +- pkg/util/fileutil/fileutil.go | 10 ++- pkg/util/wavefileutil/wavefileutil.go | 5 +- pkg/wshrpc/wshclient/wshclient.go | 6 ++ pkg/wshrpc/wshrpctypes.go | 2 + pkg/wshrpc/wshserver/wshserver.go | 28 ++++++ 14 files changed, 174 insertions(+), 83 deletions(-) diff --git a/frontend/app/block/blockframe.tsx b/frontend/app/block/blockframe.tsx index 33c33c1263..f5464c3f4d 100644 --- a/frontend/app/block/blockframe.tsx +++ b/frontend/app/block/blockframe.tsx @@ -607,7 +607,8 @@ const BlockFrame_Default_Component = (props: BlockFrameProps) => { "--magnified-block-blur": `${magnifiedBlockBlur}px`, } as React.CSSProperties } - inert={preview ? "1" : undefined} // this does exist in the DOM, just not in react + // @ts-ignore: inert does exist in the DOM, just not in react + inert={preview ? "1" : undefined} // > {preview || viewModel == null ? null : ( diff --git a/frontend/app/modals/conntypeahead.tsx b/frontend/app/modals/conntypeahead.tsx index 5a9831d062..77df024433 100644 --- a/frontend/app/modals/conntypeahead.tsx +++ b/frontend/app/modals/conntypeahead.tsx @@ -377,13 +377,10 @@ const ChangeConnectionBlockModal = React.memo( // typeahead was opened. good candidate for verbose log level. //console.log("unable to load wsl list from backend. using blank list: ", e) }); - ///////// - // TODO-S3 - // this needs an rpc call to generate a list of s3 profiles - const newS3List = []; - setS3List(newS3List); - ///////// - }, [changeConnModalOpen, setConnList]); + RpcApi.ConnListAWSCommand(TabRpcClient, { timeout: 2000 }) + .then((s3List) => setS3List(s3List ?? [])) + .catch((e) => console.log("unable to load s3 list from backend:", e)); + }, [changeConnModalOpen]); const changeConnection = React.useCallback( async (connName: string) => { @@ -393,10 +390,13 @@ const ChangeConnectionBlockModal = React.memo( if (connName == blockData?.meta?.connection) { return; } + const isAws = connName?.startsWith("aws:"); const oldCwd = blockData?.meta?.file ?? ""; let newCwd: string; if (oldCwd == "") { newCwd = ""; + } else if (isAws) { + newCwd = "/"; } else { newCwd = "~"; } diff --git a/frontend/app/store/global.ts b/frontend/app/store/global.ts index dcd161cf8a..37d2022b05 100644 --- a/frontend/app/store/global.ts +++ b/frontend/app/store/global.ts @@ -597,6 +597,17 @@ function getConnStatusAtom(conn: string): PrimitiveAtom { wshenabled: false, }; rtn = atom(connStatus); + } else if (conn.startsWith("aws:")) { + const connStatus: ConnStatus = { + connection: conn, + connected: true, + error: null, + status: "connected", + hasconnected: true, + activeconnnum: 0, + wshenabled: false, + }; + rtn = atom(connStatus); } else { const connStatus: ConnStatus = { connection: conn, diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index f3f9cc7474..1bf5583ebd 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -52,6 +52,11 @@ class RpcApiType { return client.wshRpcCall("connlist", null, opts); } + // command "connlistaws" [call] + ConnListAWSCommand(client: WshClient, opts?: RpcOpts): Promise { + return client.wshRpcCall("connlistaws", null, opts); + } + // command "connreinstallwsh" [call] ConnReinstallWshCommand(client: WshClient, data: ConnExtData, opts?: RpcOpts): Promise { return client.wshRpcCall("connreinstallwsh", data, opts); diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index f2a5edf54e..e64d9b28cf 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -382,13 +382,7 @@ export class PreviewModel implements ViewModel { }); this.normFilePath = atom>(async (get) => { const fileInfo = await get(this.statFile); - if (fileInfo == null) { - return null; - } - if (fileInfo.isdir) { - return fileInfo.dir; - } - return fileInfo.dir + "/" + fileInfo.name; + return fileInfo?.path; }); this.loadableStatFilePath = loadable(this.statFilePath); this.connection = atom>(async (get) => { @@ -406,12 +400,13 @@ export class PreviewModel implements ViewModel { }); this.statFile = atom>(async (get) => { const fileName = get(this.metaFilePath); + const path = await this.formatRemoteUri(fileName, get); if (fileName == null) { return null; } const statFile = await RpcApi.FileInfoCommand(TabRpcClient, { info: { - path: await this.formatRemoteUri(fileName, get), + path, }, }); console.log("stat file", statFile); @@ -427,12 +422,14 @@ export class PreviewModel implements ViewModel { const fullFileAtom = atom>(async (get) => { const fileName = get(this.metaFilePath); + const path = await this.formatRemoteUri(fileName, get); if (fileName == null) { return null; } + console.log("full file path", path); const file = await RpcApi.FileReadCommand(TabRpcClient, { info: { - path: await this.formatRemoteUri(fileName, get), + path, }, }); console.log("full file", file); @@ -442,16 +439,15 @@ export class PreviewModel implements ViewModel { this.fileContentSaved = atom(null) as PrimitiveAtom; const fileContentAtom = atom( async (get) => { - const _ = get(this.metaFilePath); const newContent = get(this.newFileContent); + const savedContent = get(this.fileContentSaved); + const fullFile = await get(fullFileAtom); if (newContent != null) { return newContent; } - const savedContent = get(this.fileContentSaved); if (savedContent != null) { return savedContent; } - const fullFile = await get(fullFileAtom); return base64ToString(fullFile?.data64); }, (_, set, update: string) => { @@ -716,7 +712,19 @@ export class PreviewModel implements ViewModel { if (filePath == null) { return; } - await navigator.clipboard.writeText(filePath); + const conn = await globalStore.get(this.connection); + if (conn) { + // remote path + if (conn.startsWith("aws:")) { + // TODO: We need a better way to handle s3 paths + await navigator.clipboard.writeText(`${conn}:s3://${filePath}`); + } else { + await navigator.clipboard.writeText(`wsh://${conn}/${filePath}`); + } + } else { + // local path + await navigator.clipboard.writeText(filePath); + } }), }); menuItems.push({ @@ -860,8 +868,17 @@ export class PreviewModel implements ViewModel { } async formatRemoteUri(path: string, get: Getter): Promise { + console.log("formatRemoteUri", path); const conn = (await get(this.connection)) ?? "local"; - return `wsh://${conn}/${path}`; + // TODO: We need a better way to handle s3 paths + var retVal: string; + if (conn.startsWith("aws:")) { + retVal = `${conn}:s3://${path ?? ""}`; + } else { + retVal = `wsh://${conn}/${path}`; + } + console.log("formatted", retVal); + return retVal; } } diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index b6fac4f82f..e883ef3fb6 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -77,7 +77,7 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected := "/path/to/file" + expected := "path/to/file" if c.Path != expected { t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } @@ -99,7 +99,7 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected = "/path/to/file" + expected = "path/to/file" if c.Path != expected { t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } @@ -122,7 +122,7 @@ func TestParseURI_WSHRemoteShorthand(t *testing.T) { if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected = "/path/to/file" + expected = "path/to/file" if c.Path != expected { t.Fatalf("expected path to be \"%q\", got \"%q\"", expected, c.Path) } diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index efb968a93f..c3f2c9824a 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -47,6 +47,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { + log.Printf("Read: %v", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { return nil, fmt.Errorf(ErrorParsingConnection, data.Info.Path) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 07f8965e94..cd91c230d3 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -60,27 +60,34 @@ func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wsh func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { bucket := conn.Host objectKey := conn.Path + log.Printf("s3fs.ReadStream: %v", conn.GetFullURI()) rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) go func() { defer close(rtn) - if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { - entries, err := c.ListEntries(ctx, conn, nil) - if err != nil { - rtn <- wshutil.RespErr[wshrpc.FileData](err) - return - } - entryBuf := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) - for _, entry := range entries { - entryBuf = append(entryBuf, entry) - if len(entryBuf) == wshrpc.DirChunkSize { - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entryBuf}} - entryBuf = make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + finfo, err := c.Stat(ctx, conn) + if err != nil { + rtn <- wshutil.RespErr[wshrpc.FileData](err) + return + } + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Info: finfo}} + if finfo.IsDir { + listEntriesCh := c.ListEntriesStream(ctx, conn, nil) + defer func() { + go func() { + for range listEntriesCh { + } + }() + }() + for respUnion := range listEntriesCh { + if respUnion.Error != nil { + rtn <- wshutil.RespErr[wshrpc.FileData](respUnion.Error) + return + } + resp := respUnion.Response + if len(resp.FileInfo) > 0 { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: resp.FileInfo}} } } - if len(entryBuf) > 0 { - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: entryBuf}} - } - return } else { var result *s3.GetObjectOutput var err error @@ -102,10 +109,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da log.Printf("error getting object %v:%v: %v", bucket, objectKey, err) var noKey *types.NoSuchKey if errors.As(err, &noKey) { - log.Printf("Can't get object %s from bucket %s. No such key exists.\n", objectKey, bucket) err = noKey - } else { - log.Printf("Couldn't get object %v:%v. Here's why: %v\n", bucket, objectKey, err) } rtn <- wshutil.RespErr[wshrpc.FileData](err) return @@ -389,10 +393,11 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect } if bucket.Name != nil { entries = append(entries, &wshrpc.FileInfo{ - Path: fmt.Sprintf("%s://%s/", conn.Scheme, *bucket.Name), // add trailing slash to indicate directory - Name: *bucket.Name, - ModTime: bucket.CreationDate.UnixMilli(), - IsDir: true, + Path: *bucket.Name + "/", + Name: *bucket.Name, + ModTime: bucket.CreationDate.UnixMilli(), + IsDir: true, + MimeType: "directory", }) numFetched++ } @@ -414,14 +419,15 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Prefix: aws.String(objectKeyPrefix), } objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) - parentPath := getParentPathUri(conn) + parentPath := getParentPath(conn) if parentPath != "" { rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: []*wshrpc.FileInfo{ { - Path: parentPath, - Name: "..", - IsDir: true, - Size: 0, + Path: parentPath, + Name: "..", + IsDir: true, + Size: 0, + MimeType: "directory", }, }}} } @@ -448,17 +454,19 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) if strings.Count(name, "/") > 0 { name = strings.SplitN(name, "/", 2)[0] - name = name + "/" // add trailing slash to indicate directory + path := fmt.Sprintf("%s/%s/", conn.GetPathWithHost(), name) if entryMap[name] == nil { if _, ok := prevUsedDirKeys[name]; !ok { entryMap[name] = &wshrpc.FileInfo{ - Path: conn.GetFullURI() + name, + Path: path, Name: name, IsDir: true, Dir: objectKeyPrefix, ModTime: lastModTime, Size: 0, } + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) + prevUsedDirKeys[name] = struct{}{} numFetched++ } @@ -468,6 +476,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect continue } + path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) size := int64(0) if obj.Size != nil { size = *obj.Size @@ -476,10 +485,11 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Name: name, IsDir: false, Dir: objectKeyPrefix, - Path: conn.GetFullURI() + name, + Path: path, ModTime: lastModTime, Size: size, } + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) numFetched++ } } @@ -508,12 +518,14 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc bucketName := conn.Host objectKey := conn.Path if bucketName == "" || bucketName == "/" { + // root, refers to list all buckets return &wshrpc.FileInfo{ - Name: "/", - IsDir: true, - Size: 0, - ModTime: 0, - Path: fmt.Sprintf("%s://", conn.Scheme), + Name: "/", + IsDir: true, + Size: 0, + ModTime: 0, + Path: "/", + MimeType: "directory", }, nil } if objectKey == "" || objectKey == "/" { @@ -535,10 +547,12 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc if exists { return &wshrpc.FileInfo{ - Name: bucketName, - IsDir: true, - Size: 0, - ModTime: 0, + Name: bucketName, + Path: bucketName, + IsDir: true, + Size: 0, + ModTime: 0, + MimeType: "directory", }, nil } else { return nil, fmt.Errorf("bucket %v does not exist", bucketName) @@ -547,14 +561,15 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc result, err := c.client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ Bucket: aws.String(bucketName), Key: aws.String(objectKey), + ObjectAttributes: []types.ObjectAttributes{ + types.ObjectAttributesObjectSize, + }, }) if err != nil { var noKey *types.NoSuchKey var notFound *types.NotFound if errors.As(err, &noKey) || errors.As(err, ¬Found) { err = fs.ErrNotExist - } else { - log.Printf("Couldn't get object %v:%v. Here's why: %v\n", bucketName, objectKey, err) } return nil, err } @@ -566,14 +581,16 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc if result.LastModified != nil { lastModified = result.LastModified.UnixMilli() } - return &wshrpc.FileInfo{ + rtn := &wshrpc.FileInfo{ Name: objectKey, - Path: conn.GetFullURI(), - Dir: getParentPathUri(conn), + Path: conn.GetPathWithHost(), + Dir: getParentPath(conn), IsDir: false, Size: size, ModTime: lastModified, - }, nil + } + fileutil.AddMimeTypeToFileInfo(rtn.Path, rtn) + return rtn, nil } func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { @@ -762,14 +779,6 @@ func (c S3Client) GetCapability() wshrpc.FileShareCapability { } } -func getParentPathUri(conn *connparse.Connection) string { - parentPath := getParentPath(conn) - if parentPath == "" { - return "" - } - return fmt.Sprintf("%s://%s", conn.Scheme, parentPath) -} - func getParentPath(conn *connparse.Connection) string { hostAndPath := conn.GetPathWithHost() return getParentPathString(hostAndPath) @@ -777,7 +786,7 @@ func getParentPath(conn *connparse.Connection) string { func getParentPathString(hostAndPath string) string { parentPath := "" - slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) + slashIndices := slashRe.FindAllStringIndex(hostAndPath) if slashIndices != nil && len(slashIndices) > 0 { if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 205fe33f4e..aee3f94a1e 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -122,7 +122,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error getting file info: %w", err)) } singleFile := finfo != nil && !finfo.IsDir - pathPrefix := getPathPrefix(conn) + var pathPrefix string if !singleFile && srcHasSlash { pathPrefix = cleanedPath } else { diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 15a5e49e8f..dda8c93cf0 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -69,7 +69,6 @@ func WinSymlinkDir(path string, bits os.FileMode) bool { // does not return "application/octet-stream" as this is considered a detection failure // can pass an existing fileInfo to avoid re-statting the file // falls back to text/plain for 0 byte files - func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { if fileInfo == nil { statRtn, err := os.Stat(path) @@ -148,6 +147,15 @@ func DetectMimeTypeWithDirEnt(path string, dirEnt fs.DirEntry) string { return "" } +func AddMimeTypeToFileInfo(path string, fileInfo *wshrpc.FileInfo) { + if fileInfo == nil { + return + } + if fileInfo.MimeType == "" { + fileInfo.MimeType = DetectMimeType(path, ToFsFileInfo(fileInfo), false) + } +} + var ( systemBinDirs = []string{ "/bin/", diff --git a/pkg/util/wavefileutil/wavefileutil.go b/pkg/util/wavefileutil/wavefileutil.go index 81b09cf288..7e56921870 100644 --- a/pkg/util/wavefileutil/wavefileutil.go +++ b/pkg/util/wavefileutil/wavefileutil.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -13,7 +14,7 @@ const ( func WaveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) - return &wshrpc.FileInfo{ + rtn := &wshrpc.FileInfo{ Path: path, Name: wf.Name, Opts: &wf.Opts, @@ -21,6 +22,8 @@ func WaveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { Meta: &wf.Meta, SupportsMkdir: false, } + fileutil.AddMimeTypeToFileInfo(path, rtn) + return rtn } func WaveFileListToFileInfoList(wfList []*filestore.WaveFile) []*wshrpc.FileInfo { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 91a1e92625..fec7167344 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -70,6 +70,12 @@ func ConnListCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) ([]string, error) return resp, err } +// command "connlistaws", wshserver.ConnListAWSCommand +func ConnListAWSCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) ([]string, error) { + resp, err := sendRpcRequestCallHelper[[]string](w, "connlistaws", nil, opts) + return resp, err +} + // command "connreinstallwsh", wshserver.ConnReinstallWshCommand func ConnReinstallWshCommand(w *wshutil.WshRpc, data wshrpc.ConnExtData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "connreinstallwsh", data, opts) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 6a7ff9086c..7552c2e380 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -107,6 +107,7 @@ const ( Command_ConnConnect = "connconnect" Command_ConnDisconnect = "conndisconnect" Command_ConnList = "connlist" + Command_ConnListAWS = "connlistaws" Command_WslList = "wsllist" Command_WslDefaultDistro = "wsldefaultdistro" Command_DismissWshFail = "dismisswshfail" @@ -199,6 +200,7 @@ type WshRpcInterface interface { ConnConnectCommand(ctx context.Context, connRequest ConnRequest) error ConnDisconnectCommand(ctx context.Context, connName string) error ConnListCommand(ctx context.Context) ([]string, error) + ConnListAWSCommand(ctx context.Context) ([]string, error) WslListCommand(ctx context.Context) ([]string, error) WslDefaultDistroCommand(ctx context.Context) (string, error) DismissWshFailCommand(ctx context.Context, connName string) error diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 98238651ff..d2a2876ea9 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -24,6 +24,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/genconn" "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/suggestion" @@ -31,6 +32,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/telemetry/telemetrydata" "github.com/wavetermdev/waveterm/pkg/util/envutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/iterfn" "github.com/wavetermdev/waveterm/pkg/util/shellutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" @@ -510,6 +512,15 @@ func termCtxWithLogBlockId(ctx context.Context, logBlockId string) context.Conte } func (ws *WshServer) ConnEnsureCommand(ctx context.Context, data wshrpc.ConnExtData) error { + // TODO: if we add proper wsh connections via aws, we'll need to handle that here + if strings.HasPrefix(data.ConnName, "aws:") { + profiles := awsconn.ParseProfiles() + for profile := range profiles { + if strings.HasPrefix(data.ConnName, profile) { + return nil + } + } + } ctx = genconn.ContextWithConnData(ctx, data.LogBlockId) ctx = termCtxWithLogBlockId(ctx, data.LogBlockId) if strings.HasPrefix(data.ConnName, "wsl://") { @@ -520,6 +531,10 @@ func (ws *WshServer) ConnEnsureCommand(ctx context.Context, data wshrpc.ConnExtD } func (ws *WshServer) ConnDisconnectCommand(ctx context.Context, connName string) error { + // TODO: if we add proper wsh connections via aws, we'll need to handle that here + if strings.HasPrefix(connName, "aws:") { + return nil + } if strings.HasPrefix(connName, "wsl://") { distroName := strings.TrimPrefix(connName, "wsl://") conn := wslconn.GetWslConn(distroName) @@ -540,6 +555,10 @@ func (ws *WshServer) ConnDisconnectCommand(ctx context.Context, connName string) } func (ws *WshServer) ConnConnectCommand(ctx context.Context, connRequest wshrpc.ConnRequest) error { + // TODO: if we add proper wsh connections via aws, we'll need to handle that here + if strings.HasPrefix(connRequest.Host, "aws:") { + return nil + } ctx = genconn.ContextWithConnData(ctx, connRequest.LogBlockId) ctx = termCtxWithLogBlockId(ctx, connRequest.LogBlockId) connName := connRequest.Host @@ -563,6 +582,10 @@ func (ws *WshServer) ConnConnectCommand(ctx context.Context, connRequest wshrpc. } func (ws *WshServer) ConnReinstallWshCommand(ctx context.Context, data wshrpc.ConnExtData) error { + // TODO: if we add proper wsh connections via aws, we'll need to handle that here + if strings.HasPrefix(data.ConnName, "aws:") { + return nil + } ctx = genconn.ContextWithConnData(ctx, data.LogBlockId) ctx = termCtxWithLogBlockId(ctx, data.LogBlockId) connName := data.ConnName @@ -632,6 +655,11 @@ func (ws *WshServer) ConnListCommand(ctx context.Context) ([]string, error) { return conncontroller.GetConnectionsList() } +func (ws *WshServer) ConnListAWSCommand(ctx context.Context) ([]string, error) { + profilesMap := awsconn.ParseProfiles() + return iterfn.MapKeysToSorted(profilesMap), nil +} + func (ws *WshServer) WslListCommand(ctx context.Context) ([]string, error) { distros, err := wsl.RegisteredDistros(ctx) if err != nil { From 6ace12a6bfd87343a0e54ab0be60d9857065448a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 12:32:42 -0800 Subject: [PATCH 40/65] save --- .../app/view/preview/directorypreview.tsx | 31 ++++++---------- frontend/app/view/preview/preview.tsx | 35 +++++++++---------- pkg/remote/fileshare/s3fs/s3fs.go | 11 ++++-- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index 83046a6987..fe73822c1b 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -9,7 +9,7 @@ import { ContextMenuModel } from "@/app/store/contextmenu"; import { PLATFORM, atoms, createBlock, getApi, globalStore } from "@/app/store/global"; import { RpcApi } from "@/app/store/wshclientapi"; import { TabRpcClient } from "@/app/store/wshrpcutil"; -import type { PreviewModel } from "@/app/view/preview/preview"; +import { formatRemoteUri, type PreviewModel } from "@/app/view/preview/preview"; import { checkKeyPressed, isCharacterKeyEvent } from "@/util/keyutil"; import { fireAndForget, isBlank, makeConnRoute, makeNativeLabel } from "@/util/util"; import { offset, useDismiss, useFloating, useInteractions } from "@floating-ui/react"; @@ -683,7 +683,6 @@ function TableBody({ setSearch={setSearch} idx={idx} handleFileContextMenu={handleFileContextMenu} - ref={(el) => (rowRefs.current[idx] = el)} key={idx} /> ))} @@ -696,7 +695,6 @@ function TableBody({ setSearch={setSearch} idx={idx + table.getTopRows().length} handleFileContextMenu={handleFileContextMenu} - ref={(el) => (rowRefs.current[idx] = el)} key={idx} /> ))} @@ -715,29 +713,22 @@ type TableRowProps = { handleFileContextMenu: (e: any, finfo: FileInfo) => Promise; }; -const TableRow = React.forwardRef(function ( - { model, row, focusIndex, setFocusIndex, setSearch, idx, handleFileContextMenu }: TableRowProps, - ref: React.RefObject -) { +const TableRow = React.forwardRef(function ({ + model, + row, + focusIndex, + setFocusIndex, + setSearch, + idx, + handleFileContextMenu, +}: TableRowProps) { const dirPath = useAtomValue(model.normFilePath); const connection = useAtomValue(model.connection); - const formatRemoteUri = useCallback( - (path: string) => { - let conn: string; - if (!connection) { - conn = "local"; - } else { - conn = connection; - } - return `wsh://${conn}/${path}`; - }, - [connection] - ); const dragItem: DraggedFile = { relName: row.getValue("name") as string, absParent: dirPath, - uri: formatRemoteUri(row.getValue("path") as string), + uri: formatRemoteUri(row.getValue("path") as string, connection), }; const [_, drag] = useDrag( () => ({ diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index e64d9b28cf..8abc863bc3 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -715,12 +715,7 @@ export class PreviewModel implements ViewModel { const conn = await globalStore.get(this.connection); if (conn) { // remote path - if (conn.startsWith("aws:")) { - // TODO: We need a better way to handle s3 paths - await navigator.clipboard.writeText(`${conn}:s3://${filePath}`); - } else { - await navigator.clipboard.writeText(`wsh://${conn}/${filePath}`); - } + await navigator.clipboard.writeText(formatRemoteUri(filePath, conn)); } else { // local path await navigator.clipboard.writeText(filePath); @@ -868,17 +863,7 @@ export class PreviewModel implements ViewModel { } async formatRemoteUri(path: string, get: Getter): Promise { - console.log("formatRemoteUri", path); - const conn = (await get(this.connection)) ?? "local"; - // TODO: We need a better way to handle s3 paths - var retVal: string; - if (conn.startsWith("aws:")) { - retVal = `${conn}:s3://${path ?? ""}`; - } else { - retVal = `wsh://${conn}/${path}`; - } - console.log("formatted", retVal); - return retVal; + return formatRemoteUri(path, await get(this.connection)); } } @@ -1244,4 +1229,18 @@ const OpenFileModal = memo( } ); -export { makePreviewModel, PreviewView }; +function formatRemoteUri(path: string, connection: string): string { + console.log("formatRemoteUri", path); + connection = connection ?? "local"; + // TODO: We need a better way to handle s3 paths + var retVal: string; + if (connection.startsWith("aws:")) { + retVal = `${connection}:s3://${path ?? ""}`; + } else { + retVal = `wsh://${connection}/${path}`; + } + console.log("formatted", retVal); + return retVal; +} + +export { formatRemoteUri, makePreviewModel, PreviewView }; diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index cd91c230d3..344ee316bf 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -174,9 +174,11 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, // get the object if it's a single file operation var singleFileResult *s3.GetObjectOutput + // this ensures we don't leak the object if we error out before copying it + closeSingleFileResult := true defer func() { // in case we error out before the object gets copied, make sure to close it - if singleFileResult != nil { + if singleFileResult != nil && closeSingleFileResult { utilfn.GracefulClose(singleFileResult.Body, "s3fs", conn.Path) } }() @@ -224,7 +226,6 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, tarPathPrefix) - go func() { defer func() { tarClose() @@ -236,6 +237,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, // close the objects when we're done defer func() { for key, obj := range objMap { + log.Printf("closing object %v", key) utilfn.GracefulClose(obj.Body, "s3fs", key) } }() @@ -352,6 +354,8 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, return } }() + // we've handed singleFileResult off to the tar writer, so we don't want to close it + closeSingleFileResult = false return rtn } @@ -786,7 +790,7 @@ func getParentPath(conn *connparse.Connection) string { func getParentPathString(hostAndPath string) string { parentPath := "" - slashIndices := slashRe.FindAllStringIndex(hostAndPath) + slashIndices := slashRe.FindAllStringIndex(hostAndPath, 0) if slashIndices != nil && len(slashIndices) > 0 { if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] @@ -794,6 +798,7 @@ func getParentPathString(hostAndPath string) string { parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] } } + log.Printf("hostAndPath: %v, parentPath: %v", hostAndPath, parentPath) return parentPath } From c81ef48a355de1fb1de722f9581f07834dca7155 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 12:43:49 -0800 Subject: [PATCH 41/65] Update pkg/remote/fileshare/s3fs/s3fs.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 344ee316bf..1d6410c751 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -790,7 +790,7 @@ func getParentPath(conn *connparse.Connection) string { func getParentPathString(hostAndPath string) string { parentPath := "" - slashIndices := slashRe.FindAllStringIndex(hostAndPath, 0) + slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) if slashIndices != nil && len(slashIndices) > 0 { if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] From 501b79d5c2fbb486e58ae077116329afb5da85d1 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 13:10:13 -0800 Subject: [PATCH 42/65] save --- pkg/remote/fileshare/s3fs/s3fs.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 1d6410c751..4231e75f6a 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -125,6 +125,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da ModTime: result.LastModified.UnixMilli(), Path: conn.GetFullURI(), } + fileutil.AddMimeTypeToFileInfo(finfo.Path, finfo) log.Printf("file info: %v", finfo) rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Info: finfo}} if size == 0 { @@ -397,7 +398,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect } if bucket.Name != nil { entries = append(entries, &wshrpc.FileInfo{ - Path: *bucket.Name + "/", + Path: *bucket.Name, Name: *bucket.Name, ModTime: bucket.CreationDate.UnixMilli(), IsDir: true, @@ -458,7 +459,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) if strings.Count(name, "/") > 0 { name = strings.SplitN(name, "/", 2)[0] - path := fmt.Sprintf("%s/%s/", conn.GetPathWithHost(), name) + path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) if entryMap[name] == nil { if _, ok := prevUsedDirKeys[name]; !ok { entryMap[name] = &wshrpc.FileInfo{ @@ -572,10 +573,18 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc if err != nil { var noKey *types.NoSuchKey var notFound *types.NotFound - if errors.As(err, &noKey) || errors.As(err, ¬Found) { - err = fs.ErrNotExist + if !errors.As(err, &noKey) && !errors.As(err, ¬Found) { + return nil, err + } else { + return &wshrpc.FileInfo{ + Name: objectKey, + Path: conn.GetPathWithHost(), + IsDir: true, + Size: 0, + ModTime: 0, + MimeType: "directory", + }, nil } - return nil, err } size := int64(0) if result.ObjectSize != nil { @@ -789,7 +798,7 @@ func getParentPath(conn *connparse.Connection) string { } func getParentPathString(hostAndPath string) string { - parentPath := "" + parentPath := "/" slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) if slashIndices != nil && len(slashIndices) > 0 { if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { From abed821c82c8b750e4d0ecefc314bd2befe14165 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 13:34:25 -0800 Subject: [PATCH 43/65] fix putfile --- pkg/remote/fileshare/s3fs/s3fs.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 4231e75f6a..0b44132d7f 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,6 +5,7 @@ package s3fs import ( "archive/tar" + "bytes" "context" "encoding/base64" "errors" @@ -607,20 +608,37 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc } func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { + log.Printf("PutFile: %v", conn.GetFullURI()) if data.At != nil { + log.Printf("PutFile: offset %d and size %d", data.At.Offset, data.At.Size) return errors.Join(errors.ErrUnsupported, fmt.Errorf("file data offset and size not supported")) } bucket := conn.Host objectKey := conn.Path if bucket == "" || bucket == "/" || objectKey == "" || objectKey == "/" { + log.Printf("PutFile: bucket and object key must be specified") return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket and object key must be specified")) } - _, err := c.client.PutObject(ctx, &s3.PutObjectInput{ + if len(data.Data64) == 0 { + data.Data64 = base64.StdEncoding.EncodeToString([]byte("\n")) + } + contentMaxLength := base64.StdEncoding.DecodedLen(len(data.Data64)) + decodedBody := make([]byte, contentMaxLength) + contentLength, err := base64.StdEncoding.Decode(decodedBody, []byte(data.Data64)) + if err != nil { + log.Printf("PutFile: error decoding data: %v", err) + return err + } + bodyReaderSeeker := bytes.NewReader(decodedBody[:contentLength]) + _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(objectKey), - Body: base64.NewDecoder(base64.StdEncoding, strings.NewReader(data.Data64)), - ContentLength: aws.Int64(int64(base64.StdEncoding.DecodedLen(len(data.Data64)))), + Body: bodyReaderSeeker, + ContentLength: aws.Int64(int64(contentLength)), }) + if err != nil { + log.Printf("PutFile: error putting object %v:%v: %v", bucket, objectKey, err) + } return err } From a74e1ebc0ad4a6591c8fad4bd70eab75dfe11613 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 13:36:39 -0800 Subject: [PATCH 44/65] fix putfile better --- pkg/remote/fileshare/s3fs/s3fs.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 0b44132d7f..42f1435a02 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -619,15 +619,20 @@ func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data log.Printf("PutFile: bucket and object key must be specified") return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket and object key must be specified")) } - if len(data.Data64) == 0 { - data.Data64 = base64.StdEncoding.EncodeToString([]byte("\n")) - } contentMaxLength := base64.StdEncoding.DecodedLen(len(data.Data64)) - decodedBody := make([]byte, contentMaxLength) - contentLength, err := base64.StdEncoding.Decode(decodedBody, []byte(data.Data64)) - if err != nil { - log.Printf("PutFile: error decoding data: %v", err) - return err + var decodedBody []byte + var contentLength int + var err error + if contentMaxLength > 0 { + decodedBody = make([]byte, contentMaxLength) + contentLength, err = base64.StdEncoding.Decode(decodedBody, []byte(data.Data64)) + if err != nil { + log.Printf("PutFile: error decoding data: %v", err) + return err + } + } else { + decodedBody = []byte("\n") + contentLength = 1 } bodyReaderSeeker := bytes.NewReader(decodedBody[:contentLength]) _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ From d75b7cd297282ad7f265f88f211810d268b6e714 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 16:15:03 -0800 Subject: [PATCH 45/65] fix parent dirs --- frontend/app/store/wshclientapi.ts | 5 ++ .../app/view/preview/directorypreview.tsx | 8 ++- frontend/app/view/preview/preview.tsx | 33 ++++------ pkg/remote/fileshare/fileshare.go | 4 +- pkg/remote/fileshare/fstype/fstype.go | 7 ++- pkg/remote/fileshare/fsutil/fsutil.go | 38 +++++++++++ pkg/remote/fileshare/s3fs/s3fs.go | 63 +++++-------------- pkg/remote/fileshare/wavefs/wavefs.go | 14 +++-- pkg/remote/fileshare/wshfs/wshfs.go | 8 +-- pkg/util/wavefileutil/wavefileutil.go | 2 + pkg/wshrpc/wshclient/wshclient.go | 6 ++ pkg/wshrpc/wshrpctypes.go | 60 ++++++++++-------- pkg/wshrpc/wshserver/wshserver.go | 10 +++ 13 files changed, 146 insertions(+), 112 deletions(-) create mode 100644 pkg/remote/fileshare/fsutil/fsutil.go diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 1bf5583ebd..0a8e1456a5 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -187,6 +187,11 @@ class RpcApiType { return client.wshRpcCall("fileinfo", data, opts); } + // command "filejoin" [call] + FileJoinCommand(client: WshClient, data: string[], opts?: RpcOpts): Promise { + return client.wshRpcCall("filejoin", data, opts); + } + // command "filelist" [call] FileListCommand(client: WshClient, data: FileListData, opts?: RpcOpts): Promise { return client.wshRpcCall("filelist", data, opts); diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index fe73822c1b..b00687541d 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -11,7 +11,7 @@ import { RpcApi } from "@/app/store/wshclientapi"; import { TabRpcClient } from "@/app/store/wshrpcutil"; import { formatRemoteUri, type PreviewModel } from "@/app/view/preview/preview"; import { checkKeyPressed, isCharacterKeyEvent } from "@/util/keyutil"; -import { fireAndForget, isBlank, makeConnRoute, makeNativeLabel } from "@/util/util"; +import { fireAndForget, isBlank, makeNativeLabel } from "@/util/util"; import { offset, useDismiss, useFloating, useInteractions } from "@floating-ui/react"; import { Column, @@ -528,8 +528,10 @@ function TableBody({ const fileName = finfo.path.split("/").pop(); let parentFileInfo: FileInfo; try { - parentFileInfo = await RpcApi.RemoteFileJoinCommand(TabRpcClient, [normPath, ".."], { - route: makeConnRoute(conn), + parentFileInfo = await RpcApi.FileInfoCommand(TabRpcClient, { + info: { + path: await model.formatRemoteUri(finfo.dir, globalStore.get), + }, }); } catch (e) { console.log("could not get parent file info. using child file info as fallback"); diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 35394731de..08f3979cdf 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -400,6 +400,7 @@ export class PreviewModel implements ViewModel { }); this.statFile = atom>(async (get) => { const fileName = get(this.metaFilePath); + console.log("stat file", fileName); const path = await this.formatRemoteUri(fileName, get); if (fileName == null) { return null; @@ -579,10 +580,11 @@ export class PreviewModel implements ViewModel { } async getParentInfo(fileInfo: FileInfo): Promise { - const conn = await globalStore.get(this.connection); try { - const parentFileInfo = await RpcApi.RemoteFileJoinCommand(TabRpcClient, [fileInfo.path, ".."], { - route: makeConnRoute(conn), + const parentFileInfo = await RpcApi.FileInfoCommand(TabRpcClient, { + info: { + path: await this.formatRemoteUri(fileInfo.dir, globalStore.get), + }, }); console.log("parent file info", parentFileInfo); return parentFileInfo; @@ -601,11 +603,13 @@ export class PreviewModel implements ViewModel { this.updateOpenFileModalAndError(false); return true; } - const conn = await globalStore.get(this.connection); try { - const newFileInfo = await RpcApi.RemoteFileJoinCommand(TabRpcClient, [fileInfo.path, ".."], { - route: makeConnRoute(conn), + const newFileInfo = await RpcApi.FileInfoCommand(TabRpcClient, { + info: { + path: await this.formatRemoteUri(fileInfo.dir, globalStore.get), + }, }); + console.log("parent file info", newFileInfo); if (newFileInfo.path != "" && newFileInfo.notfound) { console.log("does not exist, ", newFileInfo.path); this.goParentDirectory({ fileInfo: newFileInfo }); @@ -616,7 +620,7 @@ export class PreviewModel implements ViewModel { refocusNode(this.blockId); } catch (e) { globalStore.set(this.openFileError, e.message); - console.error("Error opening file", [fileInfo.dir, ".."], e); + console.error("Error opening file", fileInfo.dir, e); } } @@ -682,22 +686,12 @@ export class PreviewModel implements ViewModel { } async handleOpenFile(filePath: string) { - const fileInfo = await globalStore.get(this.statFile); - if (fileInfo == null) { - this.updateOpenFileModalAndError(false); - return true; - } - const conn = await globalStore.get(this.connection); try { - const newFileInfo = await RpcApi.RemoteFileJoinCommand(TabRpcClient, [fileInfo.dir, filePath], { - route: makeConnRoute(conn), - }); - this.updateOpenFileModalAndError(false); - this.goHistory(newFileInfo.path); + this.goHistory(filePath); refocusNode(this.blockId); } catch (e) { globalStore.set(this.openFileError, e.message); - console.error("Error opening file", fileInfo.dir, filePath, e); + console.error("Error opening file", filePath, e); } } @@ -1119,7 +1113,6 @@ const fetchSuggestions = async ( }; function PreviewView({ - blockId, blockRef, contentRef, model, diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index c3f2c9824a..558da7e551 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -163,10 +163,10 @@ func Delete(ctx context.Context, data wshrpc.CommandDeleteFileData) error { return client.Delete(ctx, conn, data.Recursive) } -func Join(ctx context.Context, path string, parts ...string) (string, error) { +func Join(ctx context.Context, path string, parts ...string) (*wshrpc.FileInfo, error) { client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return "", fmt.Errorf(ErrorParsingConnection, path) + return nil, fmt.Errorf(ErrorParsingConnection, path) } return client.Join(ctx, conn, parts...) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index dd5a83cb42..cc67ddeab9 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -5,6 +5,7 @@ package fstype import ( "context" + "os" "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -13,7 +14,9 @@ import ( ) const ( - DefaultTimeout = 30 * time.Second + DefaultTimeout = 30 * time.Second + FileMode os.FileMode = 0644 + DirMode os.FileMode = 0755 | os.ModeDir ) type FileShareClient interface { @@ -44,7 +47,7 @@ type FileShareClient interface { // Delete deletes the entry at the given path Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error // Join joins the given parts to the connection path - Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) + Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) // GetConnectionType returns the type of connection for the fileshare GetConnectionType() string // GetCapability returns the capability of the fileshare diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go new file mode 100644 index 0000000000..6ee5e70f40 --- /dev/null +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -0,0 +1,38 @@ +package fsutil + +import ( + "regexp" + "strings" + + "github.com/wavetermdev/waveterm/pkg/remote/connparse" +) + +var slashRe = regexp.MustCompile(`/`) + +func GetParentPath(conn *connparse.Connection) string { + hostAndPath := conn.GetPathWithHost() + return GetParentPathString(hostAndPath) +} + +func GetParentPathString(hostAndPath string) string { + parentPath := "/" + slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) + if slashIndices != nil && len(slashIndices) > 0 { + if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] + } else if len(slashIndices) > 1 { + parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] + } + } + return parentPath +} + +func GetPathPrefix(conn *connparse.Connection) string { + fullUri := conn.GetFullURI() + pathPrefix := fullUri + lastSlash := strings.LastIndex(fullUri, "/") + if lastSlash > 10 && lastSlash < len(fullUri)-1 { + pathPrefix = fullUri[:lastSlash+1] + } + return pathPrefix +} diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 42f1435a02..ebad01b178 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -13,9 +13,7 @@ import ( "io" "io/fs" "log" - "os" "path" - "regexp" "strings" "sync" "time" @@ -27,6 +25,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" @@ -36,11 +35,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - FileMode os.FileMode = 0644 - DirMode os.FileMode = 0755 | os.ModeDir -) - type S3Client struct { client *s3.Client } @@ -125,6 +119,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da Size: size, ModTime: result.LastModified.UnixMilli(), Path: conn.GetFullURI(), + Dir: fsutil.GetParentPath(conn), } fileutil.AddMimeTypeToFileInfo(finfo.Path, finfo) log.Printf("file info: %v", finfo) @@ -224,7 +219,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } else if singleFile || includeDir { // if we're including the directory itself, we need to remove the last part of the path - tarPathPrefix = getParentPathString(tarPathPrefix) + tarPathPrefix = fsutil.GetParentPathString(tarPathPrefix) } rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, tarPathPrefix) @@ -321,11 +316,11 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, // default vals assume entry is dir, since mapEntry might not exist modTime := int64(time.Now().Unix()) - mode := DirMode + mode := fstype.DirMode size := int64(numChildren) if isFile { - mode = FileMode + mode = fstype.FileMode size = *mapEntry.ContentLength if mapEntry.LastModified != nil { modTime = mapEntry.LastModified.UnixMilli() @@ -374,8 +369,6 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return entries, nil } -var slashRe = regexp.MustCompile(`/`) - func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { bucket := conn.Host objectKeyPrefix := conn.Path @@ -401,6 +394,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect entries = append(entries, &wshrpc.FileInfo{ Path: *bucket.Name, Name: *bucket.Name, + Dir: "/", ModTime: bucket.CreationDate.UnixMilli(), IsDir: true, MimeType: "directory", @@ -425,11 +419,12 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Prefix: aws.String(objectKeyPrefix), } objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) - parentPath := getParentPath(conn) + parentPath := fsutil.GetParentPath(conn) if parentPath != "" { rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: []*wshrpc.FileInfo{ { Path: parentPath, + Dir: fsutil.GetParentPathString(parentPath), Name: "..", IsDir: true, Size: 0, @@ -531,6 +526,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc Size: 0, ModTime: 0, Path: "/", + Dir: "/", MimeType: "directory", }, nil } @@ -555,6 +551,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: bucketName, Path: bucketName, + Dir: "/", IsDir: true, Size: 0, ModTime: 0, @@ -580,6 +577,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: objectKey, Path: conn.GetPathWithHost(), + Dir: fsutil.GetParentPath(conn), IsDir: true, Size: 0, ModTime: 0, @@ -598,7 +596,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc rtn := &wshrpc.FileInfo{ Name: objectKey, Path: conn.GetPathWithHost(), - Dir: getParentPath(conn), + Dir: fsutil.GetParentPath(conn), IsDir: false, Size: size, ModTime: lastModified, @@ -668,7 +666,7 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C overwrite := opts != nil && opts.Overwrite merge := opts != nil && opts.Merge destHasSlash := strings.HasSuffix(destConn.Path, "/") - destPrefix := getPathPrefix(destConn) + destPrefix := fsutil.GetPathPrefix(destConn) destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") if destBucket == "" || destBucket == "/" { return fmt.Errorf("destination bucket must be specified") @@ -787,7 +785,7 @@ func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recurs return err } -func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { +func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { var joinParts []string if conn.Host == "" || conn.Host == "/" { if conn.Path == "" || conn.Path == "/" { @@ -801,7 +799,9 @@ func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts .. joinParts = append([]string{conn.Host, conn.Path}, parts...) } - return fmt.Sprintf("%s://%s", conn.Scheme, strings.Join(joinParts, "/")), nil + conn.Path = strings.Join(joinParts, "/") + + return c.Stat(ctx, conn) } func (c S3Client) GetConnectionType() string { @@ -815,35 +815,6 @@ func (c S3Client) GetCapability() wshrpc.FileShareCapability { } } -func getParentPath(conn *connparse.Connection) string { - hostAndPath := conn.GetPathWithHost() - return getParentPathString(hostAndPath) -} - -func getParentPathString(hostAndPath string) string { - parentPath := "/" - slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) - if slashIndices != nil && len(slashIndices) > 0 { - if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] - } else if len(slashIndices) > 1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] - } - } - log.Printf("hostAndPath: %v, parentPath: %v", hostAndPath, parentPath) - return parentPath -} - -func getPathPrefix(conn *connparse.Connection) string { - fullUri := conn.GetFullURI() - pathPrefix := fullUri - lastSlash := strings.LastIndex(fullUri, "/") - if lastSlash > 10 && lastSlash < len(fullUri)-1 { - pathPrefix = fullUri[:lastSlash+1] - } - return pathPrefix -} - func cleanPath(path string) (string, error) { if path == "" { return "", fmt.Errorf("path is empty") diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index aee3f94a1e..f22ce7610d 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/tarcopy" @@ -249,11 +250,11 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, filteredList = append(filteredList, file) } for dir := range dirMap { - dirName := prefix + dir + "/" + dirName := prefix + dir filteredList = append(filteredList, &wshrpc.FileInfo{ - Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, dirName), + Path: strings.Join([]string{conn.GetPathWithHost(), dirName}, "/"), Name: dirName, - Dir: dirName, + Dir: fsutil.GetParentPathString(dirName), Size: 0, IsDir: true, SupportsMkdir: false, @@ -587,13 +588,14 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu return nil } -func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { +func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { newPath := path.Join(append([]string{conn.Path}, parts...)...) newPath, err := cleanPath(newPath) if err != nil { - return "", fmt.Errorf("error cleaning path: %w", err) + return nil, fmt.Errorf("error cleaning path: %w", err) } - return newPath, nil + conn.Path = newPath + return c.Stat(ctx, conn) } func (c WaveClient) GetCapability() wshrpc.FileShareCapability { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index e4bf8943e1..9d1255a115 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -133,12 +133,8 @@ func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection, recur return wshclient.RemoteFileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: conn.Path, Recursive: recursive}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { - finfo, err := wshclient.RemoteFileJoinCommand(RpcClient, append([]string{conn.Path}, parts...), &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) - if err != nil { - return "", err - } - return finfo.Path, nil +func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { + return wshclient.RemoteFileJoinCommand(RpcClient, append([]string{conn.Path}, parts...), &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) GetConnectionType() string { diff --git a/pkg/util/wavefileutil/wavefileutil.go b/pkg/util/wavefileutil/wavefileutil.go index 7e56921870..7334bce7aa 100644 --- a/pkg/util/wavefileutil/wavefileutil.go +++ b/pkg/util/wavefileutil/wavefileutil.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -16,6 +17,7 @@ func WaveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) rtn := &wshrpc.FileInfo{ Path: path, + Dir: fsutil.GetParentPathString(path), Name: wf.Name, Opts: &wf.Opts, Size: wf.Size, diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index fec7167344..bc84337805 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -232,6 +232,12 @@ func FileInfoCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp return resp, err } +// command "filejoin", wshserver.FileJoinCommand +func FileJoinCommand(w *wshutil.WshRpc, data []string, opts *wshrpc.RpcOpts) (*wshrpc.FileInfo, error) { + resp, err := sendRpcRequestCallHelper[*wshrpc.FileInfo](w, "filejoin", data, opts) + return resp, err +} + // command "filelist", wshserver.FileListCommand func FileListCommand(w *wshutil.WshRpc, data wshrpc.FileListData, opts *wshrpc.RpcOpts) ([]*wshrpc.FileInfo, error) { resp, err := sendRpcRequestCallHelper[[]*wshrpc.FileInfo](w, "filelist", data, opts) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index d683a90e76..62c23223ab 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -51,33 +51,36 @@ const ( // TODO generate these constants from the interface const ( - Command_Authenticate = "authenticate" // special - Command_AuthenticateToken = "authenticatetoken" // special - Command_Dispose = "dispose" // special (disposes of the route, for multiproxy only) - Command_RouteAnnounce = "routeannounce" // special (for routing) - Command_RouteUnannounce = "routeunannounce" // special (for routing) - Command_Message = "message" - Command_GetMeta = "getmeta" - Command_SetMeta = "setmeta" - Command_SetView = "setview" - Command_ControllerInput = "controllerinput" - Command_ControllerRestart = "controllerrestart" - Command_ControllerStop = "controllerstop" - Command_ControllerResync = "controllerresync" - Command_FileAppend = "fileappend" - Command_FileAppendIJson = "fileappendijson" - Command_Mkdir = "mkdir" - Command_ResolveIds = "resolveids" - Command_BlockInfo = "blockinfo" - Command_CreateBlock = "createblock" - Command_DeleteBlock = "deleteblock" - Command_FileWrite = "filewrite" - Command_FileRead = "fileread" - Command_FileReadStream = "filereadstream" - Command_FileMove = "filemove" - Command_FileCopy = "filecopy" - Command_FileStreamTar = "filestreamtar" - Command_FileShareCapability = "filesharecapability" + Command_Authenticate = "authenticate" // special + Command_AuthenticateToken = "authenticatetoken" // special + Command_Dispose = "dispose" // special (disposes of the route, for multiproxy only) + Command_RouteAnnounce = "routeannounce" // special (for routing) + Command_RouteUnannounce = "routeunannounce" // special (for routing) + Command_Message = "message" + Command_GetMeta = "getmeta" + Command_SetMeta = "setmeta" + Command_SetView = "setview" + Command_ControllerInput = "controllerinput" + Command_ControllerRestart = "controllerrestart" + Command_ControllerStop = "controllerstop" + Command_ControllerResync = "controllerresync" + Command_Mkdir = "mkdir" + Command_ResolveIds = "resolveids" + Command_BlockInfo = "blockinfo" + Command_CreateBlock = "createblock" + Command_DeleteBlock = "deleteblock" + + Command_FileWrite = "filewrite" + Command_FileRead = "fileread" + Command_FileReadStream = "filereadstream" + Command_FileMove = "filemove" + Command_FileCopy = "filecopy" + Command_FileStreamTar = "filestreamtar" + Command_FileAppend = "fileappend" + Command_FileAppendIJson = "fileappendijson" + Command_FileJoin = "filejoin" + Command_FileShareCapability = "filesharecapability" + Command_EventPublish = "eventpublish" Command_EventRecv = "eventrecv" Command_EventSub = "eventsub" @@ -162,6 +165,7 @@ type WshRpcInterface interface { DeleteBlockCommand(ctx context.Context, data CommandDeleteBlockData) error DeleteSubBlockCommand(ctx context.Context, data CommandDeleteBlockData) error WaitForRouteCommand(ctx context.Context, data CommandWaitForRouteData) (bool, error) + FileMkdirCommand(ctx context.Context, data FileData) error FileCreateCommand(ctx context.Context, data FileData) error FileDeleteCommand(ctx context.Context, data CommandDeleteFileData) error @@ -175,7 +179,9 @@ type WshRpcInterface interface { FileCopyCommand(ctx context.Context, data CommandFileCopyData) error FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) + FileJoinCommand(ctx context.Context, paths []string) (*FileInfo, error) FileListStreamCommand(ctx context.Context, data FileListData) <-chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] + FileShareCapabilityCommand(ctx context.Context, path string) (FileShareCapability, error) EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 06db621010..08be8a3de9 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -430,6 +430,16 @@ func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.Com return nil } +func (ws *WshServer) FileJoinCommand(ctx context.Context, paths []string) (*wshrpc.FileInfo, error) { + if len(paths) < 2 { + if len(paths) == 0 { + return nil, fmt.Errorf("no paths provided") + } + return fileshare.Stat(ctx, paths[0]) + } + return fileshare.Join(ctx, paths[0], paths[1:]...) +} + func (ws *WshServer) FileShareCapabilityCommand(ctx context.Context, path string) (wshrpc.FileShareCapability, error) { return fileshare.GetCapability(ctx, path) } From c3714b35623df2466f29d759f9254b3523850fe5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 16:21:15 -0800 Subject: [PATCH 46/65] Update pkg/remote/fileshare/fsutil/fsutil.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/fileshare/fsutil/fsutil.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index 6ee5e70f40..2b69256b58 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -15,16 +15,20 @@ func GetParentPath(conn *connparse.Connection) string { } func GetParentPathString(hostAndPath string) string { - parentPath := "/" - slashIndices := slashRe.FindAllStringIndex(hostAndPath, -1) - if slashIndices != nil && len(slashIndices) > 0 { - if slashIndices[len(slashIndices)-1][0] != len(hostAndPath)-1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-1][0]+1] - } else if len(slashIndices) > 1 { - parentPath = hostAndPath[:slashIndices[len(slashIndices)-2][0]+1] - } - } - return parentPath + if hostAndPath == "" || hostAndPath == "/" { + return "/" + } + + // Remove trailing slash if present + if strings.HasSuffix(hostAndPath, "/") { + hostAndPath = hostAndPath[:len(hostAndPath)-1] + } + + lastSlash := strings.LastIndex(hostAndPath, "/") + if lastSlash <= 0 { + return "/" + } + return hostAndPath[:lastSlash+1] } func GetPathPrefix(conn *connparse.Connection) string { From 9933e12649f164b2ae9337c9f36d5212578b581f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 16:21:53 -0800 Subject: [PATCH 47/65] Update pkg/remote/fileshare/fsutil/fsutil.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/fileshare/fsutil/fsutil.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index 2b69256b58..4087036850 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -31,11 +31,16 @@ func GetParentPathString(hostAndPath string) string { return hostAndPath[:lastSlash+1] } +const minURILength = 10 // Minimum length for a valid URI (e.g., "s3://bucket") + func GetPathPrefix(conn *connparse.Connection) string { fullUri := conn.GetFullURI() + if fullUri == "" { + return "" + } pathPrefix := fullUri lastSlash := strings.LastIndex(fullUri, "/") - if lastSlash > 10 && lastSlash < len(fullUri)-1 { + if lastSlash > minURILength && lastSlash < len(fullUri)-1 { pathPrefix = fullUri[:lastSlash+1] } return pathPrefix From b63b1960e4cf760c219480e4f7752a3676178a26 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 16:24:14 -0800 Subject: [PATCH 48/65] clean up formatRemoteUri --- frontend/app/view/preview/preview.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 08f3979cdf..12171cf393 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -1227,16 +1227,14 @@ const OpenFileModal = memo( ); function formatRemoteUri(path: string, connection: string): string { - console.log("formatRemoteUri", path); connection = connection ?? "local"; // TODO: We need a better way to handle s3 paths - var retVal: string; + let retVal: string; if (connection.startsWith("aws:")) { retVal = `${connection}:s3://${path ?? ""}`; } else { retVal = `wsh://${connection}/${path}`; } - console.log("formatted", retVal); return retVal; } From 9a0d3d9c9bdca588f8b3fe7755f43e0d204c1621 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 16:44:41 -0800 Subject: [PATCH 49/65] fix local single file copy --- pkg/wshrpc/wshremote/wshremote.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 098b81e57f..ba7200fff0 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -458,8 +458,10 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) } defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcPathCleaned) - destFilePath := filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) + var destFilePath string if destHasSlash { + destFilePath = filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) + } else { destFilePath = destPathCleaned } _, err = copyFileFunc(destFilePath, srcFileStat, file) @@ -485,11 +487,9 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C numFiles++ nextpath := filepath.Join(destPathCleaned, next.Name) log.Printf("RemoteFileCopyCommand: copying %q to %q\n", next.Name, nextpath) - if singleFile { + if singleFile && !destHasSlash { // custom flag to indicate that the source is a single file, not a directory the contents of a directory - if !destHasSlash { - nextpath = destPathCleaned - } + nextpath = destPathCleaned } finfo := next.FileInfo() n, err := copyFileFunc(nextpath, finfo, reader) From 9e1bed9daa523737d7fa1647e08de50bd7bd26ab Mon Sep 17 00:00:00 2001 From: sawka Date: Tue, 11 Feb 2025 17:28:13 -0800 Subject: [PATCH 50/65] fix scrollintoview, also add pageup/pagedown handlers --- frontend/app/suggestion/suggestion.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/frontend/app/suggestion/suggestion.tsx b/frontend/app/suggestion/suggestion.tsx index 84b6340204..ac2b9767cb 100644 --- a/frontend/app/suggestion/suggestion.tsx +++ b/frontend/app/suggestion/suggestion.tsx @@ -232,6 +232,18 @@ const SuggestionControlInner: React.FC = ({ return () => document.removeEventListener("mousedown", handleClickOutside); }, [onClose, anchorRef]); + useEffect(() => { + if (dropdownRef.current) { + const children = dropdownRef.current.children; + if (children[selectedIndex]) { + (children[selectedIndex] as HTMLElement).scrollIntoView({ + behavior: "auto", + block: "nearest", + }); + } + } + }, [selectedIndex]); + const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "ArrowDown") { e.preventDefault(); @@ -255,6 +267,12 @@ const SuggestionControlInner: React.FC = ({ setQuery(tabResult); } } + } else if (e.key === "PageDown") { + e.preventDefault(); + setSelectedIndex((prev) => Math.min(prev + 10, suggestions.length - 1)); + } else if (e.key === "PageUp") { + e.preventDefault(); + setSelectedIndex((prev) => Math.max(prev - 10, 0)); } }; return ( From 65fe8b125ba47d4f1520fa193a694451cc60b924 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 11 Feb 2025 17:31:57 -0800 Subject: [PATCH 51/65] save --- cmd/wsh/cmd/wshcmd-connserver.go | 5 +- cmd/wsh/cmd/wshcmd-file-util.go | 3 +- cmd/wsh/cmd/wshcmd-file.go | 5 +- pkg/remote/fileshare/fsutil/fsutil.go | 276 ++++++++++++++++++++++++-- pkg/remote/fileshare/s3fs/s3fs.go | 7 +- pkg/remote/fileshare/wavefs/wavefs.go | 34 ++-- pkg/remote/fileshare/wshfs/wshfs.go | 4 +- pkg/util/fileutil/fileutil.go | 108 ---------- pkg/util/iochan/iochan.go | 10 +- pkg/util/utilfn/utilfn.go | 16 ++ pkg/web/web.go | 6 +- pkg/wshutil/wshrpc.go | 4 +- pkg/wshutil/wshrpcio.go | 5 +- 13 files changed, 306 insertions(+), 177 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-connserver.go b/cmd/wsh/cmd/wshcmd-connserver.go index 678ea77cc5..d9c7a8bc07 100644 --- a/cmd/wsh/cmd/wshcmd-connserver.go +++ b/cmd/wsh/cmd/wshcmd-connserver.go @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/util/packetparser" "github.com/wavetermdev/waveterm/pkg/util/sigutil" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -162,9 +163,7 @@ func serverRunRouter(jwtToken string) error { // just ignore and drain the rawCh (stdin) // when stdin is closed, shutdown defer wshutil.DoShutdown("", 0, true) - for range rawCh { - // ignore - } + utilfn.DrainChannelSafe(rawCh, "serverRunRouter:stdin") }() go func() { for msg := range termProxy.FromRemoteCh { diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 98970cf858..811a196c23 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -92,7 +93,7 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { func streamReadFromFile(ctx context.Context, fileData wshrpc.FileData, writer io.Writer) error { ch := wshclient.FileReadStreamCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) - return fileutil.ReadFileStreamToWriter(ctx, ch, writer) + return fsutil.ReadFileStreamToWriter(ctx, ch, writer) } type fileListResult struct { diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index d7e499f560..a0ca112a2a 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -550,10 +550,7 @@ func fileListRun(cmd *cobra.Command, args []string) error { filesChan := wshclient.FileListStreamCommand(RpcClient, wshrpc.FileListData{Path: path, Opts: &wshrpc.FileListOpts{All: recursive}}, &wshrpc.RpcOpts{Timeout: 2000}) // Drain the channel when done - defer func() { - for range filesChan { - } - }() + defer utilfn.DrainChannelSafe(filesChan, "fileListRun") if longForm { return filePrintLong(filesChan) } diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index 4087036850..80d1166e43 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -1,10 +1,20 @@ package fsutil import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "io/fs" "regexp" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) var slashRe = regexp.MustCompile(`/`) @@ -15,20 +25,20 @@ func GetParentPath(conn *connparse.Connection) string { } func GetParentPathString(hostAndPath string) string { - if hostAndPath == "" || hostAndPath == "/" { - return "/" - } - - // Remove trailing slash if present - if strings.HasSuffix(hostAndPath, "/") { - hostAndPath = hostAndPath[:len(hostAndPath)-1] - } - - lastSlash := strings.LastIndex(hostAndPath, "/") - if lastSlash <= 0 { - return "/" - } - return hostAndPath[:lastSlash+1] + if hostAndPath == "" || hostAndPath == "/" { + return "/" + } + + // Remove trailing slash if present + if strings.HasSuffix(hostAndPath, "/") { + hostAndPath = hostAndPath[:len(hostAndPath)-1] + } + + lastSlash := strings.LastIndex(hostAndPath, "/") + if lastSlash <= 0 { + return "/" + } + return hostAndPath[:lastSlash+1] } const minURILength = 10 // Minimum length for a valid URI (e.g., "s3://bucket") @@ -45,3 +55,241 @@ func GetPathPrefix(conn *connparse.Connection) string { } return pathPrefix } + +/* +if srcFileStat.IsDir() { + srcPathPrefix := filepath.Dir(srcPathCleaned) + if strings.HasSuffix(srcUri, "/") { + srcPathPrefix = srcPathCleaned + } + err = filepath.Walk(srcPathCleaned, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + srcFilePath := path + destFilePath := filepath.Join(destPathCleaned, strings.TrimPrefix(path, srcPathPrefix)) + var file *os.File + if !info.IsDir() { + file, err = os.Open(srcFilePath) + if err != nil { + return fmt.Errorf("cannot open file %q: %w", srcFilePath, err) + } + defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcFilePath) + } + _, err = copyFileFunc(destFilePath, info, file) + return err + }) + if err != nil { + return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) + } + } else { + file, err := os.Open(srcPathCleaned) + if err != nil { + return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) + } + defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcPathCleaned) + var destFilePath string + if destHasSlash { + destFilePath = filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) + } else { + destFilePath = destPathCleaned + } + _, err = copyFileFunc(destFilePath, srcFileStat, file) + if err != nil { + return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) + } + } +*/ + +type CopyFunc func(ctx context.Context, srcPath, destPath string) error +type ListEntriesPrefix func(ctx context.Context, prefix string) ([]string, error) + +func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix ListEntriesPrefix, copyFunc CopyFunc) error { + merge := opts != nil && opts.Merge + overwrite := opts != nil && opts.Overwrite + srcHasSlash := strings.HasSuffix(srcConn.Path, "/") + srcFileName, err := cleanPathPrefix(srcConn.Path) + if err != nil { + return fmt.Errorf("error cleaning source path: %w", err) + } + destHasSlash := strings.HasSuffix(destConn.Path, "/") + destFileName, err := cleanPathPrefix(destConn.Path) + if err != nil { + return fmt.Errorf("error cleaning destination path: %w", err) + } + destInfo, err := c.Stat(ctx, destConn) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting destination file info: %w", err) + } + destEntries := make(map[string]any) + if destInfo != nil { + if destInfo.IsDir { + if !overwrite { + return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) + } + err = c.Delete(ctx, destConn, false) + if err != nil { + return fmt.Errorf("error deleting conflicting destination file: %w", err) + } else { + entries, err := listEntriesPrefix(ctx, GetParentPath(destConn)) + if err != nil { + return fmt.Errorf("error listing destination directory: %w", err) + } + for _, entry := range entries { + destEntries[entry] = struct{}{} + } + } + } + } + + srcInfo, err := c.Stat(ctx, srcConn) + if err != nil { + return fmt.Errorf("error getting source file info: %w", err) + } + if srcInfo.IsDir { + entries, err := listEntriesPrefix(ctx, srcConn.Path) + if err != nil { + return fmt.Errorf("error listing source directory: %w", err) + } + // TODO: Finish implementing logic to match local copy in wshremote + for _, entry := range entries { + var destEntryPath string + if destHasSlash { + destEntryPath = destFileName + "/" + entry + } else { + + if _, ok := destEntries[entry]; ok { + if !merge { + return fmt.Errorf("destination already exists, merge not specified: %v", destConn.GetFullURI()) + } + } + } + } else { + return fmt.Errorf("copy between different hosts not supported") + } +} + +// cleanPathPrefix corrects paths for prefix filesystems (i.e. ones that don't have directories) +func cleanPathPrefix(path string) (string, error) { + if path == "" { + return "", fmt.Errorf("path is empty") + } + if strings.HasPrefix(path, "/") { + path = path[1:] + } + if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { + return "", fmt.Errorf("path cannot start with ~, ., or ..") + } + var newParts []string + for _, part := range strings.Split(path, "/") { + if part == ".." { + if len(newParts) > 0 { + newParts = newParts[:len(newParts)-1] + } + } else if part != "." { + newParts = append(newParts, part) + } + } + return strings.Join(newParts, "/"), nil +} + +func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { + var fileData *wshrpc.FileData + firstPk := true + isDir := false + drain := true + defer func() { + if drain { + utilfn.DrainChannelSafe(readCh, "ReadFileStream") + } + }() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %v", context.Cause(ctx)) + case respUnion, ok := <-readCh: + if !ok { + drain = false + return nil + } + if respUnion.Error != nil { + return respUnion.Error + } + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if resp.Info == nil { + return fmt.Errorf("stream file protocol error, first pk fileinfo is empty") + } + fileData = &resp + if fileData.Info.IsDir { + isDir = true + } + fileInfoCallback(*fileData.Info) + continue + } + if isDir { + if len(resp.Entries) == 0 { + continue + } + if resp.Data64 != "" { + return fmt.Errorf("stream file protocol error, directory entry has data") + } + if err := dirCallback(resp.Entries); err != nil { + return err + } + } else { + if resp.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + if err := fileCallback(decoder); err != nil { + return err + } + } + } + } +} + +func ReadStreamToFileData(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData]) (*wshrpc.FileData, error) { + var fileData *wshrpc.FileData + var dataBuf bytes.Buffer + var entries []*wshrpc.FileInfo + err := ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + fileData = &wshrpc.FileData{ + Info: &finfo, + } + }, func(fileEntries []*wshrpc.FileInfo) error { + entries = append(entries, fileEntries...) + return nil + }, func(data io.Reader) error { + if _, err := io.Copy(&dataBuf, data); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + if fileData == nil { + return nil, fmt.Errorf("stream file protocol error, no file info") + } + if !fileData.Info.IsDir { + fileData.Data64 = base64.StdEncoding.EncodeToString(dataBuf.Bytes()) + } else { + fileData.Entries = entries + } + return fileData, nil +} + +func ReadFileStreamToWriter(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], writer io.Writer) error { + return ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { + }, func(entries []*wshrpc.FileInfo) error { + return nil + }, func(data io.Reader) error { + _, err := io.Copy(writer, data) + return err + }) +} diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index ebad01b178..57a09d9bd5 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -49,7 +49,7 @@ func NewS3Client(config *aws.Config) *S3Client { func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { rtnCh := c.ReadStream(ctx, conn, data) - return fileutil.ReadStreamToFileData(ctx, rtnCh) + return fsutil.ReadStreamToFileData(ctx, rtnCh) } func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { @@ -68,10 +68,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da if finfo.IsDir { listEntriesCh := c.ListEntriesStream(ctx, conn, nil) defer func() { - go func() { - for range listEntriesCh { - } - }() + utilfn.DrainChannelSafe(listEntriesCh, "s3fs.ReadStream") }() for respUnion := range listEntriesCh { if respUnion.Error != nil { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index f22ce7610d..08938479db 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -432,44 +432,36 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar } func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - if srcConn.Host == destConn.Host { - host := srcConn.Host - srcFileName, err := cleanPath(srcConn.Path) - if err != nil { - return fmt.Errorf("error cleaning source path: %w", err) - } - destFileName, err := cleanPath(destConn.Path) - if err != nil { - return fmt.Errorf("error cleaning destination path: %w", err) - } - err = filestore.WFS.MakeFile(ctx, host, destFileName, wshrpc.FileMeta{}, wshrpc.FileOpts{}) - if err != nil { - return fmt.Errorf("error making source blockfile: %w", err) - } - _, dataBuf, err := filestore.WFS.ReadFile(ctx, host, srcFileName) + return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, srcPath, destPath string) error { + srcHost := srcConn.Host + srcFileName := strings.TrimPrefix(srcPath, srcHost+"/") + destHost := destConn.Host + destFileName := strings.TrimPrefix(destPath, destHost+"/") + _, dataBuf, err := filestore.WFS.ReadFile(ctx, srcHost, srcFileName) if err != nil { return fmt.Errorf("error reading source blockfile: %w", err) } - err = filestore.WFS.WriteFile(ctx, host, destFileName, dataBuf) + err = filestore.WFS.WriteFile(ctx, destHost, destFileName, dataBuf) if err != nil { return fmt.Errorf("error writing to destination blockfile: %w", err) } wps.Broker.Publish(wps.WaveEvent{ Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, host).String()}, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, destHost).String()}, Data: &wps.WSFileEventData{ - ZoneId: host, + ZoneId: destHost, FileName: destFileName, FileOp: wps.FileOp_Invalidate, }, }) return nil - } else { - return fmt.Errorf("copy between different hosts not supported") - } + }) } func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { + if srcConn.Scheme == connparse.ConnectionTypeWave && destConn.Scheme == connparse.ConnectionTypeWave { + return c.CopyInternal(ctx, srcConn, destConn, opts) + } zoneId := destConn.Host if zoneId == "" { return fmt.Errorf("zoneid not found in connection") diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 9d1255a115..ae0930e864 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,7 +9,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" - "github.com/wavetermdev/waveterm/pkg/util/fileutil" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -29,7 +29,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { rtnCh := c.ReadStream(ctx, conn, data) - return fileutil.ReadStreamToFileData(ctx, rtnCh) + return fsutil.ReadStreamToFileData(ctx, rtnCh) } func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index dda8c93cf0..426fe1154e 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -4,10 +4,6 @@ package fileutil import ( - "bytes" - "context" - "encoding/base64" - "fmt" "io" "io/fs" "mime" @@ -253,107 +249,3 @@ func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { IsDirInternal: fi.IsDir, } } - -func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { - var fileData *wshrpc.FileData - firstPk := true - isDir := false - drain := true - defer func() { - if drain { - go func() { - for range readCh { - } - }() - } - }() - - for { - select { - case <-ctx.Done(): - return fmt.Errorf("context cancelled: %v", context.Cause(ctx)) - case respUnion, ok := <-readCh: - if !ok { - drain = false - return nil - } - if respUnion.Error != nil { - return respUnion.Error - } - resp := respUnion.Response - if firstPk { - firstPk = false - // first packet has the fileinfo - if resp.Info == nil { - return fmt.Errorf("stream file protocol error, first pk fileinfo is empty") - } - fileData = &resp - if fileData.Info.IsDir { - isDir = true - } - fileInfoCallback(*fileData.Info) - continue - } - if isDir { - if len(resp.Entries) == 0 { - continue - } - if resp.Data64 != "" { - return fmt.Errorf("stream file protocol error, directory entry has data") - } - if err := dirCallback(resp.Entries); err != nil { - return err - } - } else { - if resp.Data64 == "" { - continue - } - decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) - if err := fileCallback(decoder); err != nil { - return err - } - } - } - } -} - -func ReadStreamToFileData(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData]) (*wshrpc.FileData, error) { - var fileData *wshrpc.FileData - var dataBuf bytes.Buffer - var entries []*wshrpc.FileInfo - err := ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { - fileData = &wshrpc.FileData{ - Info: &finfo, - } - }, func(fileEntries []*wshrpc.FileInfo) error { - entries = append(entries, fileEntries...) - return nil - }, func(data io.Reader) error { - if _, err := io.Copy(&dataBuf, data); err != nil { - return err - } - return nil - }) - if err != nil { - return nil, err - } - if fileData == nil { - return nil, fmt.Errorf("stream file protocol error, no file info") - } - if !fileData.Info.IsDir { - fileData.Data64 = base64.StdEncoding.EncodeToString(dataBuf.Bytes()) - } else { - fileData.Entries = entries - } - return fileData, nil -} - -func ReadFileStreamToWriter(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], writer io.Writer) error { - return ReadFileStream(ctx, readCh, func(finfo wshrpc.FileInfo) { - }, func(entries []*wshrpc.FileInfo) error { - return nil - }, func(data io.Reader) error { - _, err := io.Copy(writer, data) - return err - }) -} diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 8177dae73f..4bb5292cf4 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -14,6 +14,7 @@ import ( "log" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -62,7 +63,7 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn go func() { defer func() { if ctx.Err() != nil { - drainChannel(ch) + utilfn.DrainChannelSafe(ch, "WriterChan") } callback() }() @@ -99,10 +100,3 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn } }() } - -func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet]) { - go func() { - for range ch { - } - }() -} diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index 14d6204a52..f7e36a0d8e 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -1046,3 +1046,19 @@ func GracefulClose(closer io.Closer, debugName string, closerName string) bool { } return closed } + +// DrainChannelSafe will drain a channel until it is empty or until a timeout is reached. +// Warning: This function will panic if the channel is not drained within the timeout. +func DrainChannelSafe[T any](ch <-chan T, debugStr string) { + drainTimeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + go func() { + defer cancel() + for { + select { + case <-drainTimeoutCtx.Done(): + panic(debugStr + ": timeout draining channel") + case <-ch: + } + } + }() +} diff --git a/pkg/web/web.go b/pkg/web/web.go index 7450d6cb5a..1e89f4bca9 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -27,6 +27,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/schema" "github.com/wavetermdev/waveterm/pkg/service" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -258,10 +259,7 @@ func handleRemoteStreamFile(w http.ResponseWriter, req *http.Request, conn strin return } // if loop didn't finish naturally clear it out - go func() { - for range rtnCh { - } - }() + utilfn.DrainChannelSafe(rtnCh, "handleRemoteStreamFile") }() ctx := req.Context() for { diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 400d0070ce..128987137e 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -730,9 +730,7 @@ func (w *WshRpc) setServerDone() { defer w.Lock.Unlock() w.ServerDone = true close(w.CtxDoneCh) - for range w.CtxDoneCh { - // drain channel - } + utilfn.DrainChannelSafe(w.InputCh, "wshrpc.setServerDone") } func (w *WshRpc) retrySendTimeout(resId string) { diff --git a/pkg/wshutil/wshrpcio.go b/pkg/wshutil/wshrpcio.go index 9aa5f1609b..7db864626b 100644 --- a/pkg/wshutil/wshrpcio.go +++ b/pkg/wshutil/wshrpcio.go @@ -25,10 +25,7 @@ func AdaptOutputChToStream(outputCh chan []byte, output io.Writer) error { drain := false defer func() { if drain { - go func() { - for range outputCh { - } - }() + utilfn.DrainChannelSafe(outputCh, "AdaptOutputChToStream") } }() for msg := range outputCh { From efab0bf75b2af65b6cc2b5ff1dbd144507394b01 Mon Sep 17 00:00:00 2001 From: sawka Date: Tue, 11 Feb 2025 17:54:20 -0800 Subject: [PATCH 52/65] fix propagation --- frontend/app/suggestion/suggestion.tsx | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/frontend/app/suggestion/suggestion.tsx b/frontend/app/suggestion/suggestion.tsx index ac2b9767cb..9a4e77ede1 100644 --- a/frontend/app/suggestion/suggestion.tsx +++ b/frontend/app/suggestion/suggestion.tsx @@ -247,19 +247,26 @@ const SuggestionControlInner: React.FC = ({ const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "ArrowDown") { e.preventDefault(); + e.stopPropagation(); setSelectedIndex((prev) => Math.min(prev + 1, suggestions.length - 1)); } else if (e.key === "ArrowUp") { e.preventDefault(); + e.stopPropagation(); setSelectedIndex((prev) => Math.max(prev - 1, 0)); - } else if (e.key === "Enter" && selectedIndex >= 0) { + } else if (e.key === "Enter") { e.preventDefault(); - onSelect(suggestions[selectedIndex], query); - onClose(); + e.stopPropagation(); + if (selectedIndex >= 0 && selectedIndex < suggestions.length) { + onSelect(suggestions[selectedIndex], query); + onClose(); + } } else if (e.key === "Escape") { e.preventDefault(); + e.stopPropagation(); onClose(); } else if (e.key === "Tab") { e.preventDefault(); + e.stopPropagation(); const suggestion = suggestions[selectedIndex]; if (suggestion != null) { const tabResult = onTab?.(suggestion, query); @@ -269,9 +276,11 @@ const SuggestionControlInner: React.FC = ({ } } else if (e.key === "PageDown") { e.preventDefault(); + e.stopPropagation(); setSelectedIndex((prev) => Math.min(prev + 10, suggestions.length - 1)); } else if (e.key === "PageUp") { e.preventDefault(); + e.stopPropagation(); setSelectedIndex((prev) => Math.max(prev - 10, 0)); } }; From c98ef362bbaac90ed235b1c64f5a72fee2e7ea4e Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 12 Feb 2025 18:59:59 -0800 Subject: [PATCH 53/65] fix branch and drain channel fn --- frontend/app/view/preview/preview.tsx | 8 +- pkg/remote/fileshare/fsutil/fsutil.go | 49 ++-- pkg/remote/fileshare/s3fs/s3fs.go | 342 +++++++++++++++----------- pkg/remote/fileshare/wavefs/wavefs.go | 44 +++- pkg/util/utilfn/utilfn.go | 11 +- 5 files changed, 286 insertions(+), 168 deletions(-) diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 12171cf393..8c331aa282 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -609,9 +609,8 @@ export class PreviewModel implements ViewModel { path: await this.formatRemoteUri(fileInfo.dir, globalStore.get), }, }); - console.log("parent file info", newFileInfo); if (newFileInfo.path != "" && newFileInfo.notfound) { - console.log("does not exist, ", newFileInfo.path); + console.log("parent does not exist, ", newFileInfo.path); this.goParentDirectory({ fileInfo: newFileInfo }); return; } @@ -686,6 +685,11 @@ export class PreviewModel implements ViewModel { } async handleOpenFile(filePath: string) { + const fileInfo = await globalStore.get(this.statFile); + this.updateOpenFileModalAndError(false); + if (fileInfo == null) { + return true; + } try { this.goHistory(filePath); refocusNode(this.blockId); diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index 80d1166e43..db5899c27c 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -8,11 +8,13 @@ import ( "fmt" "io" "io/fs" + "log" "regexp" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -105,24 +107,27 @@ type CopyFunc func(ctx context.Context, srcPath, destPath string) error type ListEntriesPrefix func(ctx context.Context, prefix string) ([]string, error) func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix ListEntriesPrefix, copyFunc CopyFunc) error { - merge := opts != nil && opts.Merge + // merge := opts != nil && opts.Merge overwrite := opts != nil && opts.Overwrite srcHasSlash := strings.HasSuffix(srcConn.Path, "/") srcFileName, err := cleanPathPrefix(srcConn.Path) if err != nil { return fmt.Errorf("error cleaning source path: %w", err) } - destHasSlash := strings.HasSuffix(destConn.Path, "/") - destFileName, err := cleanPathPrefix(destConn.Path) + // destHasSlash := strings.HasSuffix(destConn.Path, "/") + // destFileName, err := cleanPathPrefix(destConn.Path) if err != nil { return fmt.Errorf("error cleaning destination path: %w", err) } destInfo, err := c.Stat(ctx, destConn) + destExists := err == nil && !destInfo.NotFound if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("error getting destination file info: %w", err) } destEntries := make(map[string]any) - if destInfo != nil { + destParentPrefix := GetParentPath(destConn) + "/" + if destExists { + log.Printf("destInfo: %v", destInfo) if destInfo.IsDir { if !overwrite { return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) @@ -131,7 +136,7 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec if err != nil { return fmt.Errorf("error deleting conflicting destination file: %w", err) } else { - entries, err := listEntriesPrefix(ctx, GetParentPath(destConn)) + entries, err := listEntriesPrefix(ctx, destParentPrefix) if err != nil { return fmt.Errorf("error listing destination directory: %w", err) } @@ -147,26 +152,42 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec return fmt.Errorf("error getting source file info: %w", err) } if srcInfo.IsDir { - entries, err := listEntriesPrefix(ctx, srcConn.Path) + srcPathPrefix := srcFileName + if !srcHasSlash { + srcPathPrefix += "/" + } + entries, err := listEntriesPrefix(ctx, srcPathPrefix) if err != nil { return fmt.Errorf("error listing source directory: %w", err) } + tree := pathtree.NewTree(srcPathPrefix, "/") + // srcName := path.Base(srcFileName) // TODO: Finish implementing logic to match local copy in wshremote for _, entry := range entries { - var destEntryPath string - if destHasSlash { - destEntryPath = destFileName + "/" + entry - } else { + tree.Add(entry) + } + if err = tree.Walk(func(path string, numChildren int) error { + log.Printf("path: %s, numChildren: %d", path, numChildren) + /* - if _, ok := destEntries[entry]; ok { - if !merge { - return fmt.Errorf("destination already exists, merge not specified: %v", destConn.GetFullURI()) + relativePath := strings.TrimPrefix(entry, srcPathPrefix) + if !srcHasSlash { + relativePath = srcName + "/" + relativePath } - } + destPath := destParentPrefix + relativePath + if _, ok := destEntries[destPath]; ok { + if !overwrite { + return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) + } + }*/ + return nil + }); err != nil { + return fmt.Errorf("error walking source directory: %w", err) } } else { return fmt.Errorf("copy between different hosts not supported") } + return nil } // cleanPathPrefix corrects paths for prefix filesystems (i.e. ones that don't have directories) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 57a09d9bd5..d3e9977468 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -261,50 +261,44 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } + errs := make([]error, 0) // mutex to protect the tree and objMap since we're fetching objects concurrently treeMapMutex := sync.Mutex{} // wait group to await the finished fetches wg := sync.WaitGroup{} - - // Fetch all the matching objects concurrently - var output *s3.ListObjectsV2Output - objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) - for objectPaginator.HasMorePages() { - output, err = objectPaginator.NextPage(ctx) + getObjectAndFileInfo := func(obj *types.Object) { + defer wg.Done() + result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: obj.Key, + }) if err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](err) + errs = append(errs, err) return } - errs := make([]error, 0) - getObjectAndFileInfo := func(obj *types.Object) { - defer wg.Done() - result, err := c.client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: obj.Key, - }) - if err != nil { - errs = append(errs, err) - return - } - path := *obj.Key - if wholeBucket { - path = bucket + "/" + path - } - treeMapMutex.Lock() - defer treeMapMutex.Unlock() - objMap[path] = result - tree.Add(path) - } - for _, obj := range output.Contents { - wg.Add(1) - go getObjectAndFileInfo(&obj) - } - if len(errs) > 0 { - rtn <- wshutil.RespErr[iochantypes.Packet](errors.Join(errs...)) - return + path := *obj.Key + if wholeBucket { + path = bucket + "/" + path } + treeMapMutex.Lock() + defer treeMapMutex.Unlock() + objMap[path] = result + tree.Add(path) + } + + if err := c.listFilesPrefix(ctx, input, func(obj *types.Object) (bool, error) { + wg.Add(1) + go getObjectAndFileInfo(obj) + return true, nil + }); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](err) + return } wg.Wait() + if len(errs) > 0 { + rtn <- wshutil.RespErr[iochantypes.Packet](errors.Join(errs...)) + return + } } // Walk the tree and write the tar entries @@ -409,13 +403,67 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect prevUsedDirKeys := make(map[string]any) go func() { defer close(rtn) - var err error - var output *s3.ListObjectsV2Output - input := &s3.ListObjectsV2Input{ + entryMap := make(map[string]*wshrpc.FileInfo) + if err := c.listFilesPrefix(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), Prefix: aws.String(objectKeyPrefix), + }, func(obj *types.Object) (bool, error) { + if numFetched >= numToFetch { + return false, nil + } + lastModTime := int64(0) + if obj.LastModified != nil { + lastModTime = obj.LastModified.UnixMilli() + } + if obj.Key != nil && len(*obj.Key) > len(objectKeyPrefix) { + name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) + + // we're only interested in the first level of directories + if strings.Count(name, "/") > 0 { + name = strings.SplitN(name, "/", 2)[0] + path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) + if entryMap[name] == nil { + if _, ok := prevUsedDirKeys[name]; !ok { + entryMap[name] = &wshrpc.FileInfo{ + Path: path, + Name: name, + IsDir: true, + Dir: objectKeyPrefix, + ModTime: lastModTime, + Size: 0, + } + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) + + prevUsedDirKeys[name] = struct{}{} + numFetched++ + } + } else if entryMap[name].ModTime < lastModTime { + entryMap[name].ModTime = lastModTime + } + return true, nil + } + + path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) + size := int64(0) + if obj.Size != nil { + size = *obj.Size + } + entryMap[name] = &wshrpc.FileInfo{ + Name: name, + IsDir: false, + Dir: objectKeyPrefix, + Path: path, + ModTime: lastModTime, + Size: size, + } + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) + numFetched++ + } + return true, nil + }); err != nil { + rtn <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) + return } - objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) parentPath := fsutil.GetParentPath(conn) if parentPath != "" { rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: []*wshrpc.FileInfo{ @@ -429,84 +477,17 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect }, }}} } - for objectPaginator.HasMorePages() { - output, err = objectPaginator.NextPage(ctx) - if err != nil { - var noBucket *types.NoSuchBucket - if !awsconn.CheckAccessDeniedErr(&err) && errors.As(err, &noBucket) { - err = noBucket - } - rtn <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) - break - } else { - entryMap := make(map[string]*wshrpc.FileInfo, len(output.Contents)) - for _, obj := range output.Contents { - if numFetched >= numToFetch { - break - } - lastModTime := int64(0) - if obj.LastModified != nil { - lastModTime = obj.LastModified.UnixMilli() - } - if obj.Key != nil && len(*obj.Key) > len(objectKeyPrefix) { - name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) - if strings.Count(name, "/") > 0 { - name = strings.SplitN(name, "/", 2)[0] - path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) - if entryMap[name] == nil { - if _, ok := prevUsedDirKeys[name]; !ok { - entryMap[name] = &wshrpc.FileInfo{ - Path: path, - Name: name, - IsDir: true, - Dir: objectKeyPrefix, - ModTime: lastModTime, - Size: 0, - } - fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) - - prevUsedDirKeys[name] = struct{}{} - numFetched++ - } - } else if entryMap[name].ModTime < lastModTime { - entryMap[name].ModTime = lastModTime - } - continue - } - - path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) - size := int64(0) - if obj.Size != nil { - size = *obj.Size - } - entryMap[name] = &wshrpc.FileInfo{ - Name: name, - IsDir: false, - Dir: objectKeyPrefix, - Path: path, - ModTime: lastModTime, - Size: size, - } - fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) - numFetched++ - } - } - entries := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) - for _, entry := range entryMap { - entries = append(entries, entry) - if len(entries) == wshrpc.DirChunkSize { - rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} - entries = make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) - } - } - if len(entries) > 0 { - rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} - } - } - if numFetched >= numToFetch { - return + entries := make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) + for _, entry := range entryMap { + entries = append(entries, entry) + if len(entries) == wshrpc.DirChunkSize { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} + entries = make([]*wshrpc.FileInfo, 0, wshrpc.DirChunkSize) } } + if len(entries) > 0 { + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: entries}} + } }() return rtn } @@ -555,7 +536,12 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc MimeType: "directory", }, nil } else { - return nil, fmt.Errorf("bucket %v does not exist", bucketName) + return &wshrpc.FileInfo{ + Name: bucketName, + Path: bucketName, + Dir: "/", + NotFound: true, + }, nil } } result, err := c.client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ @@ -566,21 +552,45 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc }, }) if err != nil { + log.Printf("error getting object %v:%v: %v", bucketName, objectKey, err) var noKey *types.NoSuchKey var notFound *types.NotFound - if !errors.As(err, &noKey) && !errors.As(err, ¬Found) { - return nil, err - } else { + if errors.As(err, &noKey) || errors.As(err, ¬Found) { + log.Printf("object not found: %v:%v", bucketName, objectKey) + // try to list a single object to see if the prefix exists + if !strings.HasSuffix(objectKey, "/") { + objectKey += "/" + } + log.Printf("trying to list %v", objectKey) + entries, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Prefix: aws.String(objectKey), + MaxKeys: aws.Int32(1), + }) + if err == nil { + if entries.Contents != nil && len(entries.Contents) > 0 { + return &wshrpc.FileInfo{ + Name: objectKey, + Path: conn.GetPathWithHost(), + Dir: fsutil.GetParentPath(conn), + IsDir: true, + Size: 0, + Mode: fstype.DirMode, + MimeType: "directory", + }, nil + } + } else if !errors.As(err, &noKey) && !errors.As(err, ¬Found) { + return nil, err + } + return &wshrpc.FileInfo{ Name: objectKey, Path: conn.GetPathWithHost(), Dir: fsutil.GetParentPath(conn), - IsDir: true, - Size: 0, - ModTime: 0, - MimeType: "directory", + NotFound: true, }, nil } + return nil, err } size := int64(0) if result.ObjectSize != nil { @@ -670,21 +680,26 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } var entries []*wshrpc.FileInfo - _, err := c.Stat(ctx, destConn) + destinfo, err := c.Stat(ctx, destConn) + destExists := err == nil && !destinfo.NotFound if err != nil { - if errors.Is(err, fs.ErrNotExist) { - entries, err = c.ListEntries(ctx, destConn, nil) - if err != nil { - return err - } - if len(entries) > 0 && !merge { - return fmt.Errorf("more than one entry exists at prefix, merge not specified") - } - } else { + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } + + if destExists { + if !overwrite { + return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) + } + } else { + entries, err = c.ListEntries(ctx, destConn, nil) + if err != nil { return err } - } else if !overwrite { - return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) + if len(entries) > 0 && !merge { + return fmt.Errorf("more than one entry exists at prefix, merge not specified") + } } readCtx, cancel := context.WithCancelCause(ctx) @@ -725,19 +740,54 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - srcBucket := srcConn.Host - srcKey := srcConn.Path - destBucket := destConn.Host - destKey := destConn.Path - if srcBucket == "" || srcBucket == "/" || srcKey == "" || srcKey == "/" || destBucket == "" || destBucket == "/" || destKey == "" || destKey == "/" { - return errors.Join(errors.ErrUnsupported, fmt.Errorf("source and destination bucket and object key must be specified")) - } - _, err := c.client.CopyObject(ctx, &s3.CopyObjectInput{ - Bucket: aws.String(destBucket), - Key: aws.String(destKey), - CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, srcKey)), + return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, prefix string) ([]string, error) { + var entries []string + err := c.listFilesPrefix(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(srcConn.Host), + Prefix: aws.String(prefix), + }, func(obj *types.Object) (bool, error) { + entries = append(entries, *obj.Key) + return true, nil + }) + return entries, err + }, func(ctx context.Context, srcPath, destPath string) error { + srcBucket := srcConn.Host + destBucket := destConn.Host + if srcBucket == "" || srcBucket == "/" || destBucket == "" || destBucket == "/" { + return fmt.Errorf("source and destination bucket must be specified") + } + _, err := c.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destPath), + CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, srcPath)), + }) + return err }) - return err +} + +func (c S3Client) listFilesPrefix(ctx context.Context, input *s3.ListObjectsV2Input, fileCallback func(*types.Object) (bool, error)) error { + var err error + var output *s3.ListObjectsV2Output + objectPaginator := s3.NewListObjectsV2Paginator(c.client, input) + for objectPaginator.HasMorePages() { + output, err = objectPaginator.NextPage(ctx) + if err != nil { + var noBucket *types.NoSuchBucket + if !awsconn.CheckAccessDeniedErr(&err) && errors.As(err, &noBucket) { + err = noBucket + } + return err + } else { + for _, obj := range output.Contents { + if cont, err := fileCallback(&obj); err != nil { + return err + } else if !cont { + return nil + } + } + } + } + return nil } func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 08938479db..5e606f9e0e 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -290,7 +290,27 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr fileInfo, err := filestore.WFS.Stat(ctx, zoneId, fileName) if err != nil { if errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("NOTFOUND: %w", err) + // attempt to list the directory + entries, err := c.ListEntries(ctx, conn, nil) + if err != nil { + return nil, fmt.Errorf("error listing entries: %w", err) + } + if len(entries) > 0 { + return &wshrpc.FileInfo{ + Path: conn.GetPathWithHost(), + Name: fileName, + Dir: fsutil.GetParentPathString(fileName), + Size: 0, + IsDir: true, + Mode: DirMode, + }, nil + } else { + return &wshrpc.FileInfo{ + Path: conn.GetPathWithHost(), + Name: fileName, + Dir: fsutil.GetParentPathString(fileName), + NotFound: true}, nil + } } return nil, fmt.Errorf("error getting file info: %w", err) } @@ -432,7 +452,27 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar } func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, srcPath, destPath string) error { + return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, prefix string) ([]string, error) { + zoneId := srcConn.Host + if zoneId == "" { + return nil, fmt.Errorf("zoneid not found in connection") + } + fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) + if err != nil { + return nil, fmt.Errorf("error listing blockfiles: %w", err) + } + var fileList []string + for _, wf := range fileListOrig { + fileList = append(fileList, wf.Name) + } + var filteredList []string + for _, file := range fileList { + if strings.HasPrefix(file, prefix) { + filteredList = append(filteredList, file) + } + } + return filteredList, nil + }, func(ctx context.Context, srcPath, destPath string) error { srcHost := srcConn.Host srcFileName := strings.TrimPrefix(srcPath, srcHost+"/") destHost := destConn.Host diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index f7e36a0d8e..99ad83f453 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -1048,16 +1048,19 @@ func GracefulClose(closer io.Closer, debugName string, closerName string) bool { } // DrainChannelSafe will drain a channel until it is empty or until a timeout is reached. -// Warning: This function will panic if the channel is not drained within the timeout. -func DrainChannelSafe[T any](ch <-chan T, debugStr string) { +// WARNING: This function will panic if the channel is not drained within the timeout. +func DrainChannelSafe[T any](ch <-chan T, debugName string) { drainTimeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) go func() { defer cancel() for { select { case <-drainTimeoutCtx.Done(): - panic(debugStr + ": timeout draining channel") - case <-ch: + panic(debugName + ": timeout draining channel") + case _, ok := <-ch: + if !ok { + return + } } } }() From 3ce3c24d037b408c8ed71ab00fd696878544afe1 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 12 Feb 2025 19:13:45 -0800 Subject: [PATCH 54/65] more centralizing of the listEntries code --- pkg/remote/fileshare/fsutil/fsutil.go | 6 ++-- pkg/remote/fileshare/s3fs/s3fs.go | 4 +-- pkg/remote/fileshare/wavefs/wavefs.go | 49 ++++++++++++++++----------- pkg/util/tarcopy/tarcopy.go | 2 +- pkg/util/utilfn/utilfn.go | 2 +- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index db5899c27c..b55c1e49d0 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -104,7 +104,7 @@ if srcFileStat.IsDir() { */ type CopyFunc func(ctx context.Context, srcPath, destPath string) error -type ListEntriesPrefix func(ctx context.Context, prefix string) ([]string, error) +type ListEntriesPrefix func(ctx context.Context, host, prefix string) ([]string, error) func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix ListEntriesPrefix, copyFunc CopyFunc) error { // merge := opts != nil && opts.Merge @@ -136,7 +136,7 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec if err != nil { return fmt.Errorf("error deleting conflicting destination file: %w", err) } else { - entries, err := listEntriesPrefix(ctx, destParentPrefix) + entries, err := listEntriesPrefix(ctx, destConn.Host, destParentPrefix) if err != nil { return fmt.Errorf("error listing destination directory: %w", err) } @@ -156,7 +156,7 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec if !srcHasSlash { srcPathPrefix += "/" } - entries, err := listEntriesPrefix(ctx, srcPathPrefix) + entries, err := listEntriesPrefix(ctx, srcConn.Host, srcPathPrefix) if err != nil { return fmt.Errorf("error listing source directory: %w", err) } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index d3e9977468..440f76494f 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -740,10 +740,10 @@ func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.C } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, prefix string) ([]string, error) { + return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, bucket, prefix string) ([]string, error) { var entries []string err := c.listFilesPrefix(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(srcConn.Host), + Bucket: aws.String(bucket), Prefix: aws.String(prefix), }, func(obj *types.Object) (bool, error) { entries = append(entries, *obj.Key) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 5e606f9e0e..af610f1324 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -452,26 +452,15 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar } func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, prefix string) ([]string, error) { - zoneId := srcConn.Host - if zoneId == "" { - return nil, fmt.Errorf("zoneid not found in connection") - } - fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) - if err != nil { - return nil, fmt.Errorf("error listing blockfiles: %w", err) - } - var fileList []string - for _, wf := range fileListOrig { - fileList = append(fileList, wf.Name) - } - var filteredList []string - for _, file := range fileList { - if strings.HasPrefix(file, prefix) { - filteredList = append(filteredList, file) - } + return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, zoneId, prefix string) ([]string, error) { + entryList := make([]string, 0) + if err := listEntriesPrefix(ctx, zoneId, prefix, func(entry string) error { + entryList = append(entryList, entry) + return nil + }); err != nil { + return nil, err } - return filteredList, nil + return entryList, nil }, func(ctx context.Context, srcPath, destPath string) error { srcHost := srcConn.Host srcFileName := strings.TrimPrefix(srcPath, srcHost+"/") @@ -498,6 +487,28 @@ func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connpar }) } +func listEntriesPrefix(ctx context.Context, zoneId, prefix string, entryCallback func(string) error) error { + if zoneId == "" { + return fmt.Errorf("zoneid not found in connection") + } + fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) + if err != nil { + return fmt.Errorf("error listing blockfiles: %w", err) + } + var fileList []string + for _, wf := range fileListOrig { + fileList = append(fileList, wf.Name) + } + for _, file := range fileList { + if strings.HasPrefix(file, prefix) { + if err := entryCallback(file); err != nil { + return err + } + } + } + return nil +} + func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { if srcConn.Scheme == connparse.ConnectionTypeWave && destConn.Scheme == connparse.ConnectionTypeWave { return c.CopyInternal(ctx, srcConn, destConn, opts) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index dcdbbe082b..d8888719de 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -87,7 +87,7 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } } -func fixPath(path string, prefix string) (string, error) { +func fixPath(path, prefix string) (string, error) { path = strings.TrimPrefix(strings.TrimPrefix(filepath.Clean(strings.TrimPrefix(path, prefix)), "/"), "\\") if strings.Contains(path, "..") { return "", fmt.Errorf("invalid tar path containing directory traversal: %s", path) diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index 99ad83f453..73535deb4d 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -1030,7 +1030,7 @@ const ( retryDelay = 10 * time.Millisecond ) -func GracefulClose(closer io.Closer, debugName string, closerName string) bool { +func GracefulClose(closer io.Closer, debugName, closerName string) bool { closed := false for retries := 0; retries < maxRetries; retries++ { if err := closer.Close(); err != nil { From 61113608779e49cdb657e157350e34d0c07cddc5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 12 Feb 2025 19:19:45 -0800 Subject: [PATCH 55/65] more notfound fixes --- pkg/remote/fileshare/wavefs/wavefs.go | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index af610f1324..6cbb331d3e 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -119,9 +119,13 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio } finfo, err := c.Stat(ctx, conn) - if err != nil && !errors.Is(err, fs.ErrNotExist) { + exists := err == nil && !finfo.NotFound + if err != nil { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error getting file info: %w", err)) } + if !exists { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("file not found: %s", conn.GetFullURI())) + } singleFile := finfo != nil && !finfo.IsDir var pathPrefix string if !singleFile && srcHasSlash { @@ -525,21 +529,25 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) var entries []*wshrpc.FileInfo - _, err := c.Stat(ctx, destConn) + destInfo, err := c.Stat(ctx, destConn) + destExists := err == nil && !destInfo.NotFound if err != nil { - if errors.Is(err, fs.ErrNotExist) { - entries, err = c.ListEntries(ctx, destConn, nil) - if err != nil { - return err - } - if len(entries) > 0 && !merge { - return fmt.Errorf("more than one entry exists at prefix, merge not specified") + return err + } + + if destExists { + if destInfo.IsDir { + if !merge { + return fmt.Errorf("destination already exists, merge not specified: %v", destConn.GetFullURI()) } + } else if !overwrite { + return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) } else { - return err + err = c.Delete(ctx, destConn, false) + if err != nil { + return fmt.Errorf("error deleting conflicting destination file: %w", err) + } } - } else if !overwrite { - return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) } readCtx, cancel := context.WithCancelCause(ctx) From 69577d2a77f59fe0c1429923362e3d5ce78d3354 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 13 Feb 2025 13:22:52 -0800 Subject: [PATCH 56/65] save --- pkg/remote/fileshare/fspath/fspath.go | 28 +++ pkg/remote/fileshare/fsutil/fsutil.go | 243 +++++++++++++++----------- pkg/remote/fileshare/s3fs/s3fs.go | 174 +++++------------- pkg/remote/fileshare/wavefs/wavefs.go | 113 ++++-------- 4 files changed, 244 insertions(+), 314 deletions(-) create mode 100644 pkg/remote/fileshare/fspath/fspath.go diff --git a/pkg/remote/fileshare/fspath/fspath.go b/pkg/remote/fileshare/fspath/fspath.go new file mode 100644 index 0000000000..88adbf91fe --- /dev/null +++ b/pkg/remote/fileshare/fspath/fspath.go @@ -0,0 +1,28 @@ +package fspath + +import ( + pathpkg "path" + "strings" +) + +const ( + // Separator is the path separator + Separator = "/" +) + +func Dir(path string) string { + return pathpkg.Dir(ToSlash(path)) +} + +func Base(path string) string { + return pathpkg.Base(ToSlash(path)) +} + +func Join(elem ...string) string { + joined := pathpkg.Join(elem...) + return ToSlash(joined) +} + +func ToSlash(path string) string { + return strings.ReplaceAll(path, "\\", Separator) +} diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index b55c1e49d0..eb53173845 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -1,6 +1,7 @@ package fsutil import ( + "archive/tar" "bytes" "context" "encoding/base64" @@ -9,36 +10,35 @@ import ( "io" "io/fs" "log" - "regexp" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fspath" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" + "github.com/wavetermdev/waveterm/pkg/util/tarcopy" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -var slashRe = regexp.MustCompile(`/`) - func GetParentPath(conn *connparse.Connection) string { hostAndPath := conn.GetPathWithHost() return GetParentPathString(hostAndPath) } func GetParentPathString(hostAndPath string) string { - if hostAndPath == "" || hostAndPath == "/" { - return "/" + if hostAndPath == "" || hostAndPath == fspath.Separator { + return fspath.Separator } // Remove trailing slash if present - if strings.HasSuffix(hostAndPath, "/") { + if strings.HasSuffix(hostAndPath, fspath.Separator) { hostAndPath = hostAndPath[:len(hostAndPath)-1] } - lastSlash := strings.LastIndex(hostAndPath, "/") + lastSlash := strings.LastIndex(hostAndPath, fspath.Separator) if lastSlash <= 0 { - return "/" + return fspath.Separator } return hostAndPath[:lastSlash+1] } @@ -51,141 +51,172 @@ func GetPathPrefix(conn *connparse.Connection) string { return "" } pathPrefix := fullUri - lastSlash := strings.LastIndex(fullUri, "/") + lastSlash := strings.LastIndex(fullUri, fspath.Separator) if lastSlash > minURILength && lastSlash < len(fullUri)-1 { pathPrefix = fullUri[:lastSlash+1] } return pathPrefix } -/* -if srcFileStat.IsDir() { - srcPathPrefix := filepath.Dir(srcPathCleaned) - if strings.HasSuffix(srcUri, "/") { - srcPathPrefix = srcPathCleaned - } - err = filepath.Walk(srcPathCleaned, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - srcFilePath := path - destFilePath := filepath.Join(destPathCleaned, strings.TrimPrefix(path, srcPathPrefix)) - var file *os.File - if !info.IsDir() { - file, err = os.Open(srcFilePath) - if err != nil { - return fmt.Errorf("cannot open file %q: %w", srcFilePath, err) - } - defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcFilePath) - } - _, err = copyFileFunc(destFilePath, info, file) - return err - }) - if err != nil { - return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) - } - } else { - file, err := os.Open(srcPathCleaned) - if err != nil { - return fmt.Errorf("cannot open file %q: %w", srcPathCleaned, err) - } - defer utilfn.GracefulClose(file, "RemoteFileCopyCommand", srcPathCleaned) - var destFilePath string - if destHasSlash { - destFilePath = filepath.Join(destPathCleaned, filepath.Base(srcPathCleaned)) - } else { - destFilePath = destPathCleaned - } - _, err = copyFileFunc(destFilePath, srcFileStat, file) - if err != nil { - return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) - } - } -*/ - type CopyFunc func(ctx context.Context, srcPath, destPath string) error type ListEntriesPrefix func(ctx context.Context, host, prefix string) ([]string, error) func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix ListEntriesPrefix, copyFunc CopyFunc) error { - // merge := opts != nil && opts.Merge + log.Printf("PrefixCopyInternal: %v -> %v", srcConn.GetFullURI(), destConn.GetFullURI()) + merge := opts != nil && opts.Merge overwrite := opts != nil && opts.Overwrite - srcHasSlash := strings.HasSuffix(srcConn.Path, "/") - srcFileName, err := cleanPathPrefix(srcConn.Path) + if overwrite && merge { + return fmt.Errorf("cannot specify both overwrite and merge") + } + srcHasSlash := strings.HasSuffix(srcConn.Path, fspath.Separator) + srcPath, err := cleanPathPrefix(srcConn.Path) if err != nil { return fmt.Errorf("error cleaning source path: %w", err) } - // destHasSlash := strings.HasSuffix(destConn.Path, "/") - // destFileName, err := cleanPathPrefix(destConn.Path) + destHasSlash := strings.HasSuffix(destConn.Path, fspath.Separator) + destPath, err := cleanPathPrefix(destConn.Path) if err != nil { return fmt.Errorf("error cleaning destination path: %w", err) } + if !srcHasSlash { + if !destHasSlash { + destPath += fspath.Separator + } + destPath += fspath.Base(srcPath) + } + destConn.Path = destPath destInfo, err := c.Stat(ctx, destConn) destExists := err == nil && !destInfo.NotFound if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("error getting destination file info: %w", err) } - destEntries := make(map[string]any) - destParentPrefix := GetParentPath(destConn) + "/" + + srcInfo, err := c.Stat(ctx, srcConn) + if err != nil { + return fmt.Errorf("error getting source file info: %w", err) + } if destExists { - log.Printf("destInfo: %v", destInfo) - if destInfo.IsDir { - if !overwrite { - return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) - } - err = c.Delete(ctx, destConn, false) + if overwrite { + err = c.Delete(ctx, destConn, true) if err != nil { return fmt.Errorf("error deleting conflicting destination file: %w", err) - } else { - entries, err := listEntriesPrefix(ctx, destConn.Host, destParentPrefix) - if err != nil { - return fmt.Errorf("error listing destination directory: %w", err) - } - for _, entry := range entries { - destEntries[entry] = struct{}{} - } } + } else if destInfo.IsDir && srcInfo.IsDir { + if !merge { + return fmt.Errorf("destination and source are both directories, neither merge nor overwrite specified: %v", destConn.GetFullURI()) + } + } else { + return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) } } - - srcInfo, err := c.Stat(ctx, srcConn) - if err != nil { - return fmt.Errorf("error getting source file info: %w", err) - } if srcInfo.IsDir { - srcPathPrefix := srcFileName if !srcHasSlash { - srcPathPrefix += "/" + srcPath += fspath.Separator } - entries, err := listEntriesPrefix(ctx, srcConn.Host, srcPathPrefix) + destPath += fspath.Separator + log.Printf("Copying directory: %v -> %v", srcPath, destPath) + entries, err := listEntriesPrefix(ctx, srcConn.Host, srcPath) if err != nil { return fmt.Errorf("error listing source directory: %w", err) } - tree := pathtree.NewTree(srcPathPrefix, "/") - // srcName := path.Base(srcFileName) - // TODO: Finish implementing logic to match local copy in wshremote + + tree := pathtree.NewTree(srcPath, fspath.Separator) for _, entry := range entries { tree.Add(entry) } - if err = tree.Walk(func(path string, numChildren int) error { - log.Printf("path: %s, numChildren: %d", path, numChildren) - /* - relativePath := strings.TrimPrefix(entry, srcPathPrefix) - if !srcHasSlash { - relativePath = srcName + "/" + relativePath - } - destPath := destParentPrefix + relativePath - if _, ok := destEntries[destPath]; ok { - if !overwrite { - return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) - } - }*/ - return nil - }); err != nil { - return fmt.Errorf("error walking source directory: %w", err) + /* tree.Walk will return the full path in the source bucket for each item. + prefixToRemove specifies how much of that path we want in the destination subtree. + If the source path has a trailing slash, we don't want to include the source directory itself in the destination subtree.*/ + prefixToRemove := srcPath + if !srcHasSlash { + prefixToRemove = fspath.Dir(srcPath) + fspath.Separator } + return tree.Walk(func(path string, numChildren int) error { + // since this is a prefix filesystem, we only care about leafs + if numChildren > 0 { + return nil + } + destFilePath := destPath + strings.TrimPrefix(path, prefixToRemove) + return copyFunc(ctx, path, destFilePath) + }) } else { - return fmt.Errorf("copy between different hosts not supported") + return copyFunc(ctx, srcPath, destPath) + } +} + +func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient, destClient fstype.FileShareClient, destPutFile func(host, path string, size int64, reader io.Reader) error, opts *wshrpc.FileCopyOpts) error { + merge := opts != nil && opts.Merge + overwrite := opts != nil && opts.Overwrite + if overwrite && merge { + return fmt.Errorf("cannot specify both overwrite and merge") + } + srcHasSlash := strings.HasSuffix(srcConn.Path, fspath.Separator) + destHasSlash := strings.HasSuffix(destConn.Path, fspath.Separator) + destPath, err := cleanPathPrefix(destConn.Path) + if err != nil { + return fmt.Errorf("error cleaning destination path: %w", err) + } + if !srcHasSlash { + if !destHasSlash { + destPath += fspath.Separator + } + destPath += fspath.Base(srcConn.Path) + } + destConn.Path = destPath + destInfo, err := destClient.Stat(ctx, destConn) + destExists := err == nil && !destInfo.NotFound + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting destination file info: %w", err) + } + + srcInfo, err := srcClient.Stat(ctx, srcConn) + if err != nil { + return fmt.Errorf("error getting source file info: %w", err) + } + if destExists { + if overwrite { + err = destClient.Delete(ctx, destConn, true) + if err != nil { + return fmt.Errorf("error deleting conflicting destination file: %w", err) + } + } else if destInfo.IsDir && srcInfo.IsDir { + if !merge { + return fmt.Errorf("destination and source are both directories, neither merge nor overwrite specified: %v", destConn.GetFullURI()) + } + } else { + return fmt.Errorf("destination already exists, overwrite not specified: %v", destConn.GetFullURI()) + } + } + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } + log.Printf("Copying: %v -> %v", srcConn.GetFullURI(), destConn.GetFullURI()) + readCtx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) + err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { + if next.Typeflag == tar.TypeDir { + return nil + } + if singleFile && srcInfo.IsDir { + return fmt.Errorf("protocol error: source is a directory, but only a single file is being copied") + } + fileName, err := cleanPathPrefix(fspath.Join(destPath, next.Name)) + if singleFile && !destHasSlash { + fileName, err = cleanPathPrefix(destConn.Path) + } + if err != nil { + return fmt.Errorf("error cleaning path: %w", err) + } + log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) + return destPutFile(destConn.Host, fileName, next.Size, reader) + }) + if err != nil { + cancel(err) + return err } return nil } @@ -195,14 +226,14 @@ func cleanPathPrefix(path string) (string, error) { if path == "" { return "", fmt.Errorf("path is empty") } - if strings.HasPrefix(path, "/") { + if strings.HasPrefix(path, fspath.Separator) { path = path[1:] } if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { return "", fmt.Errorf("path cannot start with ~, ., or ..") } var newParts []string - for _, part := range strings.Split(path, "/") { + for _, part := range strings.Split(path, fspath.Separator) { if part == ".." { if len(newParts) > 0 { newParts = newParts[:len(newParts)-1] @@ -211,7 +242,7 @@ func cleanPathPrefix(path string) (string, error) { newParts = append(newParts, part) } } - return strings.Join(newParts, "/"), nil + return fspath.Join(newParts...), nil } func ReadFileStream(ctx context.Context, readCh <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData], fileInfoCallback func(finfo wshrpc.FileInfo), dirCallback func(entries []*wshrpc.FileInfo) error, fileCallback func(data io.Reader) error) error { diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 440f76494f..2a558e699c 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -4,16 +4,13 @@ package s3fs import ( - "archive/tar" "bytes" "context" "encoding/base64" "errors" "fmt" "io" - "io/fs" "log" - "path" "strings" "sync" "time" @@ -24,6 +21,7 @@ import ( "github.com/aws/smithy-go" "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fspath" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/pathtree" @@ -164,7 +162,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } // whether the operation is on the whole bucket - wholeBucket := conn.Path == "" || conn.Path == "/" + wholeBucket := conn.Path == "" || conn.Path == fspath.Separator // get the object if it's a single file operation var singleFileResult *s3.GetObjectOutput @@ -196,7 +194,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, singleFile := singleFileResult != nil // whether to include the directory itself in the tar - includeDir := (wholeBucket && conn.Path == "") || (singleFileResult == nil && conn.Path != "" && !strings.HasSuffix(conn.Path, "/")) + includeDir := (wholeBucket && conn.Path == "") || (singleFileResult == nil && conn.Path != "" && !strings.HasSuffix(conn.Path, fspath.Separator)) timeout := fstype.DefaultTimeout if opts.Timeout > 0 { @@ -252,8 +250,8 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } } else { objectPrefix := conn.Path - if !strings.HasSuffix(objectPrefix, "/") { - objectPrefix = objectPrefix + "/" + if !strings.HasSuffix(objectPrefix, fspath.Separator) { + objectPrefix = objectPrefix + fspath.Separator } input = &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), @@ -278,7 +276,7 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, } path := *obj.Key if wholeBucket { - path = bucket + "/" + path + path = fspath.Join(bucket, path) } treeMapMutex.Lock() defer treeMapMutex.Unlock() @@ -363,7 +361,7 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { bucket := conn.Host objectKeyPrefix := conn.Path - if objectKeyPrefix != "" && !strings.HasSuffix(objectKeyPrefix, "/") { + if objectKeyPrefix != "" && !strings.HasSuffix(objectKeyPrefix, fspath.Separator) { objectKeyPrefix = objectKeyPrefix + "/" } numToFetch := wshrpc.MaxDirSize @@ -371,7 +369,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect numToFetch = min(opts.Limit, wshrpc.MaxDirSize) } numFetched := 0 - if bucket == "" || bucket == "/" { + if bucket == "" || bucket == fspath.Separator { buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](err) @@ -385,7 +383,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect entries = append(entries, &wshrpc.FileInfo{ Path: *bucket.Name, Name: *bucket.Name, - Dir: "/", + Dir: fspath.Separator, ModTime: bucket.CreationDate.UnixMilli(), IsDir: true, MimeType: "directory", @@ -419,9 +417,9 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) // we're only interested in the first level of directories - if strings.Count(name, "/") > 0 { - name = strings.SplitN(name, "/", 2)[0] - path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) + if strings.Count(name, fspath.Separator) > 0 { + name = strings.SplitN(name, fspath.Separator, 2)[0] + path := fspath.Join(conn.GetPathWithHost(), name) if entryMap[name] == nil { if _, ok := prevUsedDirKeys[name]; !ok { entryMap[name] = &wshrpc.FileInfo{ @@ -443,7 +441,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect return true, nil } - path := fmt.Sprintf("%s/%s", conn.GetPathWithHost(), name) + path := fspath.Join(conn.GetPathWithHost(), name) size := int64(0) if obj.Size != nil { size = *obj.Size @@ -473,6 +471,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect Name: "..", IsDir: true, Size: 0, + ModTime: time.Now().Unix(), MimeType: "directory", }, }}} @@ -494,21 +493,22 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect } func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { + log.Printf("Stat: %v", conn.GetFullURI()) bucketName := conn.Host objectKey := conn.Path - if bucketName == "" || bucketName == "/" { + if bucketName == "" || bucketName == fspath.Separator { // root, refers to list all buckets return &wshrpc.FileInfo{ - Name: "/", + Name: fspath.Separator, IsDir: true, Size: 0, ModTime: 0, - Path: "/", - Dir: "/", + Path: fspath.Separator, + Dir: fspath.Separator, MimeType: "directory", }, nil } - if objectKey == "" || objectKey == "/" { + if objectKey == "" || objectKey == fspath.Separator { _, err := c.client.HeadBucket(ctx, &s3.HeadBucketInput{ Bucket: aws.String(bucketName), }) @@ -519,7 +519,6 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc switch apiError.(type) { case *types.NotFound: exists = false - err = nil default: } } @@ -529,7 +528,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: bucketName, Path: bucketName, - Dir: "/", + Dir: fspath.Separator, IsDir: true, Size: 0, ModTime: 0, @@ -539,7 +538,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return &wshrpc.FileInfo{ Name: bucketName, Path: bucketName, - Dir: "/", + Dir: fspath.Separator, NotFound: true, }, nil } @@ -552,16 +551,13 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc }, }) if err != nil { - log.Printf("error getting object %v:%v: %v", bucketName, objectKey, err) var noKey *types.NoSuchKey var notFound *types.NotFound if errors.As(err, &noKey) || errors.As(err, ¬Found) { - log.Printf("object not found: %v:%v", bucketName, objectKey) // try to list a single object to see if the prefix exists - if !strings.HasSuffix(objectKey, "/") { - objectKey += "/" + if !strings.HasSuffix(objectKey, fspath.Separator) { + objectKey += fspath.Separator } - log.Printf("trying to list %v", objectKey) entries, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(bucketName), Prefix: aws.String(objectKey), @@ -669,77 +665,30 @@ func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse } func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { + if srcConn.Scheme == connparse.ConnectionTypeS3 && destConn.Scheme == connparse.ConnectionTypeS3 { + return c.CopyInternal(ctx, srcConn, destConn, opts) + } destBucket := destConn.Host - overwrite := opts != nil && opts.Overwrite - merge := opts != nil && opts.Merge - destHasSlash := strings.HasSuffix(destConn.Path, "/") - destPrefix := fsutil.GetPathPrefix(destConn) - destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") - if destBucket == "" || destBucket == "/" { + if destBucket == "" || destBucket == fspath.Separator { return fmt.Errorf("destination bucket must be specified") } - - var entries []*wshrpc.FileInfo - destinfo, err := c.Stat(ctx, destConn) - destExists := err == nil && !destinfo.NotFound - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return err - } - } - - if destExists { - if !overwrite { - return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) - } - } else { - entries, err = c.ListEntries(ctx, destConn, nil) - if err != nil { - return err - } - if len(entries) > 0 && !merge { - return fmt.Errorf("more than one entry exists at prefix, merge not specified") - } - } - - readCtx, cancel := context.WithCancelCause(ctx) - defer cancel(nil) - ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { - if next.Typeflag == tar.TypeDir { - return nil - } - fileName, err := cleanPath(path.Join(destPrefix, next.Name)) - if singleFile && !destHasSlash { - fileName, err = cleanPath(destConn.Path) - } - if err != nil { - return fmt.Errorf("error cleaning path: %w", err) - } - if !overwrite { - for _, entry := range entries { - if entry.Name == fileName { - return fmt.Errorf("destination already exists: %v", fileName) - } - } - } - log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) - _, err = c.client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(destBucket), - Key: aws.String(fileName), + return fsutil.PrefixCopyRemote(ctx, srcConn, destConn, srcClient, c, func(bucket, path string, size int64, reader io.Reader) error { + _, err := c.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(path), Body: reader, - ContentLength: aws.Int64(next.Size), + ContentLength: aws.Int64(size), }) return err - }) - if err != nil { - cancel(err) - return err - } - return nil + }, opts) } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { + srcBucket := srcConn.Host + destBucket := destConn.Host + if srcBucket == "" || srcBucket == fspath.Separator || destBucket == "" || destBucket == fspath.Separator { + return fmt.Errorf("source and destination bucket must be specified") + } return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, bucket, prefix string) ([]string, error) { var entries []string err := c.listFilesPrefix(ctx, &s3.ListObjectsV2Input{ @@ -751,15 +700,11 @@ func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse }) return entries, err }, func(ctx context.Context, srcPath, destPath string) error { - srcBucket := srcConn.Host - destBucket := destConn.Host - if srcBucket == "" || srcBucket == "/" || destBucket == "" || destBucket == "/" { - return fmt.Errorf("source and destination bucket must be specified") - } + log.Printf("Copying file %v -> %v", srcBucket+"/"+srcPath, destBucket+"/"+destPath) _, err := c.client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: aws.String(destBucket), Key: aws.String(destPath), - CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, srcPath)), + CopySource: aws.String(fspath.Join(srcBucket, srcPath)), }) return err }) @@ -793,15 +738,15 @@ func (c S3Client) listFilesPrefix(ctx context.Context, input *s3.ListObjectsV2In func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { bucket := conn.Host objectKey := conn.Path - if bucket == "" || bucket == "/" { + if bucket == "" || bucket == fspath.Separator { return errors.Join(errors.ErrUnsupported, fmt.Errorf("bucket must be specified")) } - if objectKey == "" || objectKey == "/" { + if objectKey == "" || objectKey == fspath.Separator { return errors.Join(errors.ErrUnsupported, fmt.Errorf("object key must be specified")) } if recursive { - if !strings.HasSuffix(objectKey, "/") { - objectKey = objectKey + "/" + if !strings.HasSuffix(objectKey, fspath.Separator) { + objectKey = objectKey + fspath.Separator } entries, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), @@ -834,8 +779,8 @@ func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recurs func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { var joinParts []string - if conn.Host == "" || conn.Host == "/" { - if conn.Path == "" || conn.Path == "/" { + if conn.Host == "" || conn.Host == fspath.Separator { + if conn.Path == "" || conn.Path == fspath.Separator { joinParts = parts } else { joinParts = append([]string{conn.Path}, parts...) @@ -846,7 +791,7 @@ func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts .. joinParts = append([]string{conn.Host, conn.Path}, parts...) } - conn.Path = strings.Join(joinParts, "/") + conn.Path = fspath.Join(joinParts...) return c.Stat(ctx, conn) } @@ -861,26 +806,3 @@ func (c S3Client) GetCapability() wshrpc.FileShareCapability { CanMkdir: false, } } - -func cleanPath(path string) (string, error) { - if path == "" { - return "", fmt.Errorf("path is empty") - } - if strings.HasPrefix(path, "/") { - path = path[1:] - } - if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { - return "", fmt.Errorf("s3 path cannot start with ~, ., or ..") - } - var newParts []string - for _, part := range strings.Split(path, "/") { - if part == ".." { - if len(newParts) > 0 { - newParts = newParts[:len(newParts)-1] - } - } else if part != "." { - newParts = append(newParts, part) - } - } - return strings.Join(newParts, "/"), nil -} diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 6cbb331d3e..50555a1f52 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -4,7 +4,6 @@ package wavefs import ( - "archive/tar" "context" "encoding/base64" "errors" @@ -13,13 +12,13 @@ import ( "io/fs" "log" "os" - "path" "path/filepath" "strings" "time" "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fspath" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fsutil" "github.com/wavetermdev/waveterm/pkg/util/fileutil" @@ -221,6 +220,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, if err != nil { return nil, fmt.Errorf("error cleaning path: %w", err) } + prefix += fspath.Separator fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) if err != nil { return nil, fmt.Errorf("error listing blockfiles: %w", err) @@ -246,8 +246,8 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, // first strip the prefix relPath := strings.TrimPrefix(file.Name, prefix) // then check if there is a "/" after the prefix - if strings.Contains(relPath, "/") { - dirPath := strings.Split(relPath, "/")[0] + if strings.Contains(relPath, fspath.Separator) { + dirPath := strings.Split(relPath, fspath.Separator)[0] dirMap[dirPath] = struct{}{} continue } @@ -256,7 +256,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, for dir := range dirMap { dirName := prefix + dir filteredList = append(filteredList, &wshrpc.FileInfo{ - Path: strings.Join([]string{conn.GetPathWithHost(), dirName}, "/"), + Path: fspath.Join(conn.GetPathWithHost(), dirName), Name: dirName, Dir: fsutil.GetParentPathString(dirName), Size: 0, @@ -467,9 +467,9 @@ func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connpar return entryList, nil }, func(ctx context.Context, srcPath, destPath string) error { srcHost := srcConn.Host - srcFileName := strings.TrimPrefix(srcPath, srcHost+"/") + srcFileName := strings.TrimPrefix(srcPath, srcHost+fspath.Separator) destHost := destConn.Host - destFileName := strings.TrimPrefix(destPath, destHost+"/") + destFileName := strings.TrimPrefix(destPath, destHost+fspath.Separator) _, dataBuf, err := filestore.WFS.ReadFile(ctx, srcHost, srcFileName) if err != nil { return fmt.Errorf("error reading source blockfile: %w", err) @@ -521,64 +521,27 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - overwrite := opts != nil && opts.Overwrite - merge := opts != nil && opts.Merge - destHasSlash := strings.HasSuffix(destConn.Path, "/") - destPrefix := getPathPrefix(destConn) - destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") - log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) - - var entries []*wshrpc.FileInfo - destInfo, err := c.Stat(ctx, destConn) - destExists := err == nil && !destInfo.NotFound - if err != nil { - return err - } - - if destExists { - if destInfo.IsDir { - if !merge { - return fmt.Errorf("destination already exists, merge not specified: %v", destConn.GetFullURI()) - } - } else if !overwrite { - return fmt.Errorf("file already exists at destination %q, use force to overwrite", destConn.GetFullURI()) - } else { - err = c.Delete(ctx, destConn, false) - if err != nil { - return fmt.Errorf("error deleting conflicting destination file: %w", err) - } - } - } - - readCtx, cancel := context.WithCancelCause(ctx) - ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) - err = tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader, singleFile bool) error { - if next.Typeflag == tar.TypeDir { - return nil - } - fileName, err := cleanPath(path.Join(destPrefix, next.Name)) - if singleFile && !destHasSlash { - fileName, err = cleanPath(destConn.Path) - } + return fsutil.PrefixCopyRemote(ctx, srcConn, destConn, srcClient, c, func(zoneId, path string, size int64, reader io.Reader) error { + dataBuf := make([]byte, size) + _, err := reader.Read(dataBuf) if err != nil { - return fmt.Errorf("error cleaning path: %w", err) - } - if !overwrite { - for _, entry := range entries { - if entry.Name == fileName { - return fmt.Errorf("destination already exists: %v", fileName) - } + if !errors.Is(err, io.EOF) { + return fmt.Errorf("error reading tar data: %w", err) } } - log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) - dataBuf := make([]byte, next.Size) - _, err = reader.Read(dataBuf) + _, err = filestore.WFS.Stat(ctx, zoneId, path) if err != nil { - if !errors.Is(err, io.EOF) { - return fmt.Errorf("error reading tar data: %w", err) + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting blockfile info: %w", err) + } else { + err := filestore.WFS.MakeFile(ctx, zoneId, path, wshrpc.FileMeta{}, wshrpc.FileOpts{}) + if err != nil { + return fmt.Errorf("error making blockfile: %w", err) + } } } - err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf) + + err = filestore.WFS.WriteFile(ctx, zoneId, path, dataBuf) if err != nil { return fmt.Errorf("error writing to blockfile: %w", err) } @@ -587,16 +550,12 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, Data: &wps.WSFileEventData{ ZoneId: zoneId, - FileName: fileName, + FileName: path, FileOp: wps.FileOp_Invalidate, }, }) return nil - }) - if err != nil { - return fmt.Errorf("error copying tar stream: %w", err) - } - return nil + }, opts) } func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { @@ -604,7 +563,7 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - schemeAndHost := conn.GetSchemeAndHost() + "/" + schemeAndHost := conn.GetSchemeAndHost() + fspath.Separator entries, err := c.ListEntries(ctx, conn, nil) if err != nil { @@ -640,7 +599,7 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu } func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { - newPath := path.Join(append([]string{conn.Path}, parts...)...) + newPath := fspath.Join(append([]string{conn.Path}, parts...)...) newPath, err := cleanPath(newPath) if err != nil { return nil, fmt.Errorf("error cleaning path: %w", err) @@ -657,17 +616,17 @@ func (c WaveClient) GetCapability() wshrpc.FileShareCapability { } func cleanPath(path string) (string, error) { - if path == "" { - return "", fmt.Errorf("path is empty") + if path == "" || path == fspath.Separator { + return "", nil } - if strings.HasPrefix(path, "/") { + if strings.HasPrefix(path, fspath.Separator) { path = path[1:] } if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { return "", fmt.Errorf("wavefile path cannot start with ~, ., or ..") } var newParts []string - for _, part := range strings.Split(path, "/") { + for _, part := range strings.Split(path, fspath.Separator) { if part == ".." { if len(newParts) > 0 { newParts = newParts[:len(newParts)-1] @@ -676,19 +635,9 @@ func cleanPath(path string) (string, error) { newParts = append(newParts, part) } } - return strings.Join(newParts, "/"), nil + return fspath.Join(newParts...), nil } func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } - -func getPathPrefix(conn *connparse.Connection) string { - fullUri := conn.GetFullURI() - pathPrefix := fullUri - lastSlash := strings.LastIndex(fullUri, "/") - if lastSlash > 10 && lastSlash < len(fullUri)-1 { - pathPrefix = fullUri[:lastSlash+1] - } - return pathPrefix -} From 654db392e36369b6e0dd087ab53a3d35a8832c0b Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 13 Feb 2025 15:26:07 -0800 Subject: [PATCH 57/65] save --- pkg/remote/fileshare/fspath/fspath.go | 18 +++++ pkg/remote/fileshare/s3fs/s3fs.go | 11 ++- pkg/remote/fileshare/wavefs/wavefs.go | 97 +++++++++++++-------------- 3 files changed, 68 insertions(+), 58 deletions(-) diff --git a/pkg/remote/fileshare/fspath/fspath.go b/pkg/remote/fileshare/fspath/fspath.go index 88adbf91fe..766ddacac2 100644 --- a/pkg/remote/fileshare/fspath/fspath.go +++ b/pkg/remote/fileshare/fspath/fspath.go @@ -23,6 +23,24 @@ func Join(elem ...string) string { return ToSlash(joined) } +// FirstLevelDir returns the first level directory of a path and a boolean indicating if the path has more than one level. +func FirstLevelDir(path string) (string, bool) { + if strings.Count(path, Separator) > 0 { + path = strings.SplitN(path, Separator, 2)[0] + return path, true + } + return path, false +} + +// FirstLevelDirPrefix returns the first level directory of a path with a prefix and a boolean indicating if the path has more than one level. +func FirstLevelDirPrefix(path, prefix string) (string, bool) { + if !strings.HasPrefix(path, prefix) { + return "", false + } + path, hasMoreLevels := FirstLevelDir(strings.TrimPrefix(path, prefix)) + return Join(prefix, path), hasMoreLevels +} + func ToSlash(path string) string { return strings.ReplaceAll(path, "\\", Separator) } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 2a558e699c..fffe8bf9b2 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -414,12 +414,10 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect lastModTime = obj.LastModified.UnixMilli() } if obj.Key != nil && len(*obj.Key) > len(objectKeyPrefix) { - name := strings.TrimPrefix(*obj.Key, objectKeyPrefix) - - // we're only interested in the first level of directories - if strings.Count(name, fspath.Separator) > 0 { - name = strings.SplitN(name, fspath.Separator, 2)[0] - path := fspath.Join(conn.GetPathWithHost(), name) + // get the first level directory name or file name + name, isDir := fspath.FirstLevelDir(strings.TrimPrefix(*obj.Key, objectKeyPrefix)) + path := fspath.Join(conn.GetPathWithHost(), name) + if isDir { if entryMap[name] == nil { if _, ok := prevUsedDirKeys[name]; !ok { entryMap[name] = &wshrpc.FileInfo{ @@ -441,7 +439,6 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect return true, nil } - path := fspath.Join(conn.GetPathWithHost(), name) size := int64(0) if obj.Size != nil { size = *obj.Size diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 50555a1f52..f95123cc7b 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -221,35 +221,26 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, return nil, fmt.Errorf("error cleaning path: %w", err) } prefix += fspath.Separator - fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) - if err != nil { - return nil, fmt.Errorf("error listing blockfiles: %w", err) - } var fileList []*wshrpc.FileInfo - for _, wf := range fileListOrig { - fileList = append(fileList, wavefileutil.WaveFileToFileInfo(wf)) - } - if prefix != "" { - var filteredList []*wshrpc.FileInfo - for _, file := range fileList { - if strings.HasPrefix(file.Name, prefix) { - filteredList = append(filteredList, file) + if err := listEntriesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { + if !opts.All { + path, isDir := fspath.FirstLevelDirPrefix(wf.Name, prefix) + if isDir { + dirMap[path] = struct{}{} } } - fileList = filteredList + fileList = append(fileList, wavefileutil.WaveFileToFileInfo(wf)) + return nil + }); err != nil { + return nil, fmt.Errorf("error listing entries: %w", err) } if !opts.All { var filteredList []*wshrpc.FileInfo dirMap := make(map[string]any) // the value is max modtime for _, file := range fileList { - // if there is an extra "/" after the prefix, don't include it - // first strip the prefix - relPath := strings.TrimPrefix(file.Name, prefix) - // then check if there is a "/" after the prefix - if strings.Contains(relPath, fspath.Separator) { - dirPath := strings.Split(relPath, fspath.Separator)[0] - dirMap[dirPath] = struct{}{} - continue + path, isDir := fspath.FirstLevelDirPrefix(file.Name, prefix) + if isDir { + dirMap[path] = struct{}{} } filteredList = append(filteredList, file) } @@ -458,8 +449,8 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, zoneId, prefix string) ([]string, error) { entryList := make([]string, 0) - if err := listEntriesPrefix(ctx, zoneId, prefix, func(entry string) error { - entryList = append(entryList, entry) + if err := listEntriesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { + entryList = append(entryList, wf.Name) return nil }); err != nil { return nil, err @@ -491,28 +482,6 @@ func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connpar }) } -func listEntriesPrefix(ctx context.Context, zoneId, prefix string, entryCallback func(string) error) error { - if zoneId == "" { - return fmt.Errorf("zoneid not found in connection") - } - fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) - if err != nil { - return fmt.Errorf("error listing blockfiles: %w", err) - } - var fileList []string - for _, wf := range fileListOrig { - fileList = append(fileList, wf.Name) - } - for _, file := range fileList { - if strings.HasPrefix(file, prefix) { - if err := entryCallback(file); err != nil { - return err - } - } - } - return nil -} - func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { if srcConn.Scheme == connparse.ConnectionTypeWave && destConn.Scheme == connparse.ConnectionTypeWave { return c.CopyInternal(ctx, srcConn, destConn, opts) @@ -565,17 +534,27 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu } schemeAndHost := conn.GetSchemeAndHost() + fspath.Separator - entries, err := c.ListEntries(ctx, conn, nil) + pathsToDelete := make([]string, 0) + err := listEntriesPrefix(ctx, zoneId, schemeAndHost, func(wf *filestore.WaveFile) error { + pathTrimmed := strings.TrimPrefix(wf.Name, schemeAndHost) + if strings.ContainsAny(pathTrimmed, fspath.Separator) && !recursive { + dir := fspath.Dir(pathTrimmed) + return fmt.Errorf("%v is not empty, use recursive flag to delete", dir) + } + pathsToDelete = append(pathsToDelete, wf.Name) + return nil + }) if err != nil { return fmt.Errorf("error listing blockfiles: %w", err) } - if len(entries) > 0 { - if !recursive { - return fmt.Errorf("more than one entry, use recursive flag to delete") + if len(pathsToDelete) > 0 { + var dirEntry *wshrpc.FileInfo + if dirEntry != nil && !recursive { + return fmt.Errorf("%v is not empty, use recursive flag to delete", dirEntry.Path) } errs := make([]error, 0) - for _, entry := range entries { - fileName := strings.TrimPrefix(entry.Path, schemeAndHost) + for _, entry := range pathsToDelete { + fileName := strings.TrimPrefix(entry, schemeAndHost) err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) if err != nil { errs = append(errs, fmt.Errorf("error deleting blockfile %s/%s: %w", zoneId, fileName, err)) @@ -598,6 +577,22 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu return nil } +func listEntriesPrefix(ctx context.Context, zoneId, prefix string, entryCallback func(*filestore.WaveFile) error) error { + if zoneId == "" { + return fmt.Errorf("zoneid not found in connection") + } + fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) + if err != nil { + return fmt.Errorf("error listing blockfiles: %w", err) + } + for _, wf := range fileListOrig { + if prefix == "" || strings.HasPrefix(wf.Name, prefix) { + entryCallback(wf) + } + } + return nil +} + func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (*wshrpc.FileInfo, error) { newPath := fspath.Join(append([]string{conn.Path}, parts...)...) newPath, err := cleanPath(newPath) From 73a86d03c7427c44eaea7a84a65d6c6b129b7e15 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 13 Feb 2025 18:09:23 -0800 Subject: [PATCH 58/65] save --- pkg/remote/fileshare/fspath/fspath.go | 9 ---- pkg/remote/fileshare/fsutil/fsutil.go | 21 ++++---- pkg/remote/fileshare/s3fs/s3fs.go | 70 +++++++++++++-------------- pkg/remote/fileshare/wavefs/wavefs.go | 48 ++++++++---------- 4 files changed, 64 insertions(+), 84 deletions(-) diff --git a/pkg/remote/fileshare/fspath/fspath.go b/pkg/remote/fileshare/fspath/fspath.go index 766ddacac2..e97ed1230e 100644 --- a/pkg/remote/fileshare/fspath/fspath.go +++ b/pkg/remote/fileshare/fspath/fspath.go @@ -32,15 +32,6 @@ func FirstLevelDir(path string) (string, bool) { return path, false } -// FirstLevelDirPrefix returns the first level directory of a path with a prefix and a boolean indicating if the path has more than one level. -func FirstLevelDirPrefix(path, prefix string) (string, bool) { - if !strings.HasPrefix(path, prefix) { - return "", false - } - path, hasMoreLevels := FirstLevelDir(strings.TrimPrefix(path, prefix)) - return Join(prefix, path), hasMoreLevels -} - func ToSlash(path string) string { return strings.ReplaceAll(path, "\\", Separator) } diff --git a/pkg/remote/fileshare/fsutil/fsutil.go b/pkg/remote/fileshare/fsutil/fsutil.go index eb53173845..a6b6660557 100644 --- a/pkg/remote/fileshare/fsutil/fsutil.go +++ b/pkg/remote/fileshare/fsutil/fsutil.go @@ -58,10 +58,7 @@ func GetPathPrefix(conn *connparse.Connection) string { return pathPrefix } -type CopyFunc func(ctx context.Context, srcPath, destPath string) error -type ListEntriesPrefix func(ctx context.Context, host, prefix string) ([]string, error) - -func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix ListEntriesPrefix, copyFunc CopyFunc) error { +func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, c fstype.FileShareClient, opts *wshrpc.FileCopyOpts, listEntriesPrefix func(ctx context.Context, host string, path string) ([]string, error), copyFunc func(ctx context.Context, host string, path string) error) error { log.Printf("PrefixCopyInternal: %v -> %v", srcConn.GetFullURI(), destConn.GetFullURI()) merge := opts != nil && opts.Merge overwrite := opts != nil && opts.Overwrite @@ -69,12 +66,12 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec return fmt.Errorf("cannot specify both overwrite and merge") } srcHasSlash := strings.HasSuffix(srcConn.Path, fspath.Separator) - srcPath, err := cleanPathPrefix(srcConn.Path) + srcPath, err := CleanPathPrefix(srcConn.Path) if err != nil { return fmt.Errorf("error cleaning source path: %w", err) } destHasSlash := strings.HasSuffix(destConn.Path, fspath.Separator) - destPath, err := cleanPathPrefix(destConn.Path) + destPath, err := CleanPathPrefix(destConn.Path) if err != nil { return fmt.Errorf("error cleaning destination path: %w", err) } @@ -145,7 +142,7 @@ func PrefixCopyInternal(ctx context.Context, srcConn, destConn *connparse.Connec } } -func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient, destClient fstype.FileShareClient, destPutFile func(host, path string, size int64, reader io.Reader) error, opts *wshrpc.FileCopyOpts) error { +func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient, destClient fstype.FileShareClient, destPutFile func(host string, path string, size int64, reader io.Reader) error, opts *wshrpc.FileCopyOpts) error { merge := opts != nil && opts.Merge overwrite := opts != nil && opts.Overwrite if overwrite && merge { @@ -153,7 +150,7 @@ func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connecti } srcHasSlash := strings.HasSuffix(srcConn.Path, fspath.Separator) destHasSlash := strings.HasSuffix(destConn.Path, fspath.Separator) - destPath, err := cleanPathPrefix(destConn.Path) + destPath, err := CleanPathPrefix(destConn.Path) if err != nil { return fmt.Errorf("error cleaning destination path: %w", err) } @@ -204,9 +201,9 @@ func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connecti if singleFile && srcInfo.IsDir { return fmt.Errorf("protocol error: source is a directory, but only a single file is being copied") } - fileName, err := cleanPathPrefix(fspath.Join(destPath, next.Name)) + fileName, err := CleanPathPrefix(fspath.Join(destPath, next.Name)) if singleFile && !destHasSlash { - fileName, err = cleanPathPrefix(destConn.Path) + fileName, err = CleanPathPrefix(destConn.Path) } if err != nil { return fmt.Errorf("error cleaning path: %w", err) @@ -221,8 +218,8 @@ func PrefixCopyRemote(ctx context.Context, srcConn, destConn *connparse.Connecti return nil } -// cleanPathPrefix corrects paths for prefix filesystems (i.e. ones that don't have directories) -func cleanPathPrefix(path string) (string, error) { +// CleanPathPrefix corrects paths for prefix filesystems (i.e. ones that don't have directories) +func CleanPathPrefix(path string) (string, error) { if path == "" { return "", fmt.Errorf("path is empty") } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index fffe8bf9b2..6e720d139a 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -413,47 +413,45 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect if obj.LastModified != nil { lastModTime = obj.LastModified.UnixMilli() } - if obj.Key != nil && len(*obj.Key) > len(objectKeyPrefix) { - // get the first level directory name or file name - name, isDir := fspath.FirstLevelDir(strings.TrimPrefix(*obj.Key, objectKeyPrefix)) - path := fspath.Join(conn.GetPathWithHost(), name) - if isDir { - if entryMap[name] == nil { - if _, ok := prevUsedDirKeys[name]; !ok { - entryMap[name] = &wshrpc.FileInfo{ - Path: path, - Name: name, - IsDir: true, - Dir: objectKeyPrefix, - ModTime: lastModTime, - Size: 0, - } - fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) - - prevUsedDirKeys[name] = struct{}{} - numFetched++ + // get the first level directory name or file name + name, isDir := fspath.FirstLevelDir(strings.TrimPrefix(*obj.Key, objectKeyPrefix)) + path := fspath.Join(conn.GetPathWithHost(), name) + if isDir { + if entryMap[name] == nil { + if _, ok := prevUsedDirKeys[name]; !ok { + entryMap[name] = &wshrpc.FileInfo{ + Path: path, + Name: name, + IsDir: true, + Dir: objectKeyPrefix, + ModTime: lastModTime, + Size: 0, } - } else if entryMap[name].ModTime < lastModTime { - entryMap[name].ModTime = lastModTime + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) + + prevUsedDirKeys[name] = struct{}{} + numFetched++ } - return true, nil + } else if entryMap[name].ModTime < lastModTime { + entryMap[name].ModTime = lastModTime } + return true, nil + } - size := int64(0) - if obj.Size != nil { - size = *obj.Size - } - entryMap[name] = &wshrpc.FileInfo{ - Name: name, - IsDir: false, - Dir: objectKeyPrefix, - Path: path, - ModTime: lastModTime, - Size: size, - } - fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) - numFetched++ + size := int64(0) + if obj.Size != nil { + size = *obj.Size + } + entryMap[name] = &wshrpc.FileInfo{ + Name: name, + IsDir: false, + Dir: objectKeyPrefix, + Path: path, + ModTime: lastModTime, + Size: size, } + fileutil.AddMimeTypeToFileInfo(path, entryMap[name]) + numFetched++ return true, nil }); err != nil { rtn <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index f95123cc7b..687f1c1693 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -222,11 +222,29 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, } prefix += fspath.Separator var fileList []*wshrpc.FileInfo + dirMap := make(map[string]*wshrpc.FileInfo) if err := listEntriesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { if !opts.All { - path, isDir := fspath.FirstLevelDirPrefix(wf.Name, prefix) + name, isDir := fspath.FirstLevelDir(strings.TrimPrefix(wf.Name, prefix)) if isDir { - dirMap[path] = struct{}{} + path := fspath.Join(conn.GetPathWithHost(), name) + if _, ok := dirMap[path]; ok { + if dirMap[path].ModTime < wf.ModTs { + dirMap[path].ModTime = wf.ModTs + } + return nil + } + dirMap[path] = &wshrpc.FileInfo{ + Path: path, + Name: name, + Dir: fspath.Dir(path), + Size: 0, + IsDir: true, + SupportsMkdir: false, + Mode: DirMode, + } + fileList = append(fileList, dirMap[path]) + return nil } } fileList = append(fileList, wavefileutil.WaveFileToFileInfo(wf)) @@ -234,30 +252,6 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, }); err != nil { return nil, fmt.Errorf("error listing entries: %w", err) } - if !opts.All { - var filteredList []*wshrpc.FileInfo - dirMap := make(map[string]any) // the value is max modtime - for _, file := range fileList { - path, isDir := fspath.FirstLevelDirPrefix(file.Name, prefix) - if isDir { - dirMap[path] = struct{}{} - } - filteredList = append(filteredList, file) - } - for dir := range dirMap { - dirName := prefix + dir - filteredList = append(filteredList, &wshrpc.FileInfo{ - Path: fspath.Join(conn.GetPathWithHost(), dirName), - Name: dirName, - Dir: fsutil.GetParentPathString(dirName), - Size: 0, - IsDir: true, - SupportsMkdir: false, - Mode: DirMode, - }) - } - fileList = filteredList - } if opts.Offset > 0 { if opts.Offset >= len(fileList) { fileList = nil @@ -278,7 +272,7 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName, err := cleanPath(conn.Path) + fileName, err := fsutil.CleanPathPrefix(conn.Path) if err != nil { return nil, fmt.Errorf("error cleaning path: %w", err) } From 59e3fb6e49eb694bdee78d0ce6fc6ba2c2dc074d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 13 Feb 2025 18:24:46 -0800 Subject: [PATCH 59/65] fix wavefs rm --- pkg/remote/fileshare/wavefs/wavefs.go | 96 +++++++++++++-------------- 1 file changed, 45 insertions(+), 51 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 687f1c1693..b30c4bad39 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -223,7 +223,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, prefix += fspath.Separator var fileList []*wshrpc.FileInfo dirMap := make(map[string]*wshrpc.FileInfo) - if err := listEntriesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { + if err := listFilesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { if !opts.All { name, isDir := fspath.FirstLevelDir(strings.TrimPrefix(wf.Name, prefix)) if isDir { @@ -319,8 +319,7 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat if err != nil { return fmt.Errorf("error cleaning path: %w", err) } - _, err = filestore.WFS.Stat(ctx, zoneId, fileName) - if err != nil { + if _, err := filestore.WFS.Stat(ctx, zoneId, fileName); err != nil { if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("error getting blockfile info: %w", err) } @@ -334,25 +333,20 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat meta = *data.Info.Meta } } - err := filestore.WFS.MakeFile(ctx, zoneId, fileName, meta, opts) - if err != nil { + if err := filestore.WFS.MakeFile(ctx, zoneId, fileName, meta, opts); err != nil { return fmt.Errorf("error making blockfile: %w", err) } } if data.At != nil && data.At.Offset >= 0 { - err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) - if errors.Is(err, fs.ErrNotExist) { + if err := filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf); errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { + } else if err != nil { return fmt.Errorf("error writing to blockfile: %w", err) } } else { - err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf) - if errors.Is(err, fs.ErrNotExist) { + if err := filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf); errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { + } else if err != nil { return fmt.Errorf("error writing to blockfile: %w", err) } } @@ -396,8 +390,7 @@ func (c WaveClient) AppendFile(ctx context.Context, conn *connparse.Connection, meta = *data.Info.Meta } } - err := filestore.WFS.MakeFile(ctx, zoneId, fileName, meta, opts) - if err != nil { + if err := filestore.WFS.MakeFile(ctx, zoneId, fileName, meta, opts); err != nil { return fmt.Errorf("error making blockfile: %w", err) } } @@ -429,12 +422,10 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar if srcConn.Host != destConn.Host { return fmt.Errorf("move internal, src and dest hosts do not match") } - err := c.CopyInternal(ctx, srcConn, destConn, opts) - if err != nil { + if err := c.CopyInternal(ctx, srcConn, destConn, opts); err != nil { return fmt.Errorf("error copying blockfile: %w", err) } - err = c.Delete(ctx, srcConn, opts.Recursive) - if err != nil { + if err := c.Delete(ctx, srcConn, opts.Recursive); err != nil { return fmt.Errorf("error deleting blockfile: %w", err) } return nil @@ -443,7 +434,7 @@ func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connpar func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return fsutil.PrefixCopyInternal(ctx, srcConn, destConn, c, opts, func(ctx context.Context, zoneId, prefix string) ([]string, error) { entryList := make([]string, 0) - if err := listEntriesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { + if err := listFilesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { entryList = append(entryList, wf.Name) return nil }); err != nil { @@ -459,8 +450,7 @@ func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connpar if err != nil { return fmt.Errorf("error reading source blockfile: %w", err) } - err = filestore.WFS.WriteFile(ctx, destHost, destFileName, dataBuf) - if err != nil { + if err := filestore.WFS.WriteFile(ctx, destHost, destFileName, dataBuf); err != nil { return fmt.Errorf("error writing to destination blockfile: %w", err) } wps.Broker.Publish(wps.WaveEvent{ @@ -486,26 +476,22 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse } return fsutil.PrefixCopyRemote(ctx, srcConn, destConn, srcClient, c, func(zoneId, path string, size int64, reader io.Reader) error { dataBuf := make([]byte, size) - _, err := reader.Read(dataBuf) - if err != nil { + if _, err := reader.Read(dataBuf); err != nil { if !errors.Is(err, io.EOF) { return fmt.Errorf("error reading tar data: %w", err) } } - _, err = filestore.WFS.Stat(ctx, zoneId, path) - if err != nil { + if _, err := filestore.WFS.Stat(ctx, zoneId, path); err != nil { if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("error getting blockfile info: %w", err) } else { - err := filestore.WFS.MakeFile(ctx, zoneId, path, wshrpc.FileMeta{}, wshrpc.FileOpts{}) - if err != nil { + if err := filestore.WFS.MakeFile(ctx, zoneId, path, wshrpc.FileMeta{}, wshrpc.FileOpts{}); err != nil { return fmt.Errorf("error making blockfile: %w", err) } } } - err = filestore.WFS.WriteFile(ctx, zoneId, path, dataBuf) - if err != nil { + if err := filestore.WFS.WriteFile(ctx, zoneId, path, dataBuf); err != nil { return fmt.Errorf("error writing to blockfile: %w", err) } wps.Broker.Publish(wps.WaveEvent{ @@ -526,32 +512,40 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - schemeAndHost := conn.GetSchemeAndHost() + fspath.Separator + prefix := conn.Path + + finfo, err := c.Stat(ctx, conn) + exists := err == nil && !finfo.NotFound + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting file info: %w", err) + } + if !exists { + return nil + } pathsToDelete := make([]string, 0) - err := listEntriesPrefix(ctx, zoneId, schemeAndHost, func(wf *filestore.WaveFile) error { - pathTrimmed := strings.TrimPrefix(wf.Name, schemeAndHost) - if strings.ContainsAny(pathTrimmed, fspath.Separator) && !recursive { - dir := fspath.Dir(pathTrimmed) - return fmt.Errorf("%v is not empty, use recursive flag to delete", dir) + + if finfo.IsDir { + if !recursive { + return fmt.Errorf("%v is not empty, use recursive flag to delete", prefix) } - pathsToDelete = append(pathsToDelete, wf.Name) - return nil - }) - if err != nil { - return fmt.Errorf("error listing blockfiles: %w", err) + if !strings.HasSuffix(prefix, fspath.Separator) { + prefix += fspath.Separator + } + if err := listFilesPrefix(ctx, zoneId, prefix, func(wf *filestore.WaveFile) error { + pathsToDelete = append(pathsToDelete, wf.Name) + return nil + }); err != nil { + return fmt.Errorf("error listing blockfiles: %w", err) + } + } else { + pathsToDelete = append(pathsToDelete, prefix) } if len(pathsToDelete) > 0 { - var dirEntry *wshrpc.FileInfo - if dirEntry != nil && !recursive { - return fmt.Errorf("%v is not empty, use recursive flag to delete", dirEntry.Path) - } errs := make([]error, 0) for _, entry := range pathsToDelete { - fileName := strings.TrimPrefix(entry, schemeAndHost) - err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) - if err != nil { - errs = append(errs, fmt.Errorf("error deleting blockfile %s/%s: %w", zoneId, fileName, err)) + if err := filestore.WFS.DeleteFile(ctx, zoneId, entry); err != nil { + errs = append(errs, fmt.Errorf("error deleting blockfile %s/%s: %w", zoneId, entry, err)) continue } wps.Broker.Publish(wps.WaveEvent{ @@ -559,7 +553,7 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, Data: &wps.WSFileEventData{ ZoneId: zoneId, - FileName: fileName, + FileName: entry, FileOp: wps.FileOp_Delete, }, }) @@ -571,7 +565,7 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu return nil } -func listEntriesPrefix(ctx context.Context, zoneId, prefix string, entryCallback func(*filestore.WaveFile) error) error { +func listFilesPrefix(ctx context.Context, zoneId, prefix string, entryCallback func(*filestore.WaveFile) error) error { if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } From de2e8e7e9dc80ebf346360ba499fd23a2250a579 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 13 Feb 2025 18:55:02 -0800 Subject: [PATCH 60/65] fix connparse for multiple slashes --- pkg/remote/connparse/connparse.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index d4b50d1d18..18c4e5e274 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -94,12 +94,12 @@ func GetConnNameFromContext(ctx context.Context) (string, error) { // ParseURI parses a connection URI and returns the connection type, host/path, and parameters. func ParseURI(uri string) (*Connection, error) { - split := strings.SplitN(uri, "//", 2) + split := strings.SplitN(uri, "://", 2) var scheme string var rest string if len(split) > 1 { - scheme = strings.TrimSuffix(split[0], ":") - rest = split[1] + scheme = split[0] + rest = strings.TrimPrefix(split[1], "//") } else { rest = split[0] } From 509131045dc4ace336e7deb458d1f4923ba0eef0 Mon Sep 17 00:00:00 2001 From: Sylvia Crowe Date: Thu, 13 Feb 2025 21:58:53 -0800 Subject: [PATCH 61/65] fix: revert fileContentAtom This implementation of fileContentAtom seems incorrect but it prevents the code editor from rerendering when the contents change. Because of this, it makes sense to use it in the meantime. --- frontend/app/view/preview/preview.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 4cc74ee701..94f976d15e 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -441,14 +441,14 @@ export class PreviewModel implements ViewModel { const fileContentAtom = atom( async (get) => { const newContent = get(this.newFileContent); - const savedContent = get(this.fileContentSaved); - const fullFile = await get(fullFileAtom); if (newContent != null) { return newContent; } + const savedContent = get(this.fileContentSaved); if (savedContent != null) { return savedContent; } + const fullFile = await get(fullFileAtom); return base64ToString(fullFile?.data64); }, (_, set, update: string) => { From 70c23a89a317984c6ae07624ec849778f6735ef2 Mon Sep 17 00:00:00 2001 From: Sylvia Crowe Date: Thu, 13 Feb 2025 22:02:28 -0800 Subject: [PATCH 62/65] test: fix behavior of test026 This updates the test behavior to match its description --- tests/copytests/cases/test026.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/copytests/cases/test026.sh b/tests/copytests/cases/test026.sh index e6bfcb3617..26655cce70 100755 --- a/tests/copytests/cases/test026.sh +++ b/tests/copytests/cases/test026.sh @@ -7,9 +7,9 @@ cd "$HOME/testcp" touch foo.txt # this is different from cp behavior -wsh file copy foo.txt baz/ >/dev/null 2>&1 && echo "command should have failed" && exit 1 +wsh file copy foo.txt baz/ -if [ -f baz/foo.txt ]; then - echo "baz/foo.txt should not exist" +if [ ! -f baz/foo.txt ]; then + echo "baz/foo.txt does not exist" exit 1 fi From 5470f9a70a20ac4741b95e884199d7c831156ebb Mon Sep 17 00:00:00 2001 From: Sylvia Crowe Date: Thu, 13 Feb 2025 22:11:43 -0800 Subject: [PATCH 63/65] fix: copy dir behavior with target dir Prior to this change, copying a directory to a directory that did not already exist resulted in the new directory containing the original one. This updates that so the new directory is the same as the original one but with a different name. --- pkg/wshrpc/wshremote/wshremote.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index ba7200fff0..163ee1738e 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -428,8 +428,10 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } if srcFileStat.IsDir() { - srcPathPrefix := filepath.Dir(srcPathCleaned) - if strings.HasSuffix(srcUri, "/") { + var srcPathPrefix string + if destIsDir { + srcPathPrefix = filepath.Dir(srcPathCleaned) + } else { srcPathPrefix = srcPathCleaned } err = filepath.Walk(srcPathCleaned, func(path string, info fs.FileInfo, err error) error { From c979dced970d6c8fa80dc4210dc28ef747c4d277 Mon Sep 17 00:00:00 2001 From: Sylvia Crowe Date: Thu, 13 Feb 2025 22:38:09 -0800 Subject: [PATCH 64/65] test: split up test 48 and 49 We previously only handle the case of copying the current directory to an existing directory. This adds another case for when the target directory does not exist. --- tests/copytests/cases/test048.sh | 19 +++++++++++++++++++ tests/copytests/cases/test049.sh | 3 +-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100755 tests/copytests/cases/test048.sh diff --git a/tests/copytests/cases/test048.sh b/tests/copytests/cases/test048.sh new file mode 100755 index 0000000000..9f86932d28 --- /dev/null +++ b/tests/copytests/cases/test048.sh @@ -0,0 +1,19 @@ +# copy the current directory into an existing directory +# ensure the copy succeeds and the output exists + +set -e +cd "$HOME/testcp" +mkdir foo +touch foo/bar.txt +mkdir baz +cd foo + + +wsh file copy . ../baz +cd .. + + +if [ ! -f baz/bar.txt ]; then + echo "baz/bar.txt does not exist" + exit 1 +fi diff --git a/tests/copytests/cases/test049.sh b/tests/copytests/cases/test049.sh index 51d309a959..3008c14653 100755 --- a/tests/copytests/cases/test049.sh +++ b/tests/copytests/cases/test049.sh @@ -1,11 +1,10 @@ -# copy the current directory into an existing directory +# copy the current directory into a non-existing directory # ensure the copy succeeds and the output exists set -e cd "$HOME/testcp" mkdir foo touch foo/bar.txt -mkdir baz cd foo wsh file copy . ../baz From ecbb9b04197c405fe574819bf60ccf7025fe229e Mon Sep 17 00:00:00 2001 From: Sylvia Crowe Date: Thu, 13 Feb 2025 23:43:21 -0800 Subject: [PATCH 65/65] feat: add additional merge button for failed copy This adds the following changes: - adds a button to perform a merge if a drag and drop copy fails - only shows the merge option for directories - renames the overwrite option to "delete then copy" for clarity --- .../app/view/preview/directorypreview.tsx | 47 ++++++++++++------- frontend/types/custom.d.ts | 1 + 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index b00687541d..368a3dfa7b 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -40,6 +40,7 @@ type FileCopyStatus = { copyData: CommandFileCopyData; copyError: string; allowRetry: boolean; + isDir: boolean; }; declare module "@tanstack/react-table" { @@ -731,6 +732,7 @@ const TableRow = React.forwardRef(function ({ relName: row.getValue("name") as string, absParent: dirPath, uri: formatRemoteUri(row.getValue("path") as string, connection), + isDir: row.original.isdir, }; const [_, drag] = useDrag( () => ({ @@ -898,18 +900,21 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { }); const handleDropCopy = useCallback( - async (data: CommandFileCopyData) => { + async (data: CommandFileCopyData, isDir) => { try { await RpcApi.FileCopyCommand(TabRpcClient, data, { timeout: data.opts.timeout }); setCopyStatus(null); } catch (e) { console.log("copy failed:", e); const copyError = `${e}`; - const allowRetry = copyError.endsWith("overwrite not specified"); + const allowRetry = + copyError.endsWith("overwrite not specified") || + copyError.endsWith("neither overwrite nor merge specified"); const copyStatus: FileCopyStatus = { copyError, copyData: data, allowRetry, + isDir: isDir, }; setCopyStatus(copyStatus); } @@ -943,7 +948,7 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { desturi, opts, }; - await handleDropCopy(data); + await handleDropCopy(data, draggedFile.isDir); } }, // TODO: mabe add a hover option? @@ -1104,21 +1109,26 @@ const CopyErrorOverlay = React.memo( }: { copyStatus: FileCopyStatus; setCopyStatus: (_: FileCopyStatus) => void; - handleDropCopy: (data: CommandFileCopyData) => Promise; + handleDropCopy: (data: CommandFileCopyData, isDir: boolean) => Promise; }) => { const [overlayRefCallback, _, domRect] = useDimensionsWithCallbackRef(30); const width = domRect?.width; - const handleRetryCopy = React.useCallback(async () => { - if (!copyStatus) { - return; - } - const updatedData = { - ...copyStatus.copyData, - opts: { ...copyStatus.copyData.opts, overwrite: true }, - }; - await handleDropCopy(updatedData); - }, [copyStatus.copyData]); + const handleRetryCopy = React.useCallback( + async (copyOpt?: string) => { + if (!copyStatus) { + return; + } + let overwrite = copyOpt == "overwrite"; + let merge = copyOpt == "merge"; + const updatedData = { + ...copyStatus.copyData, + opts: { ...copyStatus.copyData.opts, overwrite, merge }, + }; + await handleDropCopy(updatedData, copyStatus.isDir); + }, + [copyStatus.copyData] + ); let statusText = "Copy Error"; let errorMsg = `error: ${copyStatus?.copyError}`; @@ -1169,9 +1179,14 @@ const CopyErrorOverlay = React.memo( {copyStatus?.allowRetry && (
- + {copyStatus.isDir && ( + + )} diff --git a/frontend/types/custom.d.ts b/frontend/types/custom.d.ts index 70a7777618..d385db1654 100644 --- a/frontend/types/custom.d.ts +++ b/frontend/types/custom.d.ts @@ -427,6 +427,7 @@ declare global { uri: string; absParent: string; relName: string; + isDir: boolean; }; }