diff --git a/internal/mirror/cmd/pull/pull.go b/internal/mirror/cmd/pull/pull.go index f18a123d..714d88ee 100644 --- a/internal/mirror/cmd/pull/pull.go +++ b/internal/mirror/cmd/pull/pull.go @@ -98,6 +98,7 @@ func NewCommand() *cobra.Command { } pullflags.AddFlags(pullCmd.Flags()) + pullCmd.MarkFlagsMutuallyExclusive("include-module", "exclude-module") pullflags.ParseEnvironmentVariables() return pullCmd @@ -375,14 +376,18 @@ func (p *Puller) validateModulesAccess() error { // createModuleFilter creates the appropriate module filter based on whitelist/blacklist func (p *Puller) createModuleFilter() (*modules.Filter, error) { - filterExpressions := pullflags.ModulesBlacklist - filterType := modules.FilterTypeBlacklist + // Flags are mutually exclusive: + // - --include-module: whitelist mode (mirror only listed modules); + // - otherwise: blacklist mode via --exclude-module (skip listed modules). if pullflags.ModulesWhitelist != nil { - filterExpressions = pullflags.ModulesWhitelist - filterType = modules.FilterTypeWhitelist + filter, err := modules.NewFilter(pullflags.ModulesWhitelist, modules.FilterTypeWhitelist) + if err != nil { + return nil, fmt.Errorf("Prepare module filter: %w", err) + } + return filter, nil } - filter, err := modules.NewFilter(filterExpressions, filterType) + filter, err := modules.NewFilter(pullflags.ModulesBlacklist, modules.FilterTypeBlacklist) if err != nil { return nil, fmt.Errorf("Prepare module filter: %w", err) } diff --git a/internal/mirror/cmd/pull/pull_test.go b/internal/mirror/cmd/pull/pull_test.go index 4bc488ce..95fd2910 100644 --- a/internal/mirror/cmd/pull/pull_test.go +++ b/internal/mirror/cmd/pull/pull_test.go @@ -54,6 +54,18 @@ func TestNewCommand(t *testing.T) { assert.NotNil(t, cmd.Flags()) } +func TestNewCommandMutuallyExclusiveModuleFlags(t *testing.T) { + cmd := NewCommand() + cmd.PreRunE = nil + cmd.RunE = func(_ *cobra.Command, _ []string) error { return nil } + cmd.SetArgs([]string{"--include-module", "module-a", "--exclude-module", "module-b", "bundle-path"}) + + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "include-module") + assert.Contains(t, err.Error(), "exclude-module") +} + func TestSetupLogger(t *testing.T) { tests := []struct { name string diff --git a/internal/mirror/modules/filter.go b/internal/mirror/modules/filter.go index 7582e230..b66fdf52 100644 --- a/internal/mirror/modules/filter.go +++ b/internal/mirror/modules/filter.go @@ -123,7 +123,7 @@ func parseVersionConstraint(v string) (VersionConstraint, error) { } func parseExact(body string) (VersionConstraint, error) { - // exac match, console@=1.38.1 = registry.deckhouse.io/deckhouse/ce/modules/console:v1.38.1 + // exact match, console@=v1.38.1 -> registry.deckhouse.io/deckhouse/ce/modules/console:v1.38.1 tag, ch, _ := strings.Cut(body, "+") if tag == "" { return nil, fmt.Errorf("empty tag in %q", body) @@ -146,33 +146,37 @@ func parseSemver(v string) (VersionConstraint, error) { } func (f *Filter) ShouldMirrorReleaseChannels(moduleName string) bool { - if c, ok := f.modules[moduleName]; ok && c.IsExact() { + constraint, hasConstraint := f.modules[moduleName] + if hasConstraint && constraint.IsExact() { return false } return true } +// VersionsToMirror resolves module constraints from --include-module into concrete tags to pull. +// Returns nil when no explicit version tags should be added for this module. func (f *Filter) VersionsToMirror(mod *Module) []string { - c, ok := f.modules[mod.Name] - if !ok { + constraint, hasConstraint := f.modules[mod.Name] + if !hasConstraint { return nil } - if c.IsExact() { - if e, ok := c.(*ExactTagConstraint); ok { - return []string{e.Tag()} + if constraint.IsExact() { + exact, isExactTag := constraint.(*ExactTagConstraint) + if !isExactTag { + return nil } - return nil + return []string{exact.Tag()} } - sc, ok := c.(*SemanticVersionConstraint) - if !ok { + semver, isSemver := constraint.(*SemanticVersionConstraint) + if !isSemver { return nil } var tags []string for _, v := range mod.Versions() { - if sc.Match(v) { + if semver.Match(v) { tags = append(tags, "v"+v.String()) } } diff --git a/internal/mirror/modules/modules.go b/internal/mirror/modules/modules.go index cd4e9b35..189ade7b 100644 --- a/internal/mirror/modules/modules.go +++ b/internal/mirror/modules/modules.go @@ -180,6 +180,8 @@ type moduleData struct { func (svc *Service) pullModules(ctx context.Context) error { logger := svc.userLogger + // Temporary workspace for module OCI layouts: + // - stores intermediate pulled images; final bundle is packed later. tmpDir := filepath.Join(svc.workingDir, "modules") // List all available modules @@ -193,7 +195,7 @@ func (svc *Service) pullModules(ctx context.Context) error { return nil } - // Filter modules according to whitelist/blacklist + // Filter-out modules that are not allowed by the filter (blacklist or whitelist) filteredModules := make([]moduleData, 0) for _, moduleName := range moduleNames { mod := &Module{ @@ -280,160 +282,222 @@ func (svc *Service) pullModules(ctx context.Context) error { } func (svc *Service) pullSingleModule(ctx context.Context, module moduleData) error { - // Initialize download list for this module downloadList := NewImageDownloadList(filepath.Join(svc.rootURL, "modules", module.name)) svc.modulesDownloadList.list[module.name] = downloadList - // Determine which release channels to pull based on filter - shouldPullReleaseChannels := svc.options.Filter.ShouldMirrorReleaseChannels(module.name) + channelVersions, err := svc.discoverChannelVersions(ctx, module.name, downloadList) + if err != nil { + return err + } - // Get module release channel versions for image discovery - moduleVersions := make([]string, 0) + tags, err := svc.listTagsIfConstrained(ctx, module.name) + if err != nil { + return err + } - if shouldPullReleaseChannels && !svc.options.OnlyExtraImages { - // Fill release channels - for _, channel := range internal.GetAllDefaultReleaseChannels() { - downloadList.ModuleReleaseChannels[svc.rootURL+"/modules/"+module.name+"/release:"+channel] = nil - } + moduleVersions := svc.mergeAndDedupeVersions(module.name, module.registryPath, channelVersions, tags) - if !svc.options.DryRun { - // Pull release channels first to get version information - config := puller.PullConfig{ - Name: module.name + " release channels", - ImageSet: downloadList.ModuleReleaseChannels, - Layout: svc.layout.Module(module.name).ModulesReleaseChannels, - AllowMissingTags: true, - GetterService: svc.modulesService.Module(module.name).ReleaseChannels(), - } + if svc.options.DryRun { + svc.printDryRunPlan(module.name, downloadList, moduleVersions) + return nil + } - if err := svc.pullerService.PullImages(ctx, config); err != nil { - return fmt.Errorf("pull release channels: %w", err) - } + if !svc.options.OnlyExtraImages { + if err := svc.pullModuleImages(ctx, module.name, moduleVersions, downloadList); err != nil { + return err } + svc.pullReleaseVersionImages(ctx, module.name, moduleVersions, downloadList) + svc.pullInternalDigestImages(ctx, module.name, moduleVersions, downloadList) + } - // Extract versions from release channels. - // Does not depend on PullImages above - calls GetImage() directly - // against the remote registry, not from the local OCI layout. - moduleVersions = svc.extractVersionsFromReleaseChannels(ctx, module.name) + if err := svc.pullExtraImages(ctx, module.name, moduleVersions, downloadList); err != nil { + return err } - // Check for explicit version constraints from filter - mod := &Module{ - Name: module.name, - RegistryPath: module.registryPath, + if !svc.options.SkipVexImages { + svc.pullVexImages(ctx, module.name, downloadList) } - // Get specific versions to mirror from filter (for whitelist with version constraints) - filterVersions := svc.options.Filter.VersionsToMirror(mod) - if len(filterVersions) > 0 { - moduleVersions = append(moduleVersions, filterVersions...) + return nil +} + +// discoverChannelVersions enqueues release-channel refs into downloadList, +// optionally pulls them (skipped on DryRun), and returns versions extracted +// from those channels. +func (svc *Service) discoverChannelVersions(ctx context.Context, moduleName string, downloadList *ImageDownloadList) ([]string, error) { + if !svc.options.Filter.ShouldMirrorReleaseChannels(moduleName) || svc.options.OnlyExtraImages { + return nil, nil } - // Deduplicate versions - moduleVersions = deduplicateStrings(moduleVersions) + for _, channel := range internal.GetAllDefaultReleaseChannels() { + downloadList.ModuleReleaseChannels[svc.rootURL+"/modules/"+moduleName+"/release:"+channel] = nil + } - // In dry-run mode: print what would be pulled and return without downloading blobs - if svc.options.DryRun { - svc.userLogger.InfoLn("[dry-run] Module '" + module.name + "' images that would be pulled:") - for ref := range downloadList.ModuleReleaseChannels { - svc.userLogger.InfoLn(" " + ref) - } - for _, version := range moduleVersions { - svc.userLogger.InfoLn(" " + svc.rootURL + "/modules/" + module.name + ":" + version) + if !svc.options.DryRun { + config := puller.PullConfig{ + Name: moduleName + " release channels", + ImageSet: downloadList.ModuleReleaseChannels, + Layout: svc.layout.Module(moduleName).ModulesReleaseChannels, + AllowMissingTags: true, + GetterService: svc.modulesService.Module(moduleName).ReleaseChannels(), } - if len(moduleVersions) > 0 { - svc.userLogger.InfoLn(" (extra images discovery requires a real pull)") + + if err := svc.pullerService.PullImages(ctx, config); err != nil { + return nil, fmt.Errorf("pull release channels: %w", err) } + } + + // Calls GetImage() directly against the remote registry, not from the local + // OCI layout, so it works in DryRun too. + return svc.extractVersionsFromReleaseChannels(ctx, moduleName), nil +} + +// listTagsIfConstrained returns the module's tag list, but only when the filter +// has a non-exact (semver) constraint - exact-tag and no-constraint paths don't +// read Releases. ErrImageNotFound is logged and treated as no tags (same policy +// as validateModulesAccess for missing module repos). +func (svc *Service) listTagsIfConstrained(ctx context.Context, moduleName string) ([]string, error) { + constraint, hasConstraint := svc.options.Filter.GetConstraint(moduleName) + if !hasConstraint || constraint.IsExact() { + return nil, nil + } + + tags, err := svc.modulesService.Module(moduleName).ListTags(ctx) + switch { + case errors.Is(err, client.ErrImageNotFound): + svc.userLogger.Warnf("Skipping tag list for module %s: %v", moduleName, err) + return nil, nil + case err != nil: + return nil, fmt.Errorf("list tags for module %s: %w", moduleName, err) + } + return tags, nil +} + +// mergeAndDedupeVersions merges channel-derived versions with versions resolved +// from filter constraints over the given tags, then deduplicates. +func (svc *Service) mergeAndDedupeVersions(moduleName, registryPath string, channelVersions, tags []string) []string { + versions := append([]string(nil), channelVersions...) + + mod := &Module{ + Name: moduleName, + RegistryPath: registryPath, + Releases: tags, + } + versions = append(versions, svc.options.Filter.VersionsToMirror(mod)...) + + return deduplicateStrings(versions) +} + +// printDryRunPlan prints the set of refs that would be pulled, without downloading. +func (svc *Service) printDryRunPlan(moduleName string, downloadList *ImageDownloadList, versions []string) { + svc.userLogger.InfoLn("[dry-run] Module '" + moduleName + "' images that would be pulled:") + for ref := range downloadList.ModuleReleaseChannels { + svc.userLogger.InfoLn(" " + ref) + } + for _, version := range versions { + svc.userLogger.InfoLn(" " + svc.rootURL + "/modules/" + moduleName + ":" + version) + } + if len(versions) > 0 { + svc.userLogger.InfoLn(" (extra images discovery requires a real pull)") + } +} + +// pullModuleImages pulls module images for the given versions (modules/:vX.Y.Z). +func (svc *Service) pullModuleImages(ctx context.Context, moduleName string, versions []string, downloadList *ImageDownloadList) error { + // Guard against future call-order changes: other helpers also write to + // downloadList.Module, so checking versions directly is the only invariant. + if len(versions) == 0 { return nil } - // Skip main module images if only pulling extra images - if !svc.options.OnlyExtraImages { - // Fill module images for each version - for _, version := range moduleVersions { - downloadList.Module[svc.rootURL+"/modules/"+module.name+":"+version] = nil - } - - // Pull module images - if len(downloadList.Module) > 0 { - config := puller.PullConfig{ - Name: module.name + " images", - ImageSet: downloadList.Module, - Layout: svc.layout.Module(module.name).Modules, - AllowMissingTags: true, - GetterService: svc.modulesService.Module(module.name), - } + for _, version := range versions { + downloadList.Module[svc.rootURL+"/modules/"+moduleName+":"+version] = nil + } - if err := svc.pullerService.PullImages(ctx, config); err != nil { - return fmt.Errorf("pull module images: %w", err) - } - } + config := puller.PullConfig{ + Name: moduleName + " images", + ImageSet: downloadList.Module, + Layout: svc.layout.Module(moduleName).Modules, + AllowMissingTags: true, + GetterService: svc.modulesService.Module(moduleName), + } - // Also pull release images with version tags (modules//release:v1.x.x) - // These are in addition to channel tags (alpha, beta, etc.) - if len(moduleVersions) > 0 { - releaseVersionSet := make(map[string]*puller.ImageMeta) - for _, version := range moduleVersions { - releaseVersionSet[svc.rootURL+"/modules/"+module.name+"/release:"+version] = nil - downloadList.ModuleReleaseChannels[svc.rootURL+"/modules/"+module.name+"/release:"+version] = nil - } + if err := svc.pullerService.PullImages(ctx, config); err != nil { + return fmt.Errorf("pull module images: %w", err) + } + return nil +} - config := puller.PullConfig{ - Name: module.name + " release versions", - ImageSet: releaseVersionSet, - Layout: svc.layout.Module(module.name).ModulesReleaseChannels, - AllowMissingTags: true, - GetterService: svc.modulesService.Module(module.name).ReleaseChannels(), - } +// pullReleaseVersionImages pulls modules//release:vX.Y.Z tags in addition +// to channel tags (alpha, beta, ...). These may not exist for every version, so +// errors are logged at Debug and not propagated. +func (svc *Service) pullReleaseVersionImages(ctx context.Context, moduleName string, versions []string, downloadList *ImageDownloadList) { + if len(versions) == 0 { + return + } - if err := svc.pullerService.PullImages(ctx, config); err != nil { - svc.logger.Debug(fmt.Sprintf("Failed to pull release version images for %s: %v", module.name, err)) - // Don't fail - version release images may not exist for all versions - } - } + releaseVersionSet := make(map[string]*puller.ImageMeta) + for _, version := range versions { + releaseVersionSet[svc.rootURL+"/modules/"+moduleName+"/release:"+version] = nil + downloadList.ModuleReleaseChannels[svc.rootURL+"/modules/"+moduleName+"/release:"+version] = nil + } - // Extract and pull internal digest images from module versions (images_digests.json) - // These are internal images that module uses at runtime - digestImages := svc.extractInternalDigestImages(ctx, module.name, moduleVersions) - if len(digestImages) > 0 { - // Add digest images to download list - digestImageSet := make(map[string]*puller.ImageMeta) - for _, digestRef := range digestImages { - digestImageSet[digestRef] = nil - downloadList.Module[digestRef] = nil - } + config := puller.PullConfig{ + Name: moduleName + " release versions", + ImageSet: releaseVersionSet, + Layout: svc.layout.Module(moduleName).ModulesReleaseChannels, + AllowMissingTags: true, + GetterService: svc.modulesService.Module(moduleName).ReleaseChannels(), + } - config := puller.PullConfig{ - Name: module.name + " internal images", - ImageSet: digestImageSet, - Layout: svc.layout.Module(module.name).Modules, - AllowMissingTags: true, - GetterService: svc.modulesService.Module(module.name), - } + if err := svc.pullerService.PullImages(ctx, config); err != nil { + svc.logger.Debug(fmt.Sprintf("Failed to pull release version images for %s: %v", moduleName, err)) + } +} - if err := svc.pullerService.PullImages(ctx, config); err != nil { - svc.logger.Debug(fmt.Sprintf("Failed to pull internal digest images for %s: %v", module.name, err)) - // Don't fail on missing internal images, just log warning - } - } +// pullInternalDigestImages discovers and pulls images referenced by +// images_digests.json inside each module version. Errors are logged at Debug and +// not propagated - missing internal images do not fail the whole module pull. +func (svc *Service) pullInternalDigestImages(ctx context.Context, moduleName string, versions []string, downloadList *ImageDownloadList) { + digestImages := svc.extractInternalDigestImages(ctx, moduleName, versions) + if len(digestImages) == 0 { + return } - // Extract and pull extra images from module versions - // Each extra image gets its own layout: modules//extra// - extraImagesByName := svc.findExtraImages(ctx, module.name, moduleVersions) + digestImageSet := make(map[string]*puller.ImageMeta) + for _, digestRef := range digestImages { + digestImageSet[digestRef] = nil + downloadList.Module[digestRef] = nil + } + + config := puller.PullConfig{ + Name: moduleName + " internal images", + ImageSet: digestImageSet, + Layout: svc.layout.Module(moduleName).Modules, + AllowMissingTags: true, + GetterService: svc.modulesService.Module(moduleName), + } + + if err := svc.pullerService.PullImages(ctx, config); err != nil { + svc.logger.Debug(fmt.Sprintf("Failed to pull internal digest images for %s: %v", moduleName, err)) + } +} + +// pullExtraImages discovers extra images declared by each module version and +// pulls them into per-extra layouts (modules//extra//). +func (svc *Service) pullExtraImages(ctx context.Context, moduleName string, versions []string, downloadList *ImageDownloadList) error { + extraImagesByName := svc.findExtraImages(ctx, moduleName, versions) for extraName, images := range extraImagesByName { if len(images) == 0 { continue } - // Get or create layout for this extra image - extraLayout, err := svc.layout.Module(module.name).GetOrCreateExtraLayout(extraName) + extraLayout, err := svc.layout.Module(moduleName).GetOrCreateExtraLayout(extraName) if err != nil { return fmt.Errorf("create layout for extra image %s: %w", extraName, err) } - // Build image set for this extra imageSet := make(map[string]*puller.ImageMeta) for _, img := range images { imageSet[img.FullRef] = nil @@ -441,24 +505,84 @@ func (svc *Service) pullSingleModule(ctx context.Context, module moduleData) err } config := puller.PullConfig{ - Name: module.name + "/" + extraName, + Name: moduleName + "/" + extraName, ImageSet: imageSet, Layout: extraLayout, AllowMissingTags: true, - GetterService: svc.modulesService.Module(module.name).ExtraImage(extraName), + GetterService: svc.modulesService.Module(moduleName).ExtraImage(extraName), } if err := svc.pullerService.PullImages(ctx, config); err != nil { return fmt.Errorf("pull extra image %s: %w", extraName, err) } } + return nil +} - if !svc.options.SkipVexImages { - // Find and pull VEX images for all module images - svc.pullVexImages(ctx, module.name, downloadList) +// pullVexImages finds and pulls VEX attestation images for module images +func (svc *Service) pullVexImages(ctx context.Context, moduleName string, downloadList *ImageDownloadList) { + allImages := make([]string, 0, len(downloadList.Module)+len(downloadList.ModuleExtra)) + + for img := range downloadList.Module { + allImages = append(allImages, img) + } + for img := range downloadList.ModuleExtra { + allImages = append(allImages, img) } - return nil + // Find VEX images and add to a separate set for pulling + vexImageSet := make(map[string]*puller.ImageMeta) + for _, img := range allImages { + vexImageName, err := svc.findVexImage(ctx, moduleName, img) + if err != nil { + svc.logger.Debug(fmt.Sprintf("Failed to find VEX image for %s: %v", img, err)) + continue + } + if vexImageName != "" { + svc.logger.Debug(fmt.Sprintf("Found VEX image: %s", vexImageName)) + vexImageSet[vexImageName] = nil + downloadList.Module[vexImageName] = nil + } + } + + // Pull VEX images if any found + if len(vexImageSet) > 0 { + config := puller.PullConfig{ + Name: moduleName + " VEX images", + ImageSet: vexImageSet, + Layout: svc.layout.Module(moduleName).Modules, + AllowMissingTags: true, // VEX images may not exist + GetterService: svc.modulesService.Module(moduleName), + } + + if err := svc.pullerService.PullImages(ctx, config); err != nil { + svc.logger.Debug(fmt.Sprintf("Failed to pull VEX images for %s: %v", moduleName, err)) + // Don't fail on VEX image pull errors + } + } +} + +// findVexImage checks if a VEX attestation image exists for the given image +func (svc *Service) findVexImage(ctx context.Context, moduleName string, imageRef string) (string, error) { + // VEX image reference format: sha256-xxx.att + vexImageName := strings.Replace(strings.Replace(imageRef, "@sha256:", "@sha256-", 1), "@sha256", ":sha256", 1) + ".att" + + // Extract tag from vex image name + splitIndex := strings.LastIndex(vexImageName, ":") + if splitIndex == -1 { + return "", nil + } + tag := vexImageName[splitIndex+1:] + + err := svc.modulesService.Module(moduleName).CheckImageExists(ctx, tag) + if errors.Is(err, client.ErrImageNotFound) { + return "", nil + } + if err != nil { + return "", err + } + + return vexImageName, nil } // extractVersionsFromReleaseChannels extracts version tags from pulled release channel images @@ -613,37 +737,6 @@ func extractExtraImagesJSON(img interface{ Extract() io.ReadCloser }) (map[strin } } -// digestRegex matches sha256 digests in images_digests.json -var digestRegex = regexp.MustCompile(`sha256:[a-f0-9]{64}`) - -// extractImagesDigestsJSON extracts images_digests.json from module image -// and returns list of sha256 digests. These are internal images that module uses at runtime. -func extractImagesDigestsJSON(img interface{ Extract() io.ReadCloser }) ([]string, error) { - rc := img.Extract() - defer rc.Close() - - tr := tar.NewReader(rc) - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil, fmt.Errorf("images_digests.json not found in image") - } - if err != nil { - return nil, err - } - - if hdr.Name == "images_digests.json" { - data, err := io.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("read images_digests.json: %w", err) - } - // Extract all sha256:... digests from JSON file - digests := digestRegex.FindAllString(string(data), -1) - return digests, nil - } - } -} - // extractInternalDigestImages extracts internal digest images from module versions. // It reads images_digests.json from each module version image and returns // list of image references in format "repo@sha256:..." which will be pulled @@ -697,72 +790,37 @@ func (svc *Service) extractInternalDigestImages(ctx context.Context, moduleName return digestRefs } -// pullVexImages finds and pulls VEX attestation images for module images -func (svc *Service) pullVexImages(ctx context.Context, moduleName string, downloadList *ImageDownloadList) { - allImages := make([]string, 0, len(downloadList.Module)+len(downloadList.ModuleExtra)) +// digestRegex matches sha256 digests in images_digests.json +var digestRegex = regexp.MustCompile(`sha256:[a-f0-9]{64}`) - for img := range downloadList.Module { - allImages = append(allImages, img) - } - for img := range downloadList.ModuleExtra { - allImages = append(allImages, img) - } +// extractImagesDigestsJSON extracts images_digests.json from module image +// and returns list of sha256 digests. These are internal images that module uses at runtime. +func extractImagesDigestsJSON(img interface{ Extract() io.ReadCloser }) ([]string, error) { + rc := img.Extract() + defer rc.Close() - // Find VEX images and add to a separate set for pulling - vexImageSet := make(map[string]*puller.ImageMeta) - for _, img := range allImages { - vexImageName, err := svc.findVexImage(ctx, moduleName, img) - if err != nil { - svc.logger.Debug(fmt.Sprintf("Failed to find VEX image for %s: %v", img, err)) - continue - } - if vexImageName != "" { - svc.logger.Debug(fmt.Sprintf("Found VEX image: %s", vexImageName)) - vexImageSet[vexImageName] = nil - downloadList.Module[vexImageName] = nil + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil, fmt.Errorf("images_digests.json not found in image") } - } - - // Pull VEX images if any found - if len(vexImageSet) > 0 { - config := puller.PullConfig{ - Name: moduleName + " VEX images", - ImageSet: vexImageSet, - Layout: svc.layout.Module(moduleName).Modules, - AllowMissingTags: true, // VEX images may not exist - GetterService: svc.modulesService.Module(moduleName), + if err != nil { + return nil, err } - if err := svc.pullerService.PullImages(ctx, config); err != nil { - svc.logger.Debug(fmt.Sprintf("Failed to pull VEX images for %s: %v", moduleName, err)) - // Don't fail on VEX image pull errors + if hdr.Name == "images_digests.json" { + data, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("read images_digests.json: %w", err) + } + // Extract all sha256:... digests from JSON file + digests := digestRegex.FindAllString(string(data), -1) + return digests, nil } } } -// findVexImage checks if a VEX attestation image exists for the given image -func (svc *Service) findVexImage(ctx context.Context, moduleName string, imageRef string) (string, error) { - // VEX image reference format: sha256-xxx.att - vexImageName := strings.Replace(strings.Replace(imageRef, "@sha256:", "@sha256-", 1), "@sha256", ":sha256", 1) + ".att" - - // Extract tag from vex image name - splitIndex := strings.LastIndex(vexImageName, ":") - if splitIndex == -1 { - return "", nil - } - tag := vexImageName[splitIndex+1:] - - err := svc.modulesService.Module(moduleName).CheckImageExists(ctx, tag) - if errors.Is(err, client.ErrImageNotFound) { - return "", nil - } - if err != nil { - return "", err - } - - return vexImageName, nil -} - // applyChannelAliases applies release channel tags to images with exact tag constraints func (svc *Service) applyChannelAliases(moduleName string) error { constraint, ok := svc.options.Filter.GetConstraint(moduleName) diff --git a/internal/mirror/modules/pull_modules_test.go b/internal/mirror/modules/pull_modules_test.go new file mode 100644 index 00000000..47bffcab --- /dev/null +++ b/internal/mirror/modules/pull_modules_test.go @@ -0,0 +1,409 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modules + +import ( + "context" + "errors" + "log/slog" + "strings" + "sync/atomic" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + dkplog "github.com/deckhouse/deckhouse/pkg/log" + dkpreg "github.com/deckhouse/deckhouse/pkg/registry" + upfake "github.com/deckhouse/deckhouse/pkg/registry/fake" + + "github.com/deckhouse/deckhouse-cli/pkg" + "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/log" + pkgclient "github.com/deckhouse/deckhouse-cli/pkg/registry/client" + registryservice "github.com/deckhouse/deckhouse-cli/pkg/registry/service" +) + +// ============================================================================= +// Shared fixtures +// ============================================================================= + +const ( + testHost = "fake.registry" + testModuleName = "console" + channelVersion = "v1.45.2" // version every release channel points at +) + +// defaultRegistryVersions is a small but representative tag set for tests +// that don't care about the exact list - a few patches and a few minors. +var defaultRegistryVersions = []string{"v1.40.0", "v1.40.1", "v1.41.0", "v1.45.2"} + +// ============================================================================= +// Tests: which versions get pulled +// ============================================================================= + +// Regression for --include-module @: every tag in the registry +// that satisfies the semver constraint must end up in the download list, +// not only the tag pinned by the release channel snapshot. +func TestPullModules_SemverConstraintPullsAllMatchingTags(t *testing.T) { + registryVersions := []string{ + "v1.39.0", + "v1.40.0", "v1.40.1", + "v1.41.0", "v1.42.0", "v1.43.0", + channelVersion, + } + + cases := []struct { + name string + constraint string // text after "@" in --include-module + wantTags []string // tags expected to land in the download list + rejectTag string // tag present in the registry that the constraint must reject + }{ + { + name: "implicit caret (1.40.0 -> ^1.40.0)", + constraint: "1.40.0", + wantTags: []string{"v1.40.0", "v1.40.1", "v1.41.0", "v1.42.0", "v1.43.0", channelVersion}, + rejectTag: "v1.39.0", + }, + { + name: "explicit caret (^1.40.0)", + constraint: "^1.40.0", + wantTags: []string{"v1.40.0", "v1.40.1", "v1.41.0", "v1.42.0", "v1.43.0", channelVersion}, + rejectTag: "v1.39.0", + }, + { + name: "tilde (~1.40.0 - patch only)", + constraint: "~1.40.0", + wantTags: []string{"v1.40.0", "v1.40.1", channelVersion}, + rejectTag: "v1.41.0", + }, + { + name: "explicit range (>=1.40.0 <1.43.0)", + constraint: ">=1.40.0 <1.43.0", + wantTags: []string{"v1.40.0", "v1.40.1", "v1.41.0", "v1.42.0", channelVersion}, + rejectTag: "v1.43.0", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reg := singleModuleRegistry(testModuleName, channelVersion, registryVersions) + filter := mustNewFilter(t, FilterTypeWhitelist, testModuleName+"@"+tc.constraint) + svc := newService(t, pkgclient.Adapt(upfake.NewClient(reg)), filter) + + require.NoError(t, svc.PullModules(context.Background())) + + got := pulledModuleVersionRefs(t, svc, testModuleName) + assert.ElementsMatch(t, taggedModuleRefs(testModuleName, tc.wantTags), got) + assert.NotContains(t, got, taggedModuleRef(testModuleName, tc.rejectTag), + "constraint %q must reject %s", tc.constraint, tc.rejectTag) + }) + } +} + +// Each --include-module flag carries its own constraint - the matcher must +// scope to the named module and not leak across modules. Mixes a semver and +// an exact constraint to exercise both filter branches in one shot. +func TestPullModules_PerModuleConstraintIsolation(t *testing.T) { + const ( + consoleName = "console" + commanderName = "commander" + ) + + reg := upfake.NewRegistry(testHost) + addModule(reg, consoleName, "v1.40.1", []string{"v1.40.0", "v1.40.1", "v1.41.0", channelVersion}) + addModule(reg, commanderName, "v0.5.1", []string{"v0.5.0", "v0.5.1", "v0.6.0"}) + + filter := mustNewFilter(t, FilterTypeWhitelist, + consoleName+"@~1.40.0", // tilde matches v1.40.x only + commanderName+"@=v0.6.0", // exact tag + ) + svc := newService(t, pkgclient.Adapt(upfake.NewClient(reg)), filter) + + require.NoError(t, svc.PullModules(context.Background())) + + assert.ElementsMatch(t, + taggedModuleRefs(consoleName, []string{"v1.40.0", "v1.40.1"}), + pulledModuleVersionRefs(t, svc, consoleName), + "console: tilde must match only v1.40.x") + assert.ElementsMatch(t, + taggedModuleRefs(commanderName, []string{"v0.6.0"}), + pulledModuleVersionRefs(t, svc, commanderName), + "commander: exact must match only v0.6.0 - no leak from console's tilde") +} + +// ============================================================================= +// Tests: per-module ListTags policy +// ============================================================================= + +// Per-module ListTags is only needed for non-exact constraints. The baseline +// cost of a default pull or an exact-tag pull must not regress by adding an +// unconditional per-module call. +func TestPullModules_PerModuleListTagsCallCount(t *testing.T) { + // PullModules lists tags at the registry root twice: + // - validateModulesAccess (reachability check) + // - pullModules (module-name enumeration) + const baselineRootCalls int64 = 2 + + cases := []struct { + name string + filterType FilterType + filterExprs []string + wantExtra int64 // extra ListTags calls on top of the baseline + }{ + { + name: "exact constraint skips per-module ListTags", + filterType: FilterTypeWhitelist, + filterExprs: []string{testModuleName + "@=v1.40.0"}, + wantExtra: 0, + }, + { + name: "blacklist filter (no constraint) skips per-module ListTags", + filterType: FilterTypeBlacklist, + filterExprs: nil, + wantExtra: 0, + }, + { + name: "semver constraint triggers one per-module ListTags", + filterType: FilterTypeWhitelist, + filterExprs: []string{testModuleName + "@1.40.0"}, + wantExtra: 1, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reg := singleModuleRegistry(testModuleName, channelVersion, defaultRegistryVersions) + counter := newListTagsCounter(upfake.NewClient(reg)) + filter := mustNewFilter(t, tc.filterType, tc.filterExprs...) + svc := newService(t, pkgclient.Adapt(counter), filter) + + require.NoError(t, svc.PullModules(context.Background())) + + assert.Equal(t, baselineRootCalls+tc.wantExtra, counter.calls.Load()) + }) + } +} + +// Per-module ListTags reuses validateModulesAccess's error policy: +// - ErrImageNotFound: warn-and-skip (the module repo simply isn't there). +// - any other error: fail-fast (we cannot verify the constraint and refuse +// to silently produce a partial bundle). +func TestPullModules_PerModuleListTagsErrorHandling(t *testing.T) { + transientErr := errors.New("simulated registry 503") + + cases := []struct { + name string + injected error + wantErr error // nil = pull must succeed + wantTags []string // checked only on success - the channel snapshot is the sole contributor when per-module ListTags is skipped + }{ + { + name: "ErrImageNotFound is warned and skipped", + injected: dkpreg.ErrImageNotFound, + wantTags: []string{channelVersion}, + }, + { + name: "transient error fails fast", + injected: transientErr, + wantErr: transientErr, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reg := singleModuleRegistry(testModuleName, channelVersion, defaultRegistryVersions) + client := newListTagsErrAtModule(upfake.NewClient(reg), tc.injected) + // Semver constraint to actually trigger the per-module ListTags. + filter := mustNewFilter(t, FilterTypeWhitelist, testModuleName+"@1.40.0") + svc := newService(t, pkgclient.Adapt(client), filter) + + err := svc.PullModules(context.Background()) + + if tc.wantErr == nil { + require.NoError(t, err) + assert.ElementsMatch(t, + taggedModuleRefs(testModuleName, tc.wantTags), + pulledModuleVersionRefs(t, svc, testModuleName)) + return + } + + require.Error(t, err) + assert.ErrorIs(t, err, tc.wantErr) + assert.Contains(t, err.Error(), "list tags for module "+testModuleName) + }) + } +} + +// ============================================================================= +// Service & filter builders +// ============================================================================= + +// newService wires a Service against the given fake client, with logs muted. +func newService(t *testing.T, client dkpreg.Client, filter *Filter) *Service { + t.Helper() + + logger := dkplog.NewLogger(dkplog.WithLevel(slog.LevelWarn)) + userLogger := log.NewSLogger(slog.LevelWarn) + regSvc := registryservice.NewService(client, pkg.NoEdition, logger) + + return NewService( + regSvc, + t.TempDir(), + &Options{ + BundleDir: t.TempDir(), + Filter: filter, + SkipVexImages: true, + }, + logger, + userLogger, + ) +} + +func mustNewFilter(t *testing.T, ftype FilterType, exprs ...string) *Filter { + t.Helper() + f, err := NewFilter(exprs, ftype) + require.NoError(t, err) + return f +} + +// ============================================================================= +// Registry fixture builders +// ============================================================================= + +// addModule populates a fake registry with one module's worth of refs: +// +// modules: - modules-list entry, points at channelVer +// modules/: - one image per version +// modules//release: - 5 release channels, all pointing at channelVer +func addModule(reg *upfake.Registry, name, channelVer string, versions []string) { + reg.MustAddImage("modules", name, versionImage(channelVer)) + for _, v := range versions { + reg.MustAddImage("modules/"+name, v, versionImage(v)) + } + for _, ch := range []string{"alpha", "beta", "early-access", "stable", "rock-solid"} { + reg.MustAddImage("modules/"+name+"/release", ch, versionImage(channelVer)) + } +} + +// singleModuleRegistry builds a fake registry containing exactly one module. +func singleModuleRegistry(name, channelVer string, versions []string) *upfake.Registry { + reg := upfake.NewRegistry(testHost) + addModule(reg, name, channelVer, versions) + return reg +} + +// versionImage builds a v1.Image carrying only version.json. Missing +// images_digests.json / extra_images.json is tolerated downstream. +func versionImage(version string) v1.Image { + return upfake.NewImageBuilder(). + WithFile("version.json", `{"version":"`+version+`"}`). + WithLabel("org.opencontainers.image.version", version). + MustBuild() +} + +// ============================================================================= +// Assertion helpers +// ============================================================================= + +// taggedModuleRef is the registry URL the production code uses for a single +// version-tagged module image. +func taggedModuleRef(moduleName, version string) string { + return testHost + "/modules/" + moduleName + ":" + version +} + +func taggedModuleRefs(moduleName string, versions []string) []string { + refs := make([]string, 0, len(versions)) + for _, v := range versions { + refs = append(refs, taggedModuleRef(moduleName, v)) + } + return refs +} + +// pulledModuleVersionRefs returns the version-tagged refs the service recorded +// for the given module, dropping @sha256: refs (these are added by extra-image +// resolution and are not relevant to constraint tests). +func pulledModuleVersionRefs(t *testing.T, svc *Service, moduleName string) []string { + t.Helper() + dl := svc.modulesDownloadList.Module(moduleName) + require.NotNil(t, dl, "no download list recorded for module %s", moduleName) + + refs := make([]string, 0, len(dl.Module)) + for ref := range dl.Module { + if strings.Contains(ref, "@sha256:") { + continue + } + refs = append(refs, ref) + } + return refs +} + +// ============================================================================= +// Test doubles +// ============================================================================= + +// listTagsCounter counts every ListTags call on this client and on any +// sub-client spawned via WithSegment (the counter is shared across the chain). +type listTagsCounter struct { + dkpreg.Client + calls *atomic.Int64 +} + +func newListTagsCounter(c dkpreg.Client) *listTagsCounter { + return &listTagsCounter{Client: c, calls: new(atomic.Int64)} +} + +func (c *listTagsCounter) WithSegment(segments ...string) dkpreg.Client { + return &listTagsCounter{Client: c.Client.WithSegment(segments...), calls: c.calls} +} + +func (c *listTagsCounter) ListTags(ctx context.Context, opts ...dkpreg.ListTagsOption) ([]string, error) { + c.calls.Add(1) + return c.Client.ListTags(ctx, opts...) +} + +// listTagsErrAtModule returns the configured error from ListTags only at the +// per-module path (modules/, two segments below root). Root-level +// ListTags - validateModulesAccess and module-name enumeration - pass through. +type listTagsErrAtModule struct { + dkpreg.Client + depth int + err error +} + +// perModuleDepth is the WithSegment depth at which a client points at +// modules/: root -> "modules" -> "". +const perModuleDepth = 2 + +func newListTagsErrAtModule(c dkpreg.Client, err error) *listTagsErrAtModule { + return &listTagsErrAtModule{Client: c, depth: 0, err: err} +} + +func (c *listTagsErrAtModule) WithSegment(segments ...string) dkpreg.Client { + return &listTagsErrAtModule{ + Client: c.Client.WithSegment(segments...), + depth: c.depth + len(segments), + err: c.err, + } +} + +func (c *listTagsErrAtModule) ListTags(ctx context.Context, opts ...dkpreg.ListTagsOption) ([]string, error) { + if c.depth == perModuleDepth { + return nil, c.err + } + return c.Client.ListTags(ctx, opts...) +}