From 3078a507fa49b2125a09b83987b2001bafe284d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 08:48:34 +0000 Subject: [PATCH 1/5] Add batched Apple MDM profile reconciler with in-memory desired-state handlers Optimizes the Apple profile reconciler path by scanning hosts in bounded batches (default 5k per tick via a host_uuid cursor in Redis) and computing desired state in Go using per-label-mode handlers instead of a large MySQL UNION join. Each tick: 1. ListAppleMDMHostsForReconcileBatch pulls the next batch of Apple-enrolled host UUIDs by cursor (no profile-status check). 2. ListAppleProfilesForReconcile loads the full profile catalog and label assignments once per tick. 3. BulkGetHostLabelMemberships and BulkGetHostMDMAppleProfilesByUUIDs load the per-batch label memberships and current state. 4. computeAppleReconcileDeltas dispatches each (host, profile) pair to one of four in-code handlers: no-labels, include-all, include-any, exclude-any. Broken-label and dynamic-label-timing semantics match the legacy SQL. 5. The downstream CA-throttle, user-enrollment, host-being-set-up skip, BulkUpsertMDMAppleHostProfiles, and ProcessAndEnqueueProfiles flow is reused unchanged. Gated by FLEET_MDM_APPLE_BATCHED_RECONCILER=true so the legacy path stays the default. The cursor is persisted in Redis (mysqlredis wrapper) and resets when a full pass completes, mirroring the Windows reconciler pattern. Includes unit tests for each handler and the delta computation. https://claude.ai/code/session_01Vvy1keXRKZRzDbJQd7dzDn --- cmd/fleet/cron.go | 21 + server/datastore/mysql/apple_mdm_batched.go | 337 +++++++ .../mysqlredis/apple_recon_cursor.go | 49 ++ server/fleet/apple_mdm.go | 73 ++ server/fleet/datastore.go | 36 + server/mock/datastore_mock.go | 72 ++ server/service/apple_mdm_batched.go | 831 ++++++++++++++++++ server/service/apple_mdm_batched_test.go | 450 ++++++++++ 8 files changed, 1869 insertions(+) create mode 100644 server/datastore/mysql/apple_mdm_batched.go create mode 100644 server/datastore/mysqlredis/apple_recon_cursor.go create mode 100644 server/service/apple_mdm_batched.go create mode 100644 server/service/apple_mdm_batched_test.go diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 821bb906efb..1d2047ce072 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1857,11 +1857,16 @@ func newAppleMDMProfileManagerSchedule( defaultInterval = 30 * time.Second ) + useBatchedReconciler := isEnvBoolTrue("FLEET_MDM_APPLE_BATCHED_RECONCILER") + logger = logger.With("cron", name) s := schedule.New( ctx, name, instanceID, defaultInterval, ds, ds, schedule.WithLogger(logger), schedule.WithJob("manage_apple_profiles", func(ctx context.Context) error { + if useBatchedReconciler { + return service.ReconcileAppleProfilesBatched(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) + } return service.ReconcileAppleProfiles(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) }), schedule.WithJob("manage_apple_declarations", func(ctx context.Context) error { @@ -1872,6 +1877,22 @@ func newAppleMDMProfileManagerSchedule( return s, nil } +// isEnvBoolTrue returns true if the named env var is set to a truthy value +// (1, t, T, TRUE, true, True). Used to gate the experimental batched Apple +// MDM profile reconciler. +func isEnvBoolTrue(name string) bool { + v, ok := os.LookupEnv(name) + if !ok { + return false + } + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "t", "true", "yes", "y": + return true + default: + return false + } +} + func newWindowsMDMProfileManagerSchedule( ctx context.Context, instanceID string, diff --git a/server/datastore/mysql/apple_mdm_batched.go b/server/datastore/mysql/apple_mdm_batched.go new file mode 100644 index 00000000000..fe40a9c48d8 --- /dev/null +++ b/server/datastore/mysql/apple_mdm_batched.go @@ -0,0 +1,337 @@ +package mysql + +import ( + "context" + "database/sql" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +// ListAppleMDMHostsForReconcileBatch returns up to batchSize Apple-MDM- +// enrolled hosts with host_uuid > afterHostUUID, ordered ascending by uuid, +// along with the fields the batched reconciler needs to compute desired +// state in memory. +// +// Selection criteria mirror the host-side filters in the legacy desired- +// state query (generateDesiredStateQuery): platform in (darwin, ios, ipados), +// an enabled nano_enrollment of type Device or "User Enrollment (Device)", +// and an existing nano_devices row supplying authenticate_at. +func (ds *Datastore) ListAppleMDMHostsForReconcileBatch( + ctx context.Context, + afterHostUUID string, + batchSize int, +) ([]*fleet.AppleHostReconcileInfo, error) { + const stmt = ` + SELECT + h.id AS id, + h.uuid AS uuid, + h.team_id AS team_id, + h.platform AS platform, + h.label_updated_at AS label_updated_at, + nd.authenticate_at AS device_enrolled_at + FROM hosts h + JOIN nano_enrollments ne + ON ne.device_id = h.uuid + AND ne.enabled = 1 + AND ne.type IN ('Device', 'User Enrollment (Device)') + JOIN nano_devices nd + ON nd.id = ne.device_id + WHERE + (h.platform = 'darwin' OR h.platform = 'ios' OR h.platform = 'ipados') + AND h.uuid > ? + ORDER BY h.uuid + LIMIT ? + ` + + var hosts []*fleet.AppleHostReconcileInfo + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hosts, stmt, afterHostUUID, batchSize); err != nil { + return nil, ctxerr.Wrap(ctx, err, "list apple mdm hosts for reconcile batch") + } + return hosts, nil +} + +// ListAppleProfilesForReconcile loads every Apple configuration profile in +// the system, paired with its label assignments. The result is intended to +// be loaded once per reconciliation tick and used to evaluate desired state +// per host in memory. +// +// Label assignments include broken labels (label_id IS NULL) so the +// in-memory handlers can apply the same "broken-label" semantics as the +// legacy SQL: broken include-* profiles do not apply, and broken profiles +// are exempted from removal. +func (ds *Datastore) ListAppleProfilesForReconcile(ctx context.Context) ([]*fleet.AppleProfileForReconcile, error) { + type profileRow struct { + ProfileUUID string `db:"profile_uuid"` + ProfileIdentifier string `db:"identifier"` + ProfileName string `db:"name"` + TeamID uint `db:"team_id"` + Checksum []byte `db:"checksum"` + SecretsUpdatedAt sql.NullTime `db:"secrets_updated_at"` + Scope fleet.PayloadScope `db:"scope"` + } + + const profStmt = ` + SELECT profile_uuid, identifier, name, team_id, checksum, secrets_updated_at, scope + FROM mdm_apple_configuration_profiles + ` + + var rows []profileRow + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rows, profStmt); err != nil { + return nil, ctxerr.Wrap(ctx, err, "list apple profiles for reconcile") + } + if len(rows) == 0 { + return nil, nil + } + + byUUID := make(map[string]*fleet.AppleProfileForReconcile, len(rows)) + out := make([]*fleet.AppleProfileForReconcile, 0, len(rows)) + for _, r := range rows { + p := &fleet.AppleProfileForReconcile{ + ProfileUUID: r.ProfileUUID, + ProfileIdentifier: r.ProfileIdentifier, + ProfileName: r.ProfileName, + TeamID: r.TeamID, + Checksum: r.Checksum, + Scope: r.Scope, + } + if r.SecretsUpdatedAt.Valid { + t := r.SecretsUpdatedAt.Time + p.SecretsUpdatedAt = &t + } + byUUID[r.ProfileUUID] = p + out = append(out, p) + } + + // Load label assignments, joining labels to get membership type and + // label creation time (needed by the exclude-any handler). + const labelStmt = ` + SELECT + mcpl.apple_profile_uuid AS profile_uuid, + mcpl.label_id AS label_id, + mcpl.exclude AS exclude, + mcpl.require_all AS require_all, + COALESCE(lbl.created_at, '2000-01-01 00:00:00') AS label_created_at, + COALESCE(lbl.label_membership_type, 0) AS label_membership_type + FROM mdm_configuration_profile_labels mcpl + LEFT JOIN labels lbl ON lbl.id = mcpl.label_id + WHERE mcpl.apple_profile_uuid IS NOT NULL + ` + + type labelRow struct { + ProfileUUID string `db:"profile_uuid"` + LabelID sql.NullInt64 `db:"label_id"` + Exclude bool `db:"exclude"` + RequireAll bool `db:"require_all"` + LabelCreatedAt sql.NullTime `db:"label_created_at"` + LabelMembershipType int `db:"label_membership_type"` + } + + var labelRows []labelRow + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &labelRows, labelStmt); err != nil { + return nil, ctxerr.Wrap(ctx, err, "list apple profile labels for reconcile") + } + + // Per-profile label mode discovery. A profile's mode is set by the first + // label row seen; if later rows disagree we mark it mixed and treat the + // profile as no-label (defensive — upsert enforces consistency). + type modeMarker struct { + set bool + mode fleet.AppleProfileLabelMode + mixed bool + } + modes := make(map[string]*modeMarker, len(byUUID)) + + for _, lr := range labelRows { + p, ok := byUUID[lr.ProfileUUID] + if !ok { + continue + } + + mm := modes[lr.ProfileUUID] + if mm == nil { + mm = &modeMarker{} + modes[lr.ProfileUUID] = mm + } + + var rowMode fleet.AppleProfileLabelMode + switch { + case lr.Exclude: + rowMode = fleet.AppleProfileLabelModeExcludeAny + case lr.RequireAll: + rowMode = fleet.AppleProfileLabelModeIncludeAll + default: + rowMode = fleet.AppleProfileLabelModeIncludeAny + } + + if !mm.set { + mm.mode = rowMode + mm.set = true + } else if mm.mode != rowMode { + mm.mixed = true + } + + ref := fleet.AppleProfileLabelRef{ + LabelMembershipType: lr.LabelMembershipType, + } + if lr.LabelID.Valid { + id := uint(lr.LabelID.Int64) //nolint:gosec // dismiss G115: labels.id is int unsigned in MySQL + ref.LabelID = &id + } + if lr.LabelCreatedAt.Valid { + ref.CreatedAt = lr.LabelCreatedAt.Time + } + p.Labels = append(p.Labels, ref) + } + + for uuid, mm := range modes { + p := byUUID[uuid] + if mm.mixed { + p.LabelMode = fleet.AppleProfileLabelModeNone + p.Labels = nil + continue + } + p.LabelMode = mm.mode + } + + return out, nil +} + +// BulkGetHostLabelMemberships returns, for each given host ID, the set of +// label IDs (from the provided labelIDs) the host is a member of. +// +// Both lists may be empty; in either case the result is an empty (non-nil) +// map. The IN clauses are chunked to keep total placeholders well under +// MySQL's prepared-statement parameter limit. +func (ds *Datastore) BulkGetHostLabelMemberships( + ctx context.Context, + hostIDs []uint, + labelIDs []uint, +) (map[uint]map[uint]struct{}, error) { + out := make(map[uint]map[uint]struct{}, len(hostIDs)) + if len(hostIDs) == 0 || len(labelIDs) == 0 { + return out, nil + } + + const ( + hostChunk = 5000 + labelChunk = 1000 + ) + + stmt := `SELECT host_id, label_id FROM label_membership WHERE host_id IN (?) AND label_id IN (?)` + + for hi := 0; hi < len(hostIDs); hi += hostChunk { + hEnd := min(hi+hostChunk, len(hostIDs)) + hostBatch := hostIDs[hi:hEnd] + + for li := 0; li < len(labelIDs); li += labelChunk { + lEnd := min(li+labelChunk, len(labelIDs)) + labelBatch := labelIDs[li:lEnd] + + q, args, err := sqlx.In(stmt, hostBatch, labelBatch) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "build label membership query") + } + + rows, err := ds.reader(ctx).QueryxContext(ctx, q, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "query host label memberships") + } + + for rows.Next() { + var hostID, labelID uint + if err := rows.Scan(&hostID, &labelID); err != nil { + rows.Close() + return nil, ctxerr.Wrap(ctx, err, "scan label membership row") + } + set, ok := out[hostID] + if !ok { + set = make(map[uint]struct{}) + out[hostID] = set + } + set[labelID] = struct{}{} + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, ctxerr.Wrap(ctx, err, "iterate label membership rows") + } + rows.Close() + } + } + + return out, nil +} + +// BulkGetHostMDMAppleProfilesByUUIDs returns the current host_mdm_apple_profiles +// rows for the given host UUIDs, grouped by host UUID. +// +// The returned MDMAppleProfilePayload fields mirror what the legacy +// listMDMAppleProfilesToRemoveTransaction returns. HostPlatform and +// DeviceEnrolledAt are left zero because they come from joined tables the +// in-memory reconciler already has from ListAppleMDMHostsForReconcileBatch. +func (ds *Datastore) BulkGetHostMDMAppleProfilesByUUIDs( + ctx context.Context, + hostUUIDs []string, +) (map[string][]*fleet.MDMAppleProfilePayload, error) { + out := make(map[string][]*fleet.MDMAppleProfilePayload, len(hostUUIDs)) + if len(hostUUIDs) == 0 { + return out, nil + } + + const stmt = ` + SELECT + profile_uuid, + profile_identifier, + profile_name, + host_uuid, + checksum, + secrets_updated_at, + status, + operation_type, + COALESCE(detail, '') AS detail, + command_uuid, + ignore_error, + scope + FROM host_mdm_apple_profiles + WHERE host_uuid IN (?) + ` + + const chunk = 5000 + + for i := 0; i < len(hostUUIDs); i += chunk { + end := min(i+chunk, len(hostUUIDs)) + batch := hostUUIDs[i:end] + + q, args, err := sqlx.In(stmt, batch) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "build host mdm apple profiles query") + } + + var rows []*fleet.MDMAppleProfilePayload + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rows, q, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "select host mdm apple profiles") + } + + for _, r := range rows { + out[r.HostUUID] = append(out[r.HostUUID], r) + } + } + + return out, nil +} + +// GetMDMAppleReconcileCursor returns the persisted host_uuid cursor used by +// the batched Apple MDM reconciliation cron. The bare mysql.Datastore has no +// place to persist it, so this returns "" (fresh start). The mysqlredis +// wrapper overrides this to back it with Redis. +func (ds *Datastore) GetMDMAppleReconcileCursor(_ context.Context) (string, error) { + return "", nil +} + +// SetMDMAppleReconcileCursor persists the host_uuid cursor used by the +// batched Apple MDM reconciliation cron. The bare mysql.Datastore is a +// no-op; the mysqlredis wrapper backs it with Redis. +func (ds *Datastore) SetMDMAppleReconcileCursor(_ context.Context, _ string) error { + return nil +} diff --git a/server/datastore/mysqlredis/apple_recon_cursor.go b/server/datastore/mysqlredis/apple_recon_cursor.go new file mode 100644 index 00000000000..aee0c4db96c --- /dev/null +++ b/server/datastore/mysqlredis/apple_recon_cursor.go @@ -0,0 +1,49 @@ +package mysqlredis + +import ( + "context" + "errors" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/redis" + redigo "github.com/gomodule/redigo/redis" +) + +// appleReconCursorKey is the Redis key holding the host_uuid cursor for the +// batched Apple MDM profile reconciler. +const appleReconCursorKey = "mdm:apple:recon_cursor" + +// GetMDMAppleReconcileCursor returns the persisted host_uuid cursor used by +// the batched Apple MDM reconciliation cron to bound per-tick work. +// +// Returns "" if the key is unset (fresh deployment, Redis flushed, or full +// pass complete). Loss of this key is harmless: the cron resumes from the +// beginning. The desired-state diff is recomputed every tick, so re- +// processing converges naturally. +func (d *Datastore) GetMDMAppleReconcileCursor(ctx context.Context) (string, error) { + conn := redis.ConfigureDoer(d.pool, d.pool.Get()) + defer conn.Close() + + cursor, err := redigo.String(conn.Do("GET", appleReconCursorKey)) + switch { + case err == nil: + return cursor, nil + case errors.Is(err, redigo.ErrNil): + return "", nil + default: + return "", ctxerr.Wrap(ctx, err, "get apple MDM reconcile cursor") + } +} + +// SetMDMAppleReconcileCursor persists the host_uuid cursor used by the +// batched Apple MDM reconciliation cron. An empty string indicates a full +// pass has completed; the next tick will start from the beginning. +func (d *Datastore) SetMDMAppleReconcileCursor(ctx context.Context, cursor string) error { + conn := redis.ConfigureDoer(d.pool, d.pool.Get()) + defer conn.Close() + + if _, err := conn.Do("SET", appleReconCursorKey, cursor); err != nil { + return ctxerr.Wrap(ctx, err, "set apple MDM reconcile cursor") + } + return nil +} diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 61ae3b21101..549e2a5fa76 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -439,6 +439,79 @@ type MDMAppleBulkUpsertHostProfilePayload struct { Scope PayloadScope } +// AppleHostReconcileInfo is a per-host record used by the batched Apple +// profile reconciler. It contains only the fields needed to decide which +// profiles should be installed on the host given its team, platform, and +// label membership. +type AppleHostReconcileInfo struct { + HostID uint `db:"id"` + UUID string `db:"uuid"` + TeamID *uint `db:"team_id"` + Platform string `db:"platform"` + LabelUpdatedAt time.Time `db:"label_updated_at"` + DeviceEnrolledAt *time.Time `db:"device_enrolled_at"` +} + +// EffectiveTeamID returns 0 for hosts not in a team (matching how Apple +// MDM profiles are stored, where team_id=0 means "no team / global"). +func (h *AppleHostReconcileInfo) EffectiveTeamID() uint { + if h.TeamID == nil { + return 0 + } + return *h.TeamID +} + +// AppleProfileLabelMode indicates how a profile's label assignments gate +// applicability to a host. Profiles never mix modes — a single profile is +// either no-labels, include-all, include-any, or exclude-any. +type AppleProfileLabelMode int + +const ( + AppleProfileLabelModeNone AppleProfileLabelMode = iota + AppleProfileLabelModeIncludeAll + AppleProfileLabelModeIncludeAny + AppleProfileLabelModeExcludeAny +) + +// AppleProfileLabelRef is a single label reference attached to a profile. +// A nil LabelID means the label was deleted (the assignment is "broken"). +type AppleProfileLabelRef struct { + LabelID *uint + CreatedAt time.Time + // LabelMembershipType mirrors labels.label_membership_type: 0=dynamic, + // 1=manual. Needed by the exclude-any handler so dynamic labels that + // were created after a host's last label_updated_at are treated as + // "results not yet reported" instead of "host is not a member". + LabelMembershipType int +} + +// AppleProfileForReconcile is the profile data needed by the batched +// reconciler to compute desired state per host in memory. +type AppleProfileForReconcile struct { + ProfileUUID string + ProfileIdentifier string + ProfileName string + TeamID uint // 0 means global + Checksum []byte + SecretsUpdatedAt *time.Time + Scope PayloadScope + LabelMode AppleProfileLabelMode + Labels []AppleProfileLabelRef +} + +// HasBrokenLabel reports whether any of the profile's label assignments +// reference a deleted label. Broken include-* profiles are excluded from +// desired state; broken exclude-any profiles are also excluded (never apply). +// Broken profiles are also exempt from removal in the existing flow. +func (p *AppleProfileForReconcile) HasBrokenLabel() bool { + for _, l := range p.Labels { + if l.LabelID == nil { + return true + } + } + return false +} + // MDMAppleFileVaultSummary reports the number of macOS hosts being managed with Apples disk // encryption profiles. Each host may be counted in only one of six mutually-exclusive categories: // Verified, Verifying, ActionRequired, Enforcing, Failed, RemovingEnforcement. diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 649a45db547..e2f2675b721 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -2218,6 +2218,42 @@ type Datastore interface { // the Windows MDM reconciliation cron. See GetMDMWindowsReconcileCursor. SetMDMWindowsReconcileCursor(ctx context.Context, cursor string) error + // ListAppleMDMHostsForReconcileBatch returns up to batchSize Apple MDM- + // enrolled hosts (host_uuid > afterHostUUID, ordered ascending) with the + // fields needed by the batched Apple profile reconciler to compute + // desired state in memory. Used by ReconcileAppleProfilesBatched. + ListAppleMDMHostsForReconcileBatch(ctx context.Context, afterHostUUID string, batchSize int) ([]*AppleHostReconcileInfo, error) + + // ListAppleProfilesForReconcile returns every Apple configuration + // profile in the system along with its label assignments. The result is + // used by the batched reconciler to evaluate desired state per host in + // memory. Returned profiles are intended to be small relative to the + // host population and are loaded once per tick. + ListAppleProfilesForReconcile(ctx context.Context) ([]*AppleProfileForReconcile, error) + + // BulkGetHostLabelMemberships returns the subset of (hostID, labelID) + // pairs from label_membership that are present, restricted to the + // provided host IDs and label IDs. The outer map is keyed by host ID and + // the inner set holds the label IDs the host is a member of. + BulkGetHostLabelMemberships(ctx context.Context, hostIDs []uint, labelIDs []uint) (map[uint]map[uint]struct{}, error) + + // BulkGetHostMDMAppleProfilesByUUIDs returns the current host_mdm_apple_profiles + // rows for the given host UUIDs, grouped by host UUID. Used by the + // batched reconciler to compute install/remove deltas against the + // in-memory desired state. + BulkGetHostMDMAppleProfilesByUUIDs(ctx context.Context, hostUUIDs []string) (map[string][]*MDMAppleProfilePayload, error) + + // GetMDMAppleReconcileCursor returns the persisted host_uuid cursor + // used by the batched Apple MDM reconciliation cron to bound per-tick + // work. Returns "" if no cursor is set or if the implementation does + // not support cursor persistence (the bare mysql.Datastore returns "" + // here; the mysqlredis wrapper backs it with Redis). + GetMDMAppleReconcileCursor(ctx context.Context) (string, error) + + // SetMDMAppleReconcileCursor persists the host_uuid cursor used by the + // batched Apple MDM reconciliation cron. + SetMDMAppleReconcileCursor(ctx context.Context, cursor string) error + // BulkUpsertMDMWindowsHostProfiles bulk-adds/updates records to track the // status of a profile in a host. BulkUpsertMDMWindowsHostProfiles(ctx context.Context, payload []*MDMWindowsBulkUpsertHostProfilePayload) error diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 581fb88997e..403cf2adac8 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1369,6 +1369,18 @@ type GetMDMWindowsReconcileCursorFunc func(ctx context.Context) (string, error) type SetMDMWindowsReconcileCursorFunc func(ctx context.Context, cursor string) error +type ListAppleMDMHostsForReconcileBatchFunc func(ctx context.Context, afterHostUUID string, batchSize int) ([]*fleet.AppleHostReconcileInfo, error) + +type ListAppleProfilesForReconcileFunc func(ctx context.Context) ([]*fleet.AppleProfileForReconcile, error) + +type BulkGetHostLabelMembershipsFunc func(ctx context.Context, hostIDs []uint, labelIDs []uint) (map[uint]map[uint]struct{}, error) + +type BulkGetHostMDMAppleProfilesByUUIDsFunc func(ctx context.Context, hostUUIDs []string) (map[string][]*fleet.MDMAppleProfilePayload, error) + +type GetMDMAppleReconcileCursorFunc func(ctx context.Context) (string, error) + +type SetMDMAppleReconcileCursorFunc func(ctx context.Context, cursor string) error + type BulkUpsertMDMWindowsHostProfilesFunc func(ctx context.Context, payload []*fleet.MDMWindowsBulkUpsertHostProfilePayload) error type GetMDMWindowsProfilesContentsFunc func(ctx context.Context, profileUUIDs []string) (map[string]fleet.MDMWindowsProfileContents, error) @@ -3991,6 +4003,24 @@ type DataStore struct { SetMDMWindowsReconcileCursorFunc SetMDMWindowsReconcileCursorFunc SetMDMWindowsReconcileCursorFuncInvoked bool + ListAppleMDMHostsForReconcileBatchFunc ListAppleMDMHostsForReconcileBatchFunc + ListAppleMDMHostsForReconcileBatchFuncInvoked bool + + ListAppleProfilesForReconcileFunc ListAppleProfilesForReconcileFunc + ListAppleProfilesForReconcileFuncInvoked bool + + BulkGetHostLabelMembershipsFunc BulkGetHostLabelMembershipsFunc + BulkGetHostLabelMembershipsFuncInvoked bool + + BulkGetHostMDMAppleProfilesByUUIDsFunc BulkGetHostMDMAppleProfilesByUUIDsFunc + BulkGetHostMDMAppleProfilesByUUIDsFuncInvoked bool + + GetMDMAppleReconcileCursorFunc GetMDMAppleReconcileCursorFunc + GetMDMAppleReconcileCursorFuncInvoked bool + + SetMDMAppleReconcileCursorFunc SetMDMAppleReconcileCursorFunc + SetMDMAppleReconcileCursorFuncInvoked bool + BulkUpsertMDMWindowsHostProfilesFunc BulkUpsertMDMWindowsHostProfilesFunc BulkUpsertMDMWindowsHostProfilesFuncInvoked bool @@ -9608,6 +9638,48 @@ func (s *DataStore) SetMDMWindowsReconcileCursor(ctx context.Context, cursor str return s.SetMDMWindowsReconcileCursorFunc(ctx, cursor) } +func (s *DataStore) ListAppleMDMHostsForReconcileBatch(ctx context.Context, afterHostUUID string, batchSize int) ([]*fleet.AppleHostReconcileInfo, error) { + s.mu.Lock() + s.ListAppleMDMHostsForReconcileBatchFuncInvoked = true + s.mu.Unlock() + return s.ListAppleMDMHostsForReconcileBatchFunc(ctx, afterHostUUID, batchSize) +} + +func (s *DataStore) ListAppleProfilesForReconcile(ctx context.Context) ([]*fleet.AppleProfileForReconcile, error) { + s.mu.Lock() + s.ListAppleProfilesForReconcileFuncInvoked = true + s.mu.Unlock() + return s.ListAppleProfilesForReconcileFunc(ctx) +} + +func (s *DataStore) BulkGetHostLabelMemberships(ctx context.Context, hostIDs []uint, labelIDs []uint) (map[uint]map[uint]struct{}, error) { + s.mu.Lock() + s.BulkGetHostLabelMembershipsFuncInvoked = true + s.mu.Unlock() + return s.BulkGetHostLabelMembershipsFunc(ctx, hostIDs, labelIDs) +} + +func (s *DataStore) BulkGetHostMDMAppleProfilesByUUIDs(ctx context.Context, hostUUIDs []string) (map[string][]*fleet.MDMAppleProfilePayload, error) { + s.mu.Lock() + s.BulkGetHostMDMAppleProfilesByUUIDsFuncInvoked = true + s.mu.Unlock() + return s.BulkGetHostMDMAppleProfilesByUUIDsFunc(ctx, hostUUIDs) +} + +func (s *DataStore) GetMDMAppleReconcileCursor(ctx context.Context) (string, error) { + s.mu.Lock() + s.GetMDMAppleReconcileCursorFuncInvoked = true + s.mu.Unlock() + return s.GetMDMAppleReconcileCursorFunc(ctx) +} + +func (s *DataStore) SetMDMAppleReconcileCursor(ctx context.Context, cursor string) error { + s.mu.Lock() + s.SetMDMAppleReconcileCursorFuncInvoked = true + s.mu.Unlock() + return s.SetMDMAppleReconcileCursorFunc(ctx, cursor) +} + func (s *DataStore) BulkUpsertMDMWindowsHostProfiles(ctx context.Context, payload []*fleet.MDMWindowsBulkUpsertHostProfilePayload) error { s.mu.Lock() s.BulkUpsertMDMWindowsHostProfilesFuncInvoked = true diff --git a/server/service/apple_mdm_batched.go b/server/service/apple_mdm_batched.go new file mode 100644 index 00000000000..bb70a678f5b --- /dev/null +++ b/server/service/apple_mdm_batched.go @@ -0,0 +1,831 @@ +package service + +import ( + "bytes" + "context" + "encoding/pem" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple" + "github.com/fleetdm/fleet/v4/server/mdm/apple/mobileconfig" + "github.com/fleetdm/fleet/v4/server/variables" + "github.com/google/uuid" +) + +// reconcileAppleProfilesBatchSize bounds how many distinct hosts the +// batched Apple MDM reconciliation cron processes per tick. The cron uses +// a host_uuid cursor (persisted in Redis via the mysqlredis wrapper) to +// page through the host universe in batches, smoothing the writer pressure +// that the legacy unbounded reconciliation generates during bulk events +// (team transfers, profile changes). +// +// var (not const) so tests can override it. +var reconcileAppleProfilesBatchSize = 5000 + +// ReconcileAppleProfilesBatched is an alternative implementation of +// ReconcileAppleProfiles that: +// +// - Pulls a bounded batch of Apple-MDM-enrolled host UUIDs each tick +// (via a host_uuid cursor persisted in Redis), instead of computing +// desired-vs-current state across the entire host population in one +// giant SQL UNION. +// - Loads label memberships, team membership, and the profile catalog +// for the batch and evaluates per-host desired state in memory using +// small, single-responsibility handlers (one per label mode). +// - Computes the install/remove delta against the host's current +// host_mdm_apple_profiles rows in memory, then feeds the deltas into +// the existing ProcessAndEnqueueProfiles path so all of the variable +// substitution, CA throttle, user-enrollment, and "host being set up" +// handling stays identical. +// +// The function is intended to be A/B-tested against ReconcileAppleProfiles +// via FLEET_MDM_APPLE_BATCHED_RECONCILER=true. When the cursor reaches the +// end of the universe it is reset so the next tick starts from the +// beginning. The returned error is non-nil only for fatal-to-this-tick +// failures; per-host issues are logged and skipped. +func ReconcileAppleProfilesBatched( + ctx context.Context, + ds fleet.Datastore, + commander *apple_mdm.MDMAppleCommander, + redisKeyValue fleet.AdvancedKeyValueStore, + logger *slog.Logger, + certProfilesLimit int, +) (err error) { + appConfig, err := ds.AppConfig(ctx) + if err != nil { + return fmt.Errorf("reading app config: %w", err) + } + if !appConfig.MDM.EnabledAndConfigured { + return nil + } + + assets, err := ds.GetAllMDMConfigAssetsByName(ctx, []fleet.MDMAssetName{ + fleet.MDMAssetCACert, + }, nil) + if err != nil { + return ctxerr.Wrap(ctx, err, "getting Apple SCEP") + } + block, _ := pem.Decode(assets[fleet.MDMAssetCACert].Value) + if block == nil || block.Type != "CERTIFICATE" { + return ctxerr.Wrap(ctx, errors.New("failed to decode PEM block from SCEP certificate"), "") + } + if err := ensureFleetProfiles(ctx, ds, logger, block.Bytes); err != nil { + logger.ErrorContext(ctx, "unable to ensure fleetd configuration profiles are in place", "details", err) + } + + // Read cursor (treat any error as a fresh start). + cursor, err := ds.GetMDMAppleReconcileCursor(ctx) + if err != nil { + logger.WarnContext(ctx, "failed to read apple MDM reconcile cursor; starting from beginning", "err", err) + cursor = "" + } + + hosts, err := ds.ListAppleMDMHostsForReconcileBatch(ctx, cursor, reconcileAppleProfilesBatchSize) + if err != nil { + return ctxerr.Wrap(ctx, err, "listing apple MDM hosts for reconcile batch") + } + + if len(hosts) == 0 { + // End of pass or empty universe — reset cursor if it was advanced. + if cursor != "" { + logger.InfoContext(ctx, "apple MDM reconcile pass complete; resetting cursor", "cursor", cursor) + if cerr := ds.SetMDMAppleReconcileCursor(ctx, ""); cerr != nil { + logger.WarnContext(ctx, "failed to reset apple MDM reconcile cursor", "err", cerr) + } + } + return nil + } + + // Compute next cursor before doing the work so we can advance on + // success. If we got fewer than the batch size, the universe fits in + // this tick and the next tick should restart at "". + var nextCursor string + if len(hosts) >= reconcileAppleProfilesBatchSize { + nextCursor = hosts[len(hosts)-1].UUID + } + + // Defer cursor advance on success only; on error we leave the cursor + // untouched so the next tick retries the same window. + defer func() { + if err == nil && cursor != nextCursor { + if cerr := ds.SetMDMAppleReconcileCursor(ctx, nextCursor); cerr != nil { + logger.WarnContext(ctx, "failed to advance apple MDM reconcile cursor", "err", cerr) + } + } + }() + + if cursor != "" || nextCursor != "" { + logger.InfoContext(ctx, "apple MDM reconcile tick using cursor", + "cursor", cursor, + "next_cursor", nextCursor, + "batch_size", reconcileAppleProfilesBatchSize, + "hosts_in_batch", len(hosts), + ) + } + + // Load all Apple profiles + their labels (one query per tick). Profile + // counts are small relative to host counts, so we don't bother + // paginating these. + allProfiles, err := ds.ListAppleProfilesForReconcile(ctx) + if err != nil { + return ctxerr.Wrap(ctx, err, "listing apple profiles for reconcile") + } + + // Group profiles by team_id (0 = global). Each host evaluates only + // global + its team's profiles. + profilesByTeam := make(map[uint][]*fleet.AppleProfileForReconcile, 4) + labelIDSet := make(map[uint]struct{}) + for _, p := range allProfiles { + profilesByTeam[p.TeamID] = append(profilesByTeam[p.TeamID], p) + for _, lr := range p.Labels { + if lr.LabelID != nil { + labelIDSet[*lr.LabelID] = struct{}{} + } + } + } + + // Collect host IDs and UUIDs for the batch. + hostIDs := make([]uint, 0, len(hosts)) + hostUUIDs := make([]string, 0, len(hosts)) + hostsByUUID := make(map[string]*fleet.AppleHostReconcileInfo, len(hosts)) + for _, h := range hosts { + hostIDs = append(hostIDs, h.HostID) + hostUUIDs = append(hostUUIDs, h.UUID) + hostsByUUID[h.UUID] = h + } + + // Collect label IDs we actually care about. + labelIDs := make([]uint, 0, len(labelIDSet)) + for id := range labelIDSet { + labelIDs = append(labelIDs, id) + } + + // Load host-label memberships and current host_mdm_apple_profiles rows + // for the batch. + hostLabels, err := ds.BulkGetHostLabelMemberships(ctx, hostIDs, labelIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "bulk get host label memberships") + } + + currentByHost, err := ds.BulkGetHostMDMAppleProfilesByUUIDs(ctx, hostUUIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "bulk get host mdm apple profiles") + } + + // Compute desired state per host and diff against current. + toInstall, toRemove := computeAppleReconcileDeltas(hosts, hostLabels, currentByHost, profilesByTeam) + + // Filter out macOS-only profiles from iOS/iPadOS hosts (matches legacy). + toInstall = fleet.FilterMacOSOnlyProfilesFromIOSIPadOS(toInstall) + + if len(toInstall) == 0 && len(toRemove) == 0 { + // Nothing to do this tick. Still advance the cursor. + return nil + } + + // From here on, everything mirrors the legacy ReconcileAppleProfiles + // post-listing logic so all downstream behaviour (CA throttle, user- + // enrollment fallback, host-being-set-up skip, retry on failed enqueue) + // is identical. + return executeAppleReconcileBatch( + ctx, ds, commander, redisKeyValue, logger, + appConfig, certProfilesLimit, toInstall, toRemove, + ) +} + +// computeAppleReconcileDeltas evaluates desired state for each host in the +// batch using the in-code label handlers, then diffs against the host's +// current host_mdm_apple_profiles rows to produce install and remove sets. +// +// Package-private so tests can exercise the in-memory logic without a +// database round-trip. +func computeAppleReconcileDeltas( + hosts []*fleet.AppleHostReconcileInfo, + hostLabels map[uint]map[uint]struct{}, + currentByHost map[string][]*fleet.MDMAppleProfilePayload, + profilesByTeam map[uint][]*fleet.AppleProfileForReconcile, +) (toInstall, toRemove []*fleet.MDMAppleProfilePayload) { + for _, host := range hosts { + // Build the host's desired profile set by running each applicable + // profile through the appropriate label handler. + teamProfiles := profilesByTeam[host.EffectiveTeamID()] + desired := make(map[string]*fleet.AppleProfileForReconcile, len(teamProfiles)) + + labelsForHost := hostLabels[host.HostID] // may be nil + + for _, p := range teamProfiles { + if !appleProfileAppliesToHost(p, host, labelsForHost) { + continue + } + desired[p.ProfileUUID] = p + } + + current := currentByHost[host.UUID] // []*MDMAppleProfilePayload, may be nil + currentByProfile := make(map[string]*fleet.MDMAppleProfilePayload, len(current)) + for _, c := range current { + currentByProfile[c.ProfileUUID] = c + } + + // INSTALL set: desired profiles where (no current row) OR + // (checksum differs) OR (secrets_updated_at advanced) OR + // (current operation_type is remove OR NULL) OR + // (current operation_type is install with NULL status). + for profUUID, p := range desired { + c, present := currentByProfile[profUUID] + needsInstall := false + switch { + case !present: + needsInstall = true + case !bytes.Equal(c.Checksum, p.Checksum): + needsInstall = true + case p.SecretsUpdatedAt != nil && (c.SecretsUpdatedAt == nil || c.SecretsUpdatedAt.Before(*p.SecretsUpdatedAt)): + needsInstall = true + case c.OperationType == "" || c.OperationType == fleet.MDMOperationTypeRemove: + needsInstall = true + case c.OperationType == fleet.MDMOperationTypeInstall && c.Status == nil: + needsInstall = true + } + if !needsInstall { + continue + } + + toInstall = append(toInstall, &fleet.MDMAppleProfilePayload{ + ProfileUUID: p.ProfileUUID, + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + HostUUID: host.UUID, + HostPlatform: host.Platform, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + Scope: p.Scope, + DeviceEnrolledAt: host.DeviceEnrolledAt, + }) + } + + // REMOVE set: current rows whose profile is no longer in desired, + // excluding rows whose current state is already a remove in a + // terminal/pending state, and excluding rows for broken + // label-based profiles (those linger by design). + for profUUID, c := range currentByProfile { + if _, stillDesired := desired[profUUID]; stillDesired { + continue + } + + // Match legacy "except remove operations in a terminal state + // or already pending" — skip if operation is remove with a + // non-NULL status. + if c.OperationType == fleet.MDMOperationTypeRemove && c.Status != nil { + continue + } + + // Skip broken label-based profiles (preserve legacy behavior: + // broken profiles are never removed automatically). + if isBrokenAppleProfile(profUUID, profilesByTeam) { + continue + } + + toRemove = append(toRemove, &fleet.MDMAppleProfilePayload{ + ProfileUUID: c.ProfileUUID, + ProfileIdentifier: c.ProfileIdentifier, + ProfileName: c.ProfileName, + HostUUID: host.UUID, + HostPlatform: host.Platform, + Checksum: c.Checksum, + SecretsUpdatedAt: c.SecretsUpdatedAt, + Status: c.Status, + OperationType: c.OperationType, + Detail: c.Detail, + CommandUUID: c.CommandUUID, + IgnoreError: c.IgnoreError, + Scope: c.Scope, + DeviceEnrolledAt: host.DeviceEnrolledAt, + }) + } + } + + return toInstall, toRemove +} + +// appleProfileAppliesToHost is the top-level handler dispatcher. It applies +// the team and platform gates, then routes to the per-label-mode handler. +func appleProfileAppliesToHost( + p *fleet.AppleProfileForReconcile, + host *fleet.AppleHostReconcileInfo, + hostLabels map[uint]struct{}, +) bool { + // Team gate (already filtered upstream by profilesByTeam, but + // double-check for safety so handlers stay composable). + if p.TeamID != host.EffectiveTeamID() { + return false + } + + // Platform gate. Profiles targeting macOS-only platforms are filtered + // later in FilterMacOSOnlyProfilesFromIOSIPadOS so we don't reproduce + // that here. + if !isAppleProfileEligiblePlatform(host.Platform) { + return false + } + + switch p.LabelMode { + case fleet.AppleProfileLabelModeNone: + return appleProfileHandlerNoLabels(p) + case fleet.AppleProfileLabelModeIncludeAll: + return appleProfileHandlerIncludeAll(p, hostLabels) + case fleet.AppleProfileLabelModeIncludeAny: + return appleProfileHandlerIncludeAny(p, hostLabels) + case fleet.AppleProfileLabelModeExcludeAny: + return appleProfileHandlerExcludeAny(p, host, hostLabels) + default: + return false + } +} + +func isAppleProfileEligiblePlatform(platform string) bool { + return platform == "darwin" || platform == "ios" || platform == "ipados" +} + +// appleProfileHandlerNoLabels: no labels → always applies (subject to +// outer team/platform gates). +func appleProfileHandlerNoLabels(_ *fleet.AppleProfileForReconcile) bool { + return true +} + +// appleProfileHandlerIncludeAll: host must be a member of every (non- +// broken) referenced label. If any label is broken, the profile does +// not apply (mirrors the legacy SQL where include-* with a broken label +// produces no desired-state row). +func appleProfileHandlerIncludeAll(p *fleet.AppleProfileForReconcile, hostLabels map[uint]struct{}) bool { + if len(p.Labels) == 0 { + return false + } + for _, l := range p.Labels { + if l.LabelID == nil { + return false + } + if _, ok := hostLabels[*l.LabelID]; !ok { + return false + } + } + return true +} + +// appleProfileHandlerIncludeAny: host must be a member of at least one +// referenced label. Broken labels can't match (host can't be a member of a +// deleted label). +func appleProfileHandlerIncludeAny(p *fleet.AppleProfileForReconcile, hostLabels map[uint]struct{}) bool { + for _, l := range p.Labels { + if l.LabelID == nil { + continue + } + if _, ok := hostLabels[*l.LabelID]; ok { + return true + } + } + return false +} + +// appleProfileHandlerExcludeAny: profile applies when the host is NOT a +// member of any referenced label, with two extra rules from the legacy +// SQL: +// +// - Broken labels disqualify the profile entirely (never apply). +// - For dynamic labels, we require host.label_updated_at >= label.created_at +// so we don't treat "label results not yet reported" as "not a member". +// Manual labels (membership_type=1) skip this check. +func appleProfileHandlerExcludeAny( + p *fleet.AppleProfileForReconcile, + host *fleet.AppleHostReconcileInfo, + hostLabels map[uint]struct{}, +) bool { + if len(p.Labels) == 0 { + return false + } + for _, l := range p.Labels { + if l.LabelID == nil { + return false + } + // If the host's label scan hasn't caught up with the label's + // creation, we can't trust "not a member" — skip the profile. + if l.LabelMembershipType != 1 && !l.CreatedAt.IsZero() && host.LabelUpdatedAt.Before(l.CreatedAt) { + return false + } + if _, isMember := hostLabels[*l.LabelID]; isMember { + return false + } + } + return true +} + +// isBrokenAppleProfile returns true if any label assignment on the profile +// references a deleted label. Used to skip removal of broken profiles +// (matches legacy behavior). +func isBrokenAppleProfile(profileUUID string, profilesByTeam map[uint][]*fleet.AppleProfileForReconcile) bool { + for _, ps := range profilesByTeam { + for _, p := range ps { + if p.ProfileUUID != profileUUID { + continue + } + return p.HasBrokenLabel() + } + } + // Profile not found in catalog at all — the row in host_mdm_apple_profiles + // is for a profile that has been deleted from the team or globally. It + // is safe to remove (it is NOT a "broken label" case). + return false +} + +// executeAppleReconcileBatch runs the legacy post-listing logic against the +// in-memory toInstall/toRemove sets produced by the batched listing path. +// It is intentionally a near-clone of the corresponding section of +// ReconcileAppleProfiles so that semantic parity is easy to audit. +func executeAppleReconcileBatch( + ctx context.Context, + ds fleet.Datastore, + commander *apple_mdm.MDMAppleCommander, + redisKeyValue fleet.AdvancedKeyValueStore, + logger *slog.Logger, + appConfig *fleet.AppConfig, + certProfilesLimit int, + toInstall, toRemove []*fleet.MDMAppleProfilePayload, +) error { + userEnrollmentMap := make(map[string]string) + userEnrollmentsToHostUUIDsMap := make(map[string]string) + + getHostUserEnrollmentID := func(hostUUID string) (string, error) { + userEnrollmentID, ok := userEnrollmentMap[hostUUID] + if !ok { + userNanoEnrollment, err := ds.GetNanoMDMUserEnrollment(ctx, hostUUID) + if err != nil { + return "", ctxerr.Wrap(ctx, err, "getting user enrollment for host") + } + if userNanoEnrollment != nil { + userEnrollmentID = userNanoEnrollment.ID + } + userEnrollmentMap[hostUUID] = userEnrollmentID + if userEnrollmentID != "" { + userEnrollmentsToHostUUIDsMap[userEnrollmentID] = hostUUID + } + } + return userEnrollmentID, nil + } + + isAwaitingUserEnrollment := func(prof *fleet.MDMAppleProfilePayload) (bool, error) { + if prof.Scope != fleet.PayloadScopeUser { + return false, nil + } + userEnrollmentID, err := getHostUserEnrollmentID(prof.HostUUID) + if userEnrollmentID != "" || err != nil { + return false, err + } + if prof.DeviceEnrolledAt != nil && time.Since(*prof.DeviceEnrolledAt) < hoursToWaitForUserEnrollmentAfterDeviceEnrollment*time.Hour { + return true, nil + } + return false, nil + } + + hostProfiles := make([]*fleet.MDMAppleBulkUpsertHostProfilePayload, 0, len(toInstall)+len(toRemove)) + + profileIntersection := apple_mdm.NewProfileBimap() + profileIntersection.IntersectByIdentifierAndHostUUID(toInstall, toRemove) + + hostProfilesToCleanup := []*fleet.MDMAppleProfilePayload{} + hostProfilesToInstallMap := make(map[fleet.HostProfileUUID]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(toInstall)) + + // Pre-fetch contents for CA classification when CA throttling is on. + var caProfileUUIDs map[string]struct{} + var prefetchedContents map[string]mobileconfig.Mobileconfig + if certProfilesLimit > 0 { + uniqueUUIDs := make(map[string]struct{}, len(toInstall)) + for _, p := range toInstall { + uniqueUUIDs[p.ProfileUUID] = struct{}{} + } + uuids := make([]string, 0, len(uniqueUUIDs)) + for u := range uniqueUUIDs { + uuids = append(uuids, u) + } + var err error + prefetchedContents, err = ds.GetMDMAppleProfilesContents(ctx, uuids) + if err != nil { + return ctxerr.Wrap(ctx, err, "getting profile contents for CA classification") + } + caProfileUUIDs = make(map[string]struct{}, len(prefetchedContents)) + for pUUID, content := range prefetchedContents { + fleetVars := variables.Find(string(content)) + if fleet.HasCAVariables(fleetVars) { + caProfileUUIDs[pUUID] = struct{}{} + } + } + } + + var caInstallCount int + throttledHostsByProfile := make(map[string][]string) + installTargets, removeTargets := make(map[string]*fleet.CmdTarget), make(map[string]*fleet.CmdTarget) + + for _, p := range toInstall { + if pp, ok := profileIntersection.GetMatchingProfileInCurrentState(p); ok { + if pp.Status != &fleet.MDMDeliveryFailed && bytes.Equal(pp.Checksum, p.Checksum) { + hp := &fleet.MDMAppleBulkUpsertHostProfilePayload{ + ProfileUUID: p.ProfileUUID, + HostUUID: p.HostUUID, + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + OperationType: pp.OperationType, + Status: pp.Status, + CommandUUID: pp.CommandUUID, + Detail: pp.Detail, + Scope: pp.Scope, + } + hostProfiles = append(hostProfiles, hp) + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: p.HostUUID, ProfileUUID: p.ProfileUUID}] = hp + continue + } + } + + wait, err := isAwaitingUserEnrollment(p) + if err != nil { + return err + } + if wait { + hp := &fleet.MDMAppleBulkUpsertHostProfilePayload{ + ProfileUUID: p.ProfileUUID, + HostUUID: p.HostUUID, + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + OperationType: fleet.MDMOperationTypeInstall, + Status: nil, + Scope: p.Scope, + } + hostProfiles = append(hostProfiles, hp) + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: p.HostUUID, ProfileUUID: p.ProfileUUID}] = hp + continue + } + + recentlyEnrolled := p.DeviceEnrolledAt != nil && time.Since(*p.DeviceEnrolledAt) < 1*time.Hour + _, isCA := caProfileUUIDs[p.ProfileUUID] + isThrottledCA := certProfilesLimit > 0 && isCA && !recentlyEnrolled + if isThrottledCA && caInstallCount >= certProfilesLimit { + throttledHostsByProfile[p.ProfileUUID] = append(throttledHostsByProfile[p.ProfileUUID], p.HostUUID) + continue + } + + target := installTargets[p.ProfileUUID] + if target == nil { + target = &fleet.CmdTarget{ + CmdUUID: uuid.New().String(), + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + } + installTargets[p.ProfileUUID] = target + } + + if p.Scope == fleet.PayloadScopeUser { + userEnrollmentID, err := getHostUserEnrollmentID(p.HostUUID) + if err != nil { + return err + } + if userEnrollmentID == "" { + var errorDetail string + if fleet.IsAppleMobilePlatform(p.HostPlatform) { + errorDetail = "This setting couldn't be enforced because the user channel isn't available on iOS and iPadOS hosts." + } else { + errorDetail = "This setting couldn't be enforced because the user channel doesn't exist for this host. Currently, Fleet creates the user channel for hosts that automatically enroll." + logger.WarnContext(ctx, "host does not have a user enrollment, failing profile installation", + "host_uuid", p.HostUUID, "profile_uuid", p.ProfileUUID, "profile_identifier", p.ProfileIdentifier) + } + + hp := &fleet.MDMAppleBulkUpsertHostProfilePayload{ + ProfileUUID: p.ProfileUUID, + HostUUID: p.HostUUID, + OperationType: fleet.MDMOperationTypeInstall, + Status: &fleet.MDMDeliveryFailed, + Detail: errorDetail, + CommandUUID: "", + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + Scope: p.Scope, + } + hostProfiles = append(hostProfiles, hp) + continue + } + target.EnrollmentIDs = append(target.EnrollmentIDs, userEnrollmentID) + } else { + target.EnrollmentIDs = append(target.EnrollmentIDs, p.HostUUID) + } + + if isThrottledCA { + caInstallCount++ + } + + hp := &fleet.MDMAppleBulkUpsertHostProfilePayload{ + ProfileUUID: p.ProfileUUID, + HostUUID: p.HostUUID, + OperationType: fleet.MDMOperationTypeInstall, + Status: &fleet.MDMDeliveryPending, + CommandUUID: target.CmdUUID, + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + Scope: p.Scope, + } + hostProfiles = append(hostProfiles, hp) + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: p.HostUUID, ProfileUUID: p.ProfileUUID}] = hp + } + + const throttleLogBatchSize = 1000 + for profileUUID, hostUUIDs := range throttledHostsByProfile { + for i := 0; i < len(hostUUIDs); i += throttleLogBatchSize { + end := min(i+throttleLogBatchSize, len(hostUUIDs)) + logger.InfoContext(ctx, "throttled CA certificate profile installation", + "profile.uuid", profileUUID, + "mdm.target.host.uuids", hostUUIDs[i:end], + "mdm.certificate.profiles.limit", certProfilesLimit, + "batch", fmt.Sprintf("%d-%d/%d", i+1, end, len(hostUUIDs)), + ) + } + } + + for _, p := range toRemove { + if _, ok := profileIntersection.GetMatchingProfileInDesiredState(p); ok { + hostProfilesToCleanup = append(hostProfilesToCleanup, p) + continue + } + + if p.FailedInstallOnHost() { + hostProfilesToCleanup = append(hostProfilesToCleanup, p) + continue + } + if p.PendingInstallOnHost() { + hostProfilesToCleanup = append(hostProfilesToCleanup, p) + p.IgnoreError = true + } + + target := removeTargets[p.ProfileUUID] + if target == nil { + target = &fleet.CmdTarget{ + CmdUUID: uuid.New().String(), + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + } + removeTargets[p.ProfileUUID] = target + } + + if p.Scope == fleet.PayloadScopeUser { + userEnrollmentID, err := getHostUserEnrollmentID(p.HostUUID) + if err != nil { + return err + } + if userEnrollmentID == "" { + logger.WarnContext(ctx, "host does not have a user enrollment, cannot remove user scoped profile", + "host_uuid", p.HostUUID, "profile_uuid", p.ProfileUUID, "profile_identifier", p.ProfileIdentifier) + hostProfilesToCleanup = append(hostProfilesToCleanup, p) + continue + } + target.EnrollmentIDs = append(target.EnrollmentIDs, userEnrollmentID) + } else { + target.EnrollmentIDs = append(target.EnrollmentIDs, p.HostUUID) + } + + hostProfiles = append(hostProfiles, &fleet.MDMAppleBulkUpsertHostProfilePayload{ + ProfileUUID: p.ProfileUUID, + HostUUID: p.HostUUID, + OperationType: fleet.MDMOperationTypeRemove, + Status: &fleet.MDMDeliveryPending, + CommandUUID: target.CmdUUID, + ProfileIdentifier: p.ProfileIdentifier, + ProfileName: p.ProfileName, + Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, + IgnoreError: p.IgnoreError, + Scope: p.Scope, + }) + } + + // Skip hosts currently being set up (Redis MGet). + const isBeingSetupBatchSize = 1000 + for i := 0; i < len(hostProfiles); i += isBeingSetupBatchSize { + end := min(i+isBeingSetupBatchSize, len(hostProfiles)) + batch := hostProfiles[i:end] + keyedHostUUIDs := make([]string, len(batch)) + hostUUIDToHostProfiles := make(map[string][]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(batch)) + for j, hp := range batch { + keyedHostUUIDs[j] = fleet.MDMProfileProcessingKeyPrefix + ":" + hp.HostUUID + hostUUIDToHostProfiles[hp.HostUUID] = append(hostUUIDToHostProfiles[hp.HostUUID], hp) + } + + setupHostUUIDs, err := redisKeyValue.MGet(ctx, keyedHostUUIDs) + if err != nil { + return ctxerr.Wrap(ctx, err, "filtering hosts being set up") + } + for keyedHostUUID, exists := range setupHostUUIDs { + if exists != nil { + hostUUID := strings.TrimPrefix(keyedHostUUID, fleet.MDMProfileProcessingKeyPrefix+":") + logger.DebugContext(ctx, "skipping profile reconciliation for host being set up", "host_uuid", hostUUID) + hps, ok := hostUUIDToHostProfiles[hostUUID] + if !ok { + logger.DebugContext(ctx, "expected host uuid to be present but was not, do not skip profile reconciliation", "host_uuid", hostUUID) + continue + } + for _, hp := range hps { + hp.Status = nil + hp.CommandUUID = "" + hostProfilesToInstallMap[fleet.HostProfileUUID{HostUUID: hp.HostUUID, ProfileUUID: hp.ProfileUUID}] = hp + + if hp.OperationType == fleet.MDMOperationTypeInstall { + if target, ok := installTargets[hp.ProfileUUID]; ok { + var newEnrollmentIDs []string + for _, id := range target.EnrollmentIDs { + if id != hp.HostUUID { + newEnrollmentIDs = append(newEnrollmentIDs, id) + } + } + if len(newEnrollmentIDs) == 0 { + delete(installTargets, hp.ProfileUUID) + } else { + target.EnrollmentIDs = newEnrollmentIDs + } + } + } + } + } + } + } + + commandUUIDToHostIDsCleanupMap := make(map[string][]string) + for _, hp := range hostProfilesToCleanup { + if hp.CommandUUID != "" { + commandUUIDToHostIDsCleanupMap[hp.CommandUUID] = append(commandUUIDToHostIDsCleanupMap[hp.CommandUUID], hp.HostUUID) + } + } + if len(commandUUIDToHostIDsCleanupMap) > 0 { + if err := commander.BulkDeleteHostUserCommandsWithoutResults(ctx, commandUUIDToHostIDsCleanupMap); err != nil { + return ctxerr.Wrap(ctx, err, "deleting nano commands without results") + } + } + if err := ds.BulkDeleteMDMAppleHostsConfigProfiles(ctx, hostProfilesToCleanup); err != nil { + return ctxerr.Wrap(ctx, err, "deleting profiles that didn't change") + } + + if err := ds.BulkUpsertMDMAppleHostProfiles(ctx, hostProfiles); err != nil { + return ctxerr.Wrap(ctx, err, "updating host profiles") + } + + enqueueResult, err := apple_mdm.ProcessAndEnqueueProfiles( + ctx, + ds, + logger, + appConfig, + commander, + installTargets, + removeTargets, + hostProfilesToInstallMap, + userEnrollmentsToHostUUIDsMap, + prefetchedContents, + ) + if err != nil { + for _, hp := range hostProfiles { + if hp.Status != nil && *hp.Status == fleet.MDMDeliveryPending { + hp.Status = nil + hp.CommandUUID = "" + } + } + if err := ds.BulkUpsertMDMAppleHostProfiles(ctx, hostProfiles); err != nil { + return ctxerr.Wrap(ctx, err, "reverting host profiles after failed enqueue") + } + return ctxerr.Wrap(ctx, err, "processing and enqueuing profiles") + } + + hostProfsByCmdUUID := make(map[string][]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(hostProfiles)) + for _, hp := range hostProfiles { + if hp.CommandUUID != "" { + hostProfsByCmdUUID[hp.CommandUUID] = append(hostProfsByCmdUUID[hp.CommandUUID], hp) + } + } + + var failed []*fleet.MDMAppleBulkUpsertHostProfilePayload + for cmdUUID := range enqueueResult.FailedCmdUUIDs { + for _, hp := range hostProfsByCmdUUID[cmdUUID] { + hp.CommandUUID = "" + hp.Status = nil + failed = append(failed, hp) + } + } + + if err := ds.BulkUpsertMDMAppleHostProfiles(ctx, failed); err != nil { + return ctxerr.Wrap(ctx, err, "reverting status of failed profiles") + } + + return nil +} diff --git a/server/service/apple_mdm_batched_test.go b/server/service/apple_mdm_batched_test.go new file mode 100644 index 00000000000..d84858e4d82 --- /dev/null +++ b/server/service/apple_mdm_batched_test.go @@ -0,0 +1,450 @@ +package service + +import ( + "bytes" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestAppleProfileHandlerNoLabels(t *testing.T) { + p := &fleet.AppleProfileForReconcile{LabelMode: fleet.AppleProfileLabelModeNone} + require.True(t, appleProfileHandlerNoLabels(p)) +} + +func TestAppleProfileHandlerIncludeAll(t *testing.T) { + t.Run("no labels -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{} + require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + }) + t.Run("broken label -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}} + require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + }) + t.Run("host missing a label -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }} + require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + }) + t.Run("host has all labels -> true", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }} + require.True(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}, 2: {}})) + }) +} + +func TestAppleProfileHandlerIncludeAny(t *testing.T) { + t.Run("no labels -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{} + require.False(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{})) + }) + t.Run("broken labels are ignored", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: nil}, + {LabelID: new(uint(2))}, + }} + require.True(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{2: {}})) + }) + t.Run("host in no label -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }} + require.False(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{3: {}})) + }) + t.Run("host in one label -> true", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }} + require.True(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{1: {}})) + }) +} + +func TestAppleProfileHandlerExcludeAny(t *testing.T) { + host := &fleet.AppleHostReconcileInfo{ + HostID: 100, + UUID: "h1", + Platform: "darwin", + LabelUpdatedAt: time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), + } + + t.Run("no labels -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{} + require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + }) + + t.Run("broken label -> false (never apply)", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: nil}, + }} + require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + }) + + t.Run("host is in an excluded label -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }} + require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{1: {}})) + }) + + t.Run("host is not in any excluded label -> true", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + {LabelID: new(uint(2)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }} + require.True(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{99: {}})) + }) + + t.Run("dynamic label created after host's last scan -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + { + LabelID: new(uint(1)), + CreatedAt: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), + LabelMembershipType: 0, + }, + }} + require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + }) + + t.Run("manual label created after host's last scan -> still true", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + { + LabelID: new(uint(1)), + CreatedAt: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), + LabelMembershipType: 1, + }, + }} + require.True(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + }) +} + +func TestAppleProfileAppliesToHost_TeamAndPlatformGates(t *testing.T) { + host := &fleet.AppleHostReconcileInfo{ + HostID: 1, + UUID: "h1", + TeamID: nil, + Platform: "darwin", + } + + t.Run("wrong team -> false", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{TeamID: 5, LabelMode: fleet.AppleProfileLabelModeNone} + require.False(t, appleProfileAppliesToHost(p, host, nil)) + }) + + t.Run("global team matches nil team_id host", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{TeamID: 0, LabelMode: fleet.AppleProfileLabelModeNone} + require.True(t, appleProfileAppliesToHost(p, host, nil)) + }) + + t.Run("non-apple platform -> false", func(t *testing.T) { + linuxHost := *host + linuxHost.Platform = "linux" + p := &fleet.AppleProfileForReconcile{TeamID: 0, LabelMode: fleet.AppleProfileLabelModeNone} + require.False(t, appleProfileAppliesToHost(p, &linuxHost, nil)) + }) +} + +func TestComputeAppleReconcileDeltas(t *testing.T) { + hostA := &fleet.AppleHostReconcileInfo{ + HostID: 1, UUID: "uuid-A", TeamID: nil, Platform: "darwin", + LabelUpdatedAt: time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC), + } + hostB := &fleet.AppleHostReconcileInfo{ + HostID: 2, UUID: "uuid-B", TeamID: new(uint(7)), Platform: "darwin", + LabelUpdatedAt: time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC), + } + + pGlobal := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aProfileGlobal", + ProfileIdentifier: "com.example.global", + ProfileName: "Global", + TeamID: 0, + Checksum: []byte("aaaa"), + LabelMode: fleet.AppleProfileLabelModeNone, + } + pTeam7 := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aProfileTeam7", + ProfileIdentifier: "com.example.team7", + ProfileName: "Team7", + TeamID: 7, + Checksum: []byte("bbbb"), + LabelMode: fleet.AppleProfileLabelModeNone, + } + + profilesByTeam := map[uint][]*fleet.AppleProfileForReconcile{ + 0: {pGlobal}, + 7: {pTeam7}, + } + + t.Run("desired but not present -> install", func(t *testing.T) { + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA, hostB}, + nil, + nil, + profilesByTeam, + ) + require.Empty(t, toRemove) + require.Len(t, toInstall, 2) + set := map[string]string{} + for _, p := range toInstall { + set[p.HostUUID] = p.ProfileUUID + } + require.Equal(t, "aProfileGlobal", set["uuid-A"]) + require.Equal(t, "aProfileTeam7", set["uuid-B"]) + }) + + t.Run("checksum matches and op=install,status=pending -> no-op", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aProfileGlobal", + ProfileIdentifier: "com.example.global", + ProfileName: "Global", + HostUUID: "uuid-A", + Checksum: []byte("aaaa"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryPending), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Empty(t, toInstall) + require.Empty(t, toRemove) + }) + + t.Run("checksum differs -> install", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aProfileGlobal", + HostUUID: "uuid-A", + Checksum: []byte("OLD!"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Empty(t, toRemove) + require.Len(t, toInstall, 1) + require.True(t, bytes.Equal(toInstall[0].Checksum, []byte("aaaa"))) + }) + + t.Run("op=install,status=NULL -> reinstall", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aProfileGlobal", + HostUUID: "uuid-A", + Checksum: []byte("aaaa"), + OperationType: fleet.MDMOperationTypeInstall, + Status: nil, + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Empty(t, toRemove) + require.Len(t, toInstall, 1) + }) + + t.Run("op=remove -> reinstall (host should get profile back)", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aProfileGlobal", + HostUUID: "uuid-A", + Checksum: []byte("aaaa"), + OperationType: fleet.MDMOperationTypeRemove, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Empty(t, toRemove) + require.Len(t, toInstall, 1) + }) + + t.Run("not in desired -> remove (when not a broken label profile)", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aDeletedProfile", + ProfileIdentifier: "com.deleted", + HostUUID: "uuid-A", + Checksum: []byte("xxxx"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Len(t, toInstall, 1) + require.Len(t, toRemove, 1) + require.Equal(t, "aDeletedProfile", toRemove[0].ProfileUUID) + }) + + t.Run("remove already pending -> skip", func(t *testing.T) { + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aDeletedProfile", + HostUUID: "uuid-A", + OperationType: fleet.MDMOperationTypeRemove, + Status: new(fleet.MDMDeliveryPending), + }, { + ProfileUUID: "aProfileGlobal", + HostUUID: "uuid-A", + Checksum: []byte("aaaa"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profilesByTeam, + ) + require.Empty(t, toInstall) + require.Empty(t, toRemove) + }) + + t.Run("broken label profile is not removed even when current row exists", func(t *testing.T) { + brokenProf := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aBrokenLabel", + ProfileIdentifier: "com.broken", + TeamID: 0, + LabelMode: fleet.AppleProfileLabelModeIncludeAll, + Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, + } + profByTeam := map[uint][]*fleet.AppleProfileForReconcile{ + 0: {pGlobal, brokenProf}, + } + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aBrokenLabel", + HostUUID: "uuid-A", + Checksum: []byte("xxxx"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profByTeam, + ) + require.Len(t, toInstall, 1) + require.Empty(t, toRemove) + }) +} + +func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { + host := &fleet.AppleHostReconcileInfo{ + HostID: 1, UUID: "uuid-A", TeamID: nil, Platform: "darwin", + LabelUpdatedAt: time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC), + } + + pIncludeAll := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aIncludeAll", TeamID: 0, + LabelMode: fleet.AppleProfileLabelModeIncludeAll, + Labels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}, {LabelID: new(uint(2))}}, + Checksum: []byte("c1"), + } + pExcludeAny := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aExcludeAny", TeamID: 0, + LabelMode: fleet.AppleProfileLabelModeExcludeAny, + Labels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(3)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + Checksum: []byte("c2"), + } + profByTeam := map[uint][]*fleet.AppleProfileForReconcile{0: {pIncludeAll, pExcludeAny}} + + t.Run("host has both required labels -> include-all installs", func(t *testing.T) { + hostLabels := map[uint]map[uint]struct{}{ + 1: {1: {}, 2: {}}, + } + toInstall, _ := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{host}, hostLabels, nil, profByTeam, + ) + uuids := map[string]struct{}{} + for _, p := range toInstall { + uuids[p.ProfileUUID] = struct{}{} + } + _, hasIncludeAll := uuids["aIncludeAll"] + _, hasExcludeAny := uuids["aExcludeAny"] + require.True(t, hasIncludeAll) + require.True(t, hasExcludeAny) + }) + + t.Run("host missing one include-all label -> only exclude-any installs", func(t *testing.T) { + hostLabels := map[uint]map[uint]struct{}{ + 1: {1: {}}, + } + toInstall, _ := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{host}, hostLabels, nil, profByTeam, + ) + uuids := map[string]struct{}{} + for _, p := range toInstall { + uuids[p.ProfileUUID] = struct{}{} + } + _, hasIncludeAll := uuids["aIncludeAll"] + _, hasExcludeAny := uuids["aExcludeAny"] + require.False(t, hasIncludeAll) + require.True(t, hasExcludeAny) + }) + + t.Run("host is in the excluded label -> exclude-any skipped", func(t *testing.T) { + hostLabels := map[uint]map[uint]struct{}{ + 1: {1: {}, 2: {}, 3: {}}, + } + toInstall, _ := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{host}, hostLabels, nil, profByTeam, + ) + uuids := map[string]struct{}{} + for _, p := range toInstall { + uuids[p.ProfileUUID] = struct{}{} + } + _, hasIncludeAll := uuids["aIncludeAll"] + _, hasExcludeAny := uuids["aExcludeAny"] + require.True(t, hasIncludeAll) + require.False(t, hasExcludeAny) + }) +} + +func TestAppleHostReconcileInfo_EffectiveTeamID(t *testing.T) { + h := &fleet.AppleHostReconcileInfo{TeamID: nil} + require.Equal(t, uint(0), h.EffectiveTeamID()) + + h.TeamID = new(uint(42)) + require.Equal(t, uint(42), h.EffectiveTeamID()) +} + +func TestIsBrokenAppleProfile(t *testing.T) { + good := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aGood", + LabelMode: fleet.AppleProfileLabelModeIncludeAll, + Labels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}}, + } + broken := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aBroken", + LabelMode: fleet.AppleProfileLabelModeIncludeAll, + Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, + } + byTeam := map[uint][]*fleet.AppleProfileForReconcile{ + 0: {good, broken}, + } + + require.False(t, isBrokenAppleProfile("aGood", byTeam)) + require.True(t, isBrokenAppleProfile("aBroken", byTeam)) + require.False(t, isBrokenAppleProfile("aMissing", byTeam)) +} From ffa4daf9cc1dbf1ac584e9052e0aea959eb2160b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 09:15:10 +0000 Subject: [PATCH 2/5] Always use batched Apple MDM reconciler on this branch Drop the FLEET_MDM_APPLE_BATCHED_RECONCILER toggle so loadtests don't need an env var to enable the batched path. The branch always runs ReconcileAppleProfilesBatched; the legacy ReconcileAppleProfiles function is still present for diffing/reference but no longer wired into the cron. https://claude.ai/code/session_01Vvy1keXRKZRzDbJQd7dzDn --- cmd/fleet/cron.go | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 1d2047ce072..4b8bd2553a5 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1857,17 +1857,12 @@ func newAppleMDMProfileManagerSchedule( defaultInterval = 30 * time.Second ) - useBatchedReconciler := isEnvBoolTrue("FLEET_MDM_APPLE_BATCHED_RECONCILER") - logger = logger.With("cron", name) s := schedule.New( ctx, name, instanceID, defaultInterval, ds, ds, schedule.WithLogger(logger), schedule.WithJob("manage_apple_profiles", func(ctx context.Context) error { - if useBatchedReconciler { - return service.ReconcileAppleProfilesBatched(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) - } - return service.ReconcileAppleProfiles(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) + return service.ReconcileAppleProfilesBatched(ctx, ds, commander, redisKeyValue, logger, certProfilesLimit) }), schedule.WithJob("manage_apple_declarations", func(ctx context.Context) error { return service.ReconcileAppleDeclarations(ctx, ds, commander, logger) @@ -1877,22 +1872,6 @@ func newAppleMDMProfileManagerSchedule( return s, nil } -// isEnvBoolTrue returns true if the named env var is set to a truthy value -// (1, t, T, TRUE, true, True). Used to gate the experimental batched Apple -// MDM profile reconciler. -func isEnvBoolTrue(name string) bool { - v, ok := os.LookupEnv(name) - if !ok { - return false - } - switch strings.ToLower(strings.TrimSpace(v)) { - case "1", "t", "true", "yes", "y": - return true - default: - return false - } -} - func newWindowsMDMProfileManagerSchedule( ctx context.Context, instanceID string, From bd71852cf593fca3e5fcc8438770adb203f1ce48 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 09:35:19 +0000 Subject: [PATCH 3/5] Support combining Exclude Any with Include All or Include Any Split AppleProfileForReconcile's single LabelMode + Labels into: - IncludeMode (None / All / Any) + IncludeLabels - ExcludeLabels (always "exclude any" semantic) The dispatcher composes the two gates: a profile applies iff the include gate passes (skipped when IncludeMode == None) AND the exclude gate passes (skipped when ExcludeLabels is empty). The existing per-gate handlers are unchanged in semantics; they now take []AppleProfileLabelRef directly so they're pure functions composable in any combination. Datastore loader partitions label rows by mcpl.exclude into the two slices and derives IncludeMode from the include-row require_all flag. Broken-label exemption from removal now considers labels in either slice. https://claude.ai/code/session_01Vvy1keXRKZRzDbJQd7dzDn --- server/datastore/mysql/apple_mdm_batched.go | 80 +++--- server/fleet/apple_mdm.go | 47 ++- server/service/apple_mdm_batched.go | 108 +++---- server/service/apple_mdm_batched_test.go | 303 +++++++++++++++----- 4 files changed, 370 insertions(+), 168 deletions(-) diff --git a/server/datastore/mysql/apple_mdm_batched.go b/server/datastore/mysql/apple_mdm_batched.go index fe40a9c48d8..f94d9912d3f 100644 --- a/server/datastore/mysql/apple_mdm_batched.go +++ b/server/datastore/mysql/apple_mdm_batched.go @@ -133,15 +133,18 @@ func (ds *Datastore) ListAppleProfilesForReconcile(ctx context.Context) ([]*flee return nil, ctxerr.Wrap(ctx, err, "list apple profile labels for reconcile") } - // Per-profile label mode discovery. A profile's mode is set by the first - // label row seen; if later rows disagree we mark it mixed and treat the - // profile as no-label (defensive — upsert enforces consistency). - type modeMarker struct { + // Per-profile include-mode discovery. Include labels for a single + // profile must share a single require_all value; the first include + // row sets the mode and later disagreements mark it mixed. Exclude + // rows always go to ExcludeLabels and have a single "exclude any" + // semantic (their require_all column is ignored). A profile may + // carry both an include set and an exclude set. + type includeAccum struct { set bool - mode fleet.AppleProfileLabelMode + mode fleet.AppleProfileIncludeMode mixed bool } - modes := make(map[string]*modeMarker, len(byUUID)) + includeModes := make(map[string]*includeAccum, len(byUUID)) for _, lr := range labelRows { p, ok := byUUID[lr.ProfileUUID] @@ -149,29 +152,6 @@ func (ds *Datastore) ListAppleProfilesForReconcile(ctx context.Context) ([]*flee continue } - mm := modes[lr.ProfileUUID] - if mm == nil { - mm = &modeMarker{} - modes[lr.ProfileUUID] = mm - } - - var rowMode fleet.AppleProfileLabelMode - switch { - case lr.Exclude: - rowMode = fleet.AppleProfileLabelModeExcludeAny - case lr.RequireAll: - rowMode = fleet.AppleProfileLabelModeIncludeAll - default: - rowMode = fleet.AppleProfileLabelModeIncludeAny - } - - if !mm.set { - mm.mode = rowMode - mm.set = true - } else if mm.mode != rowMode { - mm.mixed = true - } - ref := fleet.AppleProfileLabelRef{ LabelMembershipType: lr.LabelMembershipType, } @@ -182,17 +162,47 @@ func (ds *Datastore) ListAppleProfilesForReconcile(ctx context.Context) ([]*flee if lr.LabelCreatedAt.Valid { ref.CreatedAt = lr.LabelCreatedAt.Time } - p.Labels = append(p.Labels, ref) + + if lr.Exclude { + p.ExcludeLabels = append(p.ExcludeLabels, ref) + continue + } + + // Include row. + p.IncludeLabels = append(p.IncludeLabels, ref) + + var rowMode fleet.AppleProfileIncludeMode + if lr.RequireAll { + rowMode = fleet.AppleProfileIncludeAll + } else { + rowMode = fleet.AppleProfileIncludeAny + } + + ia := includeModes[lr.ProfileUUID] + if ia == nil { + ia = &includeAccum{} + includeModes[lr.ProfileUUID] = ia + } + if !ia.set { + ia.mode = rowMode + ia.set = true + } else if ia.mode != rowMode { + ia.mixed = true + } } - for uuid, mm := range modes { + for uuid, ia := range includeModes { p := byUUID[uuid] - if mm.mixed { - p.LabelMode = fleet.AppleProfileLabelModeNone - p.Labels = nil + if ia.mixed { + // Defensive: include rows disagreed on require_all (should + // be impossible in production — the upsert path enforces a + // single mode). Drop the include set so we don't guess at + // intent; exclude labels (if any) are preserved. + p.IncludeLabels = nil + p.IncludeMode = fleet.AppleProfileIncludeNone continue } - p.LabelMode = mm.mode + p.IncludeMode = ia.mode } return out, nil diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 549e2a5fa76..ff11c8ac6f4 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -461,16 +461,23 @@ func (h *AppleHostReconcileInfo) EffectiveTeamID() uint { return *h.TeamID } -// AppleProfileLabelMode indicates how a profile's label assignments gate -// applicability to a host. Profiles never mix modes — a single profile is -// either no-labels, include-all, include-any, or exclude-any. -type AppleProfileLabelMode int +// AppleProfileIncludeMode indicates how a profile's include-labels gate +// applicability to a host. Independent of exclude-labels, which always +// have "exclude any" semantics. A single profile may carry both include +// labels (with one consistent mode) and exclude labels. +type AppleProfileIncludeMode int const ( - AppleProfileLabelModeNone AppleProfileLabelMode = iota - AppleProfileLabelModeIncludeAll - AppleProfileLabelModeIncludeAny - AppleProfileLabelModeExcludeAny + // AppleProfileIncludeNone means the profile has no include labels — + // applicability is determined entirely by team, platform, and any + // exclude labels present. + AppleProfileIncludeNone AppleProfileIncludeMode = iota + // AppleProfileIncludeAll requires the host to be a member of every + // (non-broken) include label. + AppleProfileIncludeAll + // AppleProfileIncludeAny requires the host to be a member of at + // least one include label. + AppleProfileIncludeAny ) // AppleProfileLabelRef is a single label reference attached to a profile. @@ -487,6 +494,10 @@ type AppleProfileLabelRef struct { // AppleProfileForReconcile is the profile data needed by the batched // reconciler to compute desired state per host in memory. +// +// Include and exclude labels are stored separately so a profile can carry +// both: applicability becomes (include gate passes) AND (exclude gate +// passes), with each gate skipped when its slice is empty. type AppleProfileForReconcile struct { ProfileUUID string ProfileIdentifier string @@ -495,16 +506,22 @@ type AppleProfileForReconcile struct { Checksum []byte SecretsUpdatedAt *time.Time Scope PayloadScope - LabelMode AppleProfileLabelMode - Labels []AppleProfileLabelRef + IncludeMode AppleProfileIncludeMode + IncludeLabels []AppleProfileLabelRef + ExcludeLabels []AppleProfileLabelRef } -// HasBrokenLabel reports whether any of the profile's label assignments -// reference a deleted label. Broken include-* profiles are excluded from -// desired state; broken exclude-any profiles are also excluded (never apply). -// Broken profiles are also exempt from removal in the existing flow. +// HasBrokenLabel reports whether any include or exclude label on the +// profile references a deleted label. Used to keep broken-label profiles +// exempt from removal (matches legacy behaviour: a profile with a broken +// label is never auto-removed from a host that already has it). func (p *AppleProfileForReconcile) HasBrokenLabel() bool { - for _, l := range p.Labels { + for _, l := range p.IncludeLabels { + if l.LabelID == nil { + return true + } + } + for _, l := range p.ExcludeLabels { if l.LabelID == nil { return true } diff --git a/server/service/apple_mdm_batched.go b/server/service/apple_mdm_batched.go index bb70a678f5b..647aee33093 100644 --- a/server/service/apple_mdm_batched.go +++ b/server/service/apple_mdm_batched.go @@ -143,7 +143,12 @@ func ReconcileAppleProfilesBatched( labelIDSet := make(map[uint]struct{}) for _, p := range allProfiles { profilesByTeam[p.TeamID] = append(profilesByTeam[p.TeamID], p) - for _, lr := range p.Labels { + for _, lr := range p.IncludeLabels { + if lr.LabelID != nil { + labelIDSet[*lr.LabelID] = struct{}{} + } + } + for _, lr := range p.ExcludeLabels { if lr.LabelID != nil { labelIDSet[*lr.LabelID] = struct{}{} } @@ -312,59 +317,67 @@ func computeAppleReconcileDeltas( return toInstall, toRemove } -// appleProfileAppliesToHost is the top-level handler dispatcher. It applies -// the team and platform gates, then routes to the per-label-mode handler. +// appleProfileAppliesToHost is the top-level dispatcher. It applies the +// team and platform gates, then composes whichever include + exclude +// label handlers the profile carries. A profile may carry both an +// include set (in one consistent mode) and an exclude set; both gates +// must pass for the profile to apply. func appleProfileAppliesToHost( p *fleet.AppleProfileForReconcile, host *fleet.AppleHostReconcileInfo, hostLabels map[uint]struct{}, ) bool { // Team gate (already filtered upstream by profilesByTeam, but - // double-check for safety so handlers stay composable). + // double-check so handlers stay composable in isolation). if p.TeamID != host.EffectiveTeamID() { return false } - // Platform gate. Profiles targeting macOS-only platforms are filtered - // later in FilterMacOSOnlyProfilesFromIOSIPadOS so we don't reproduce - // that here. + // Platform gate. macOS-only profiles on iOS/iPadOS hosts are filtered + // later by FilterMacOSOnlyProfilesFromIOSIPadOS. if !isAppleProfileEligiblePlatform(host.Platform) { return false } - switch p.LabelMode { - case fleet.AppleProfileLabelModeNone: - return appleProfileHandlerNoLabels(p) - case fleet.AppleProfileLabelModeIncludeAll: - return appleProfileHandlerIncludeAll(p, hostLabels) - case fleet.AppleProfileLabelModeIncludeAny: - return appleProfileHandlerIncludeAny(p, hostLabels) - case fleet.AppleProfileLabelModeExcludeAny: - return appleProfileHandlerExcludeAny(p, host, hostLabels) - default: - return false + // Include gate: only run if the profile has include labels. + if p.IncludeMode != fleet.AppleProfileIncludeNone { + var ok bool + switch p.IncludeMode { + case fleet.AppleProfileIncludeAll: + ok = appleProfileHandlerIncludeAll(p.IncludeLabels, hostLabels) + case fleet.AppleProfileIncludeAny: + ok = appleProfileHandlerIncludeAny(p.IncludeLabels, hostLabels) + default: + return false + } + if !ok { + return false + } } + + // Exclude gate: only run if the profile has exclude labels. + if len(p.ExcludeLabels) > 0 { + if !appleProfileHandlerExcludeAny(p.ExcludeLabels, host, hostLabels) { + return false + } + } + + return true } func isAppleProfileEligiblePlatform(platform string) bool { return platform == "darwin" || platform == "ios" || platform == "ipados" } -// appleProfileHandlerNoLabels: no labels → always applies (subject to -// outer team/platform gates). -func appleProfileHandlerNoLabels(_ *fleet.AppleProfileForReconcile) bool { - return true -} - // appleProfileHandlerIncludeAll: host must be a member of every (non- -// broken) referenced label. If any label is broken, the profile does -// not apply (mirrors the legacy SQL where include-* with a broken label -// produces no desired-state row). -func appleProfileHandlerIncludeAll(p *fleet.AppleProfileForReconcile, hostLabels map[uint]struct{}) bool { - if len(p.Labels) == 0 { +// broken) include label. A broken label disqualifies the profile, mirroring +// the legacy SQL where include-* with a broken label produces no desired- +// state row. +func appleProfileHandlerIncludeAll(labels []fleet.AppleProfileLabelRef, hostLabels map[uint]struct{}) bool { + if len(labels) == 0 { return false } - for _, l := range p.Labels { + for _, l := range labels { if l.LabelID == nil { return false } @@ -376,10 +389,10 @@ func appleProfileHandlerIncludeAll(p *fleet.AppleProfileForReconcile, hostLabels } // appleProfileHandlerIncludeAny: host must be a member of at least one -// referenced label. Broken labels can't match (host can't be a member of a -// deleted label). -func appleProfileHandlerIncludeAny(p *fleet.AppleProfileForReconcile, hostLabels map[uint]struct{}) bool { - for _, l := range p.Labels { +// include label. Broken labels can't match (host can't be a member of a +// deleted label) so they're silently skipped. +func appleProfileHandlerIncludeAny(labels []fleet.AppleProfileLabelRef, hostLabels map[uint]struct{}) bool { + for _, l := range labels { if l.LabelID == nil { continue } @@ -390,28 +403,27 @@ func appleProfileHandlerIncludeAny(p *fleet.AppleProfileForReconcile, hostLabels return false } -// appleProfileHandlerExcludeAny: profile applies when the host is NOT a -// member of any referenced label, with two extra rules from the legacy -// SQL: +// appleProfileHandlerExcludeAny: profile passes the exclude gate when the +// host is NOT a member of any referenced label, with two safety rules: +// +// - Any broken exclude label disqualifies the profile entirely (we can't +// prove the exclusion). +// - Dynamic labels created after the host's last label scan are treated +// as "results not yet reported" — also disqualify, so we don't +// install a profile that the not-yet-scanned label would exclude. +// Manual labels (membership_type=1) skip this timing check. // -// - Broken labels disqualify the profile entirely (never apply). -// - For dynamic labels, we require host.label_updated_at >= label.created_at -// so we don't treat "label results not yet reported" as "not a member". -// Manual labels (membership_type=1) skip this check. +// Returns true when called with an empty slice — the dispatcher won't do +// that, but the handler's contract should still be "no exclusions = pass". func appleProfileHandlerExcludeAny( - p *fleet.AppleProfileForReconcile, + labels []fleet.AppleProfileLabelRef, host *fleet.AppleHostReconcileInfo, hostLabels map[uint]struct{}, ) bool { - if len(p.Labels) == 0 { - return false - } - for _, l := range p.Labels { + for _, l := range labels { if l.LabelID == nil { return false } - // If the host's label scan hasn't caught up with the label's - // creation, we can't trust "not a member" — skip the profile. if l.LabelMembershipType != 1 && !l.CreatedAt.IsZero() && host.LabelUpdatedAt.Before(l.CreatedAt) { return false } diff --git a/server/service/apple_mdm_batched_test.go b/server/service/apple_mdm_batched_test.go index d84858e4d82..5bfa778dfd9 100644 --- a/server/service/apple_mdm_batched_test.go +++ b/server/service/apple_mdm_batched_test.go @@ -9,61 +9,54 @@ import ( "github.com/stretchr/testify/require" ) -func TestAppleProfileHandlerNoLabels(t *testing.T) { - p := &fleet.AppleProfileForReconcile{LabelMode: fleet.AppleProfileLabelModeNone} - require.True(t, appleProfileHandlerNoLabels(p)) -} - func TestAppleProfileHandlerIncludeAll(t *testing.T) { t.Run("no labels -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{} - require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + require.False(t, appleProfileHandlerIncludeAll(nil, map[uint]struct{}{1: {}})) }) t.Run("broken label -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}} - require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + labels := []fleet.AppleProfileLabelRef{{LabelID: nil}} + require.False(t, appleProfileHandlerIncludeAll(labels, map[uint]struct{}{1: {}})) }) t.Run("host missing a label -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1))}, {LabelID: new(uint(2))}, - }} - require.False(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}})) + } + require.False(t, appleProfileHandlerIncludeAll(labels, map[uint]struct{}{1: {}})) }) t.Run("host has all labels -> true", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1))}, {LabelID: new(uint(2))}, - }} - require.True(t, appleProfileHandlerIncludeAll(p, map[uint]struct{}{1: {}, 2: {}})) + } + require.True(t, appleProfileHandlerIncludeAll(labels, map[uint]struct{}{1: {}, 2: {}})) }) } func TestAppleProfileHandlerIncludeAny(t *testing.T) { t.Run("no labels -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{} - require.False(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{})) + require.False(t, appleProfileHandlerIncludeAny(nil, map[uint]struct{}{})) }) t.Run("broken labels are ignored", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: nil}, {LabelID: new(uint(2))}, - }} - require.True(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{2: {}})) + } + require.True(t, appleProfileHandlerIncludeAny(labels, map[uint]struct{}{2: {}})) }) t.Run("host in no label -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1))}, {LabelID: new(uint(2))}, - }} - require.False(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{3: {}})) + } + require.False(t, appleProfileHandlerIncludeAny(labels, map[uint]struct{}{3: {}})) }) t.Run("host in one label -> true", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1))}, {LabelID: new(uint(2))}, - }} - require.True(t, appleProfileHandlerIncludeAny(p, map[uint]struct{}{1: {}})) + } + require.True(t, appleProfileHandlerIncludeAny(labels, map[uint]struct{}{1: {}})) }) } @@ -75,53 +68,50 @@ func TestAppleProfileHandlerExcludeAny(t *testing.T) { LabelUpdatedAt: time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC), } - t.Run("no labels -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{} - require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + t.Run("empty labels -> true (nothing to exclude)", func(t *testing.T) { + require.True(t, appleProfileHandlerExcludeAny(nil, host, map[uint]struct{}{})) }) t.Run("broken label -> false (never apply)", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ - {LabelID: nil}, - }} - require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + labels := []fleet.AppleProfileLabelRef{{LabelID: nil}} + require.False(t, appleProfileHandlerExcludeAny(labels, host, map[uint]struct{}{})) }) t.Run("host is in an excluded label -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, - }} - require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{1: {}})) + } + require.False(t, appleProfileHandlerExcludeAny(labels, host, map[uint]struct{}{1: {}})) }) t.Run("host is not in any excluded label -> true", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ {LabelID: new(uint(1)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, {LabelID: new(uint(2)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, - }} - require.True(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{99: {}})) + } + require.True(t, appleProfileHandlerExcludeAny(labels, host, map[uint]struct{}{99: {}})) }) t.Run("dynamic label created after host's last scan -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ { LabelID: new(uint(1)), CreatedAt: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), LabelMembershipType: 0, }, - }} - require.False(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + } + require.False(t, appleProfileHandlerExcludeAny(labels, host, map[uint]struct{}{})) }) t.Run("manual label created after host's last scan -> still true", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{Labels: []fleet.AppleProfileLabelRef{ + labels := []fleet.AppleProfileLabelRef{ { LabelID: new(uint(1)), CreatedAt: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), LabelMembershipType: 1, }, - }} - require.True(t, appleProfileHandlerExcludeAny(p, host, map[uint]struct{}{})) + } + require.True(t, appleProfileHandlerExcludeAny(labels, host, map[uint]struct{}{})) }) } @@ -134,23 +124,146 @@ func TestAppleProfileAppliesToHost_TeamAndPlatformGates(t *testing.T) { } t.Run("wrong team -> false", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{TeamID: 5, LabelMode: fleet.AppleProfileLabelModeNone} + p := &fleet.AppleProfileForReconcile{TeamID: 5, IncludeMode: fleet.AppleProfileIncludeNone} require.False(t, appleProfileAppliesToHost(p, host, nil)) }) t.Run("global team matches nil team_id host", func(t *testing.T) { - p := &fleet.AppleProfileForReconcile{TeamID: 0, LabelMode: fleet.AppleProfileLabelModeNone} + p := &fleet.AppleProfileForReconcile{TeamID: 0, IncludeMode: fleet.AppleProfileIncludeNone} require.True(t, appleProfileAppliesToHost(p, host, nil)) }) t.Run("non-apple platform -> false", func(t *testing.T) { linuxHost := *host linuxHost.Platform = "linux" - p := &fleet.AppleProfileForReconcile{TeamID: 0, LabelMode: fleet.AppleProfileLabelModeNone} + p := &fleet.AppleProfileForReconcile{TeamID: 0, IncludeMode: fleet.AppleProfileIncludeNone} require.False(t, appleProfileAppliesToHost(p, &linuxHost, nil)) }) } +func TestAppleProfileAppliesToHost_CombinedIncludeAndExclude(t *testing.T) { + host := &fleet.AppleHostReconcileInfo{ + HostID: 1, UUID: "h1", TeamID: nil, Platform: "darwin", + LabelUpdatedAt: time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC), + } + + t.Run("include-all + exclude-any: host passes both -> applies", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{1: {}, 2: {}} // in 1+2, not in 9 + require.True(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("include-all + exclude-any: include fails -> does not apply", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{1: {}} // missing label 2 + require.False(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("include-all + exclude-any: exclude fails -> does not apply", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{1: {}, 9: {}} // in label 9 -> excluded + require.False(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("include-any + exclude-any: any include matches, no exclude matches -> applies", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAny, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{2: {}} // matches include-any, not in exclude + require.True(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("include-any + exclude-any: no include matches -> does not apply", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAny, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + {LabelID: new(uint(2))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{5: {}} + require.False(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("include-any + exclude-any: in exclude label -> does not apply even when include matches", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAny, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + }, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + hostLabels := map[uint]struct{}{1: {}, 9: {}} + require.False(t, appleProfileAppliesToHost(p, host, hostLabels)) + }) + + t.Run("exclude-only profile: applies when not in any exclude label", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeNone, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(9)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + } + require.True(t, appleProfileAppliesToHost(p, host, map[uint]struct{}{1: {}})) + }) + + t.Run("include-only profile: applies when include passes", func(t *testing.T) { + p := &fleet.AppleProfileForReconcile{ + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(1))}, + }, + } + require.True(t, appleProfileAppliesToHost(p, host, map[uint]struct{}{1: {}})) + }) +} + func TestComputeAppleReconcileDeltas(t *testing.T) { hostA := &fleet.AppleHostReconcileInfo{ HostID: 1, UUID: "uuid-A", TeamID: nil, Platform: "darwin", @@ -167,7 +280,7 @@ func TestComputeAppleReconcileDeltas(t *testing.T) { ProfileName: "Global", TeamID: 0, Checksum: []byte("aaaa"), - LabelMode: fleet.AppleProfileLabelModeNone, + IncludeMode: fleet.AppleProfileIncludeNone, } pTeam7 := &fleet.AppleProfileForReconcile{ ProfileUUID: "aProfileTeam7", @@ -175,7 +288,7 @@ func TestComputeAppleReconcileDeltas(t *testing.T) { ProfileName: "Team7", TeamID: 7, Checksum: []byte("bbbb"), - LabelMode: fleet.AppleProfileLabelModeNone, + IncludeMode: fleet.AppleProfileIncludeNone, } profilesByTeam := map[uint][]*fleet.AppleProfileForReconcile{ @@ -318,13 +431,13 @@ func TestComputeAppleReconcileDeltas(t *testing.T) { require.Empty(t, toRemove) }) - t.Run("broken label profile is not removed even when current row exists", func(t *testing.T) { + t.Run("broken include label profile is not removed even when current row exists", func(t *testing.T) { brokenProf := &fleet.AppleProfileForReconcile{ ProfileUUID: "aBrokenLabel", ProfileIdentifier: "com.broken", TeamID: 0, - LabelMode: fleet.AppleProfileLabelModeIncludeAll, - Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, } profByTeam := map[uint][]*fleet.AppleProfileForReconcile{ 0: {pGlobal, brokenProf}, @@ -345,6 +458,34 @@ func TestComputeAppleReconcileDeltas(t *testing.T) { require.Len(t, toInstall, 1) require.Empty(t, toRemove) }) + + t.Run("broken exclude label profile is not removed (broken in either slice protects)", func(t *testing.T) { + brokenProf := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aBrokenExclude", + ProfileIdentifier: "com.broken.exclude", + TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeNone, + ExcludeLabels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, + } + profByTeam := map[uint][]*fleet.AppleProfileForReconcile{ + 0: {pGlobal, brokenProf}, + } + current := map[string][]*fleet.MDMAppleProfilePayload{ + "uuid-A": {{ + ProfileUUID: "aBrokenExclude", + HostUUID: "uuid-A", + Checksum: []byte("xxxx"), + OperationType: fleet.MDMOperationTypeInstall, + Status: new(fleet.MDMDeliveryVerified), + }}, + } + toInstall, toRemove := computeAppleReconcileDeltas( + []*fleet.AppleHostReconcileInfo{hostA}, + nil, current, profByTeam, + ) + require.Len(t, toInstall, 1) + require.Empty(t, toRemove) + }) } func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { @@ -355,21 +496,30 @@ func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { pIncludeAll := &fleet.AppleProfileForReconcile{ ProfileUUID: "aIncludeAll", TeamID: 0, - LabelMode: fleet.AppleProfileLabelModeIncludeAll, - Labels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}, {LabelID: new(uint(2))}}, - Checksum: []byte("c1"), + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}, {LabelID: new(uint(2))}}, + Checksum: []byte("c1"), } pExcludeAny := &fleet.AppleProfileForReconcile{ ProfileUUID: "aExcludeAny", TeamID: 0, - LabelMode: fleet.AppleProfileLabelModeExcludeAny, - Labels: []fleet.AppleProfileLabelRef{ + IncludeMode: fleet.AppleProfileIncludeNone, + ExcludeLabels: []fleet.AppleProfileLabelRef{ {LabelID: new(uint(3)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, }, Checksum: []byte("c2"), } - profByTeam := map[uint][]*fleet.AppleProfileForReconcile{0: {pIncludeAll, pExcludeAny}} + pCombined := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aCombined", TeamID: 0, + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}}, + ExcludeLabels: []fleet.AppleProfileLabelRef{ + {LabelID: new(uint(3)), CreatedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, + Checksum: []byte("c3"), + } + profByTeam := map[uint][]*fleet.AppleProfileForReconcile{0: {pIncludeAll, pExcludeAny, pCombined}} - t.Run("host has both required labels -> include-all installs", func(t *testing.T) { + t.Run("host has both required labels, not in exclude -> all three install", func(t *testing.T) { hostLabels := map[uint]map[uint]struct{}{ 1: {1: {}, 2: {}}, } @@ -382,8 +532,10 @@ func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { } _, hasIncludeAll := uuids["aIncludeAll"] _, hasExcludeAny := uuids["aExcludeAny"] + _, hasCombined := uuids["aCombined"] require.True(t, hasIncludeAll) require.True(t, hasExcludeAny) + require.True(t, hasCombined) }) t.Run("host missing one include-all label -> only exclude-any installs", func(t *testing.T) { @@ -399,11 +551,14 @@ func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { } _, hasIncludeAll := uuids["aIncludeAll"] _, hasExcludeAny := uuids["aExcludeAny"] + _, hasCombined := uuids["aCombined"] require.False(t, hasIncludeAll) require.True(t, hasExcludeAny) + // combined needs labels {1} AND not in {3}; host has {1} and not in 3 -> installs + require.True(t, hasCombined) }) - t.Run("host is in the excluded label -> exclude-any skipped", func(t *testing.T) { + t.Run("host is in the excluded label -> exclude-any and combined skipped", func(t *testing.T) { hostLabels := map[uint]map[uint]struct{}{ 1: {1: {}, 2: {}, 3: {}}, } @@ -416,8 +571,10 @@ func TestComputeAppleReconcileDeltas_LabelGates(t *testing.T) { } _, hasIncludeAll := uuids["aIncludeAll"] _, hasExcludeAny := uuids["aExcludeAny"] + _, hasCombined := uuids["aCombined"] require.True(t, hasIncludeAll) require.False(t, hasExcludeAny) + require.False(t, hasCombined) }) } @@ -431,20 +588,26 @@ func TestAppleHostReconcileInfo_EffectiveTeamID(t *testing.T) { func TestIsBrokenAppleProfile(t *testing.T) { good := &fleet.AppleProfileForReconcile{ - ProfileUUID: "aGood", - LabelMode: fleet.AppleProfileLabelModeIncludeAll, - Labels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}}, + ProfileUUID: "aGood", + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{{LabelID: new(uint(1))}}, + } + brokenInclude := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aBrokenInclude", + IncludeMode: fleet.AppleProfileIncludeAll, + IncludeLabels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, } - broken := &fleet.AppleProfileForReconcile{ - ProfileUUID: "aBroken", - LabelMode: fleet.AppleProfileLabelModeIncludeAll, - Labels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, + brokenExclude := &fleet.AppleProfileForReconcile{ + ProfileUUID: "aBrokenExclude", + IncludeMode: fleet.AppleProfileIncludeNone, + ExcludeLabels: []fleet.AppleProfileLabelRef{{LabelID: nil}}, } byTeam := map[uint][]*fleet.AppleProfileForReconcile{ - 0: {good, broken}, + 0: {good, brokenInclude, brokenExclude}, } require.False(t, isBrokenAppleProfile("aGood", byTeam)) - require.True(t, isBrokenAppleProfile("aBroken", byTeam)) + require.True(t, isBrokenAppleProfile("aBrokenInclude", byTeam)) + require.True(t, isBrokenAppleProfile("aBrokenExclude", byTeam)) require.False(t, isBrokenAppleProfile("aMissing", byTeam)) } From 483ef87838e624cb656cd0e55912137fe1748e47 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 19:27:25 +0000 Subject: [PATCH 4/5] Add diagnostic logs to batched Apple MDM reconciler Surface where a tick stalls and which commands the enqueue path fails on so the cursor-stuck symptom can be traced to a specific step. New log lines (look for cron=mdm_apple_profile_manager): - batched reconcile: listed hosts - batched reconcile: loaded profiles - batched reconcile: computed deltas (with to_install / to_remove) - batched reconcile: before bulk upsert - batched reconcile: enqueue complete (succeeded / failed cmd counts) - batched reconcile: failed command UUID (per failed cmd, with err) - batched reconcile: tick errored; cursor not advanced (with err) - batched reconcile: cursor advanced / tick complete, cursor unchanged - batched reconcile: ProcessAndEnqueueProfiles returned error The cursor-advance deferred block was rewritten as a switch so the outcome (errored / advanced / unchanged) is always logged with the cursor values, making it easy to spot ticks that stall mid-pass. https://claude.ai/code/session_01Vvy1keXRKZRzDbJQd7dzDn --- server/service/apple_mdm_batched.go | 37 ++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/service/apple_mdm_batched.go b/server/service/apple_mdm_batched.go index 647aee33093..aed40e5d1de 100644 --- a/server/service/apple_mdm_batched.go +++ b/server/service/apple_mdm_batched.go @@ -90,6 +90,8 @@ func ReconcileAppleProfilesBatched( if err != nil { return ctxerr.Wrap(ctx, err, "listing apple MDM hosts for reconcile batch") } + logger.InfoContext(ctx, "batched reconcile: listed hosts", + "cursor", cursor, "hosts_in_batch", len(hosts)) if len(hosts) == 0 { // End of pass or empty universe — reset cursor if it was advanced. @@ -113,10 +115,20 @@ func ReconcileAppleProfilesBatched( // Defer cursor advance on success only; on error we leave the cursor // untouched so the next tick retries the same window. defer func() { - if err == nil && cursor != nextCursor { + switch { + case err != nil: + logger.WarnContext(ctx, "batched reconcile: tick errored; cursor not advanced", + "cursor", cursor, "next_cursor", nextCursor, "err", err) + case cursor != nextCursor: if cerr := ds.SetMDMAppleReconcileCursor(ctx, nextCursor); cerr != nil { logger.WarnContext(ctx, "failed to advance apple MDM reconcile cursor", "err", cerr) + } else { + logger.InfoContext(ctx, "batched reconcile: cursor advanced", + "cursor", cursor, "next_cursor", nextCursor) } + default: + logger.InfoContext(ctx, "batched reconcile: tick complete, cursor unchanged", + "cursor", cursor) } }() @@ -136,6 +148,8 @@ func ReconcileAppleProfilesBatched( if err != nil { return ctxerr.Wrap(ctx, err, "listing apple profiles for reconcile") } + logger.InfoContext(ctx, "batched reconcile: loaded profiles", + "profile_count", len(allProfiles)) // Group profiles by team_id (0 = global). Each host evaluates only // global + its team's profiles. @@ -189,6 +203,10 @@ func ReconcileAppleProfilesBatched( // Filter out macOS-only profiles from iOS/iPadOS hosts (matches legacy). toInstall = fleet.FilterMacOSOnlyProfilesFromIOSIPadOS(toInstall) + logger.InfoContext(ctx, "batched reconcile: computed deltas", + "to_install", len(toInstall), "to_remove", len(toRemove), + "host_ids", len(hostIDs), "label_ids", len(labelIDs)) + if len(toInstall) == 0 && len(toRemove) == 0 { // Nothing to do this tick. Still advance the cursor. return nil @@ -790,6 +808,12 @@ func executeAppleReconcileBatch( return ctxerr.Wrap(ctx, err, "deleting profiles that didn't change") } + logger.InfoContext(ctx, "batched reconcile: before bulk upsert", + "host_profiles", len(hostProfiles), + "install_targets", len(installTargets), + "remove_targets", len(removeTargets), + "cleanup", len(hostProfilesToCleanup)) + if err := ds.BulkUpsertMDMAppleHostProfiles(ctx, hostProfiles); err != nil { return ctxerr.Wrap(ctx, err, "updating host profiles") } @@ -807,6 +831,7 @@ func executeAppleReconcileBatch( prefetchedContents, ) if err != nil { + logger.ErrorContext(ctx, "batched reconcile: ProcessAndEnqueueProfiles returned error", "err", err) for _, hp := range hostProfiles { if hp.Status != nil && *hp.Status == fleet.MDMDeliveryPending { hp.Status = nil @@ -819,6 +844,16 @@ func executeAppleReconcileBatch( return ctxerr.Wrap(ctx, err, "processing and enqueuing profiles") } + if enqueueResult != nil { + logger.InfoContext(ctx, "batched reconcile: enqueue complete", + "succeeded_cmds", len(enqueueResult.SucceededCmdUUIDs), + "failed_cmds", len(enqueueResult.FailedCmdUUIDs)) + for cmdUUID, ferr := range enqueueResult.FailedCmdUUIDs { + logger.WarnContext(ctx, "batched reconcile: failed command UUID", + "cmd_uuid", cmdUUID, "err", ferr) + } + } + hostProfsByCmdUUID := make(map[string][]*fleet.MDMAppleBulkUpsertHostProfilePayload, len(hostProfiles)) for _, hp := range hostProfiles { if hp.CommandUUID != "" { From 174b4f3eb17e9906ca38b0f7ac4b6513e4e551d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 19:45:57 +0000 Subject: [PATCH 5/5] Fix sql.NullTime scan error on label_created_at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ListAppleProfilesForReconcile used COALESCE(lbl.created_at, '2000-01-01 00:00:00') with a string-literal default, which made MySQL coerce the result column to VARCHAR. The MySQL driver returns that as []uint8, and sql.NullTime.Scan can only accept time.Time or nil — producing: sql: Scan error on column index 4, name "label_created_at": unsupported Scan, storing driver.Value type []uint8 into type *time.Time This errored on every tick, so the reconciler never reached the deferred cursor advance — the cursor stayed pinned at whatever value a prior successful run had set it to, and no further work got done. Drop the COALESCE so the column stays TIMESTAMP. NULL → invalid NullTime → zero time.Time, which the exclude-any handler already treats as "no timing check" (matching the broken-label semantics). https://claude.ai/code/session_01Vvy1keXRKZRzDbJQd7dzDn --- server/datastore/mysql/apple_mdm_batched.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/datastore/mysql/apple_mdm_batched.go b/server/datastore/mysql/apple_mdm_batched.go index f94d9912d3f..6ad1bead54e 100644 --- a/server/datastore/mysql/apple_mdm_batched.go +++ b/server/datastore/mysql/apple_mdm_batched.go @@ -106,13 +106,19 @@ func (ds *Datastore) ListAppleProfilesForReconcile(ctx context.Context) ([]*flee // Load label assignments, joining labels to get membership type and // label creation time (needed by the exclude-any handler). + // + // Do not COALESCE label_created_at to a string literal — MySQL would + // coerce the result column to VARCHAR and the driver returns []uint8, + // which sql.NullTime cannot scan. The exclude-any handler already + // treats a zero CreatedAt as "no timing check", which is the natural + // outcome of a NULL → invalid NullTime → zero time.Time. const labelStmt = ` SELECT mcpl.apple_profile_uuid AS profile_uuid, mcpl.label_id AS label_id, mcpl.exclude AS exclude, mcpl.require_all AS require_all, - COALESCE(lbl.created_at, '2000-01-01 00:00:00') AS label_created_at, + lbl.created_at AS label_created_at, COALESCE(lbl.label_membership_type, 0) AS label_membership_type FROM mdm_configuration_profile_labels mcpl LEFT JOIN labels lbl ON lbl.id = mcpl.label_id