From 81257ea47890f995a7034d5d376deab9701bbf66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 13:09:01 +0100 Subject: [PATCH 1/8] chore: added missing endpoint, verifiedby and mv refresh job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../members/identities/getMemberIdentities.ts | 3 +- .../identities/verifyMemberIdentity.ts | 1 + backend/src/api/public/v1/members/index.ts | 7 ++ .../getProjectAffiliations.ts | 116 +++++++++++++++++ ...ssing-indexes-for-project-affiliations.sql | 0 ...erified-to-member-segment-affiliations.sql | 0 ...ssing-indexes-for-project-affiliations.sql | 7 ++ ...erified-to-member-segment-affiliations.sql | 3 + backend/src/utils/mapper.ts | 1 + .../cron_service/src/jobs/refreshMvs.job.ts | 29 +++++ services/apps/cron_service/src/main.ts | 10 ++ .../lf-auth0/enrichMemberWithLFAuth0.ts | 4 + .../cvent/event-registrations/transformer.ts | 4 + .../data-access-layer/src/members/index.ts | 1 + .../src/members/projectAffiliations.ts | 118 ++++++++++++++++++ .../src/integrations/devto/processData.ts | 1 + .../src/integrations/discord/processData.ts | 2 + .../src/integrations/discourse/processData.ts | 2 + .../src/integrations/github/processData.ts | 6 + .../src/integrations/gitlab/processData.ts | 3 + .../src/integrations/groupsio/processData.ts | 6 + .../integrations/hackernews/processData.ts | 1 + .../src/integrations/linkedin/processData.ts | 1 + .../src/integrations/reddit/processData.ts | 2 + .../src/integrations/slack/processData.ts | 2 + .../integrations/stackoverflow/processData.ts | 3 + .../src/integrations/twitter/processData.ts | 3 + 27 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts create mode 100644 backend/src/database/migrations/U1772799041__add-missing-indexes-for-project-affiliations.sql create mode 100644 backend/src/database/migrations/U1773139177__add-verified-to-member-segment-affiliations.sql create mode 100644 backend/src/database/migrations/V1772799041__add-missing-indexes-for-project-affiliations.sql create mode 100644 backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql create mode 100644 services/apps/cron_service/src/jobs/refreshMvs.job.ts create mode 100644 services/libs/data-access-layer/src/members/projectAffiliations.ts diff --git a/backend/src/api/public/v1/members/identities/getMemberIdentities.ts b/backend/src/api/public/v1/members/identities/getMemberIdentities.ts index 621fecd717..4787101b21 100644 --- a/backend/src/api/public/v1/members/identities/getMemberIdentities.ts +++ b/backend/src/api/public/v1/members/identities/getMemberIdentities.ts @@ -27,11 +27,12 @@ export async function getMemberIdentities(req: Request, res: Response): Promise< const rawIdentities = await fetchMemberIdentities(qx, memberId) const identities = rawIdentities.map( - ({ id, value, platform, verified, source, createdAt, updatedAt }) => ({ + ({ id, value, platform, verified, verifiedBy, source, createdAt, updatedAt }) => ({ id, value, platform, verified, + verifiedBy: verifiedBy ?? null, source, createdAt, updatedAt, diff --git a/backend/src/api/public/v1/members/identities/verifyMemberIdentity.ts b/backend/src/api/public/v1/members/identities/verifyMemberIdentity.ts index aadf0be683..a7e725e587 100644 --- a/backend/src/api/public/v1/members/identities/verifyMemberIdentity.ts +++ b/backend/src/api/public/v1/members/identities/verifyMemberIdentity.ts @@ -54,6 +54,7 @@ function toReturn(identity: IMemberIdentity) { value: identity.value, platform: identity.platform, verified: identity.verified, + verifiedBy: identity.verifiedBy ?? null, source: identity.source, createdAt: identity.createdAt, updatedAt: identity.updatedAt, diff --git a/backend/src/api/public/v1/members/index.ts b/backend/src/api/public/v1/members/index.ts index 02d02f9dfa..b9f108de5a 100644 --- a/backend/src/api/public/v1/members/index.ts +++ b/backend/src/api/public/v1/members/index.ts @@ -7,6 +7,7 @@ import { SCOPES } from '@/security/scopes' import { getMemberIdentities } from './identities/getMemberIdentities' import { verifyMemberIdentity } from './identities/verifyMemberIdentity' import { getMemberMaintainerRoles } from './maintainer-roles/getMemberMaintainerRoles' +import { getProjectAffiliations } from './project-affiliations/getProjectAffiliations' import { resolveMemberByIdentities } from './resolveMember' import { createMemberWorkExperience } from './work-experiences/createMemberWorkExperience' import { deleteMemberWorkExperience } from './work-experiences/deleteMemberWorkExperience' @@ -37,6 +38,12 @@ export function membersRouter(): Router { safeWrap(getMemberMaintainerRoles), ) + router.get( + '/:memberId/project-affiliations', + requireScopes([SCOPES.READ_PROJECT_AFFILIATIONS]), + safeWrap(getProjectAffiliations), + ) + router.post( '/:memberId/work-experiences', requireScopes([SCOPES.WRITE_WORK_EXPERIENCES]), diff --git a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts new file mode 100644 index 0000000000..8e7aaa4d9b --- /dev/null +++ b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts @@ -0,0 +1,116 @@ +import type { Request, Response } from 'express' +import { z } from 'zod' + +import { NotFoundError } from '@crowd/common' +import { + MemberField, + fetchMemberProjectSegments, + fetchMemberSegmentAffiliationsWithOrg, + fetchMemberWorkExperienceAffiliations, + findMaintainerRoles, + findMemberById, + optionsQx, +} from '@crowd/data-access-layer' +import type { + ISegmentAffiliationWithOrg, + IWorkExperienceAffiliation, +} from '@crowd/data-access-layer' + +import { ok } from '@/utils/api' +import { validateOrThrow } from '@/utils/validation' + +const paramsSchema = z.object({ + memberId: z.uuid(), +}) + +function mapSegmentAffiliation(a: ISegmentAffiliationWithOrg) { + return { + id: a.id, + organizationId: a.organizationId, + organizationName: a.organizationName, + organizationLogo: a.organizationLogo ?? null, + verified: a.verified, + verifiedBy: a.verifiedBy ?? null, + source: 'ui', + startDate: a.dateStart ?? null, + endDate: a.dateEnd ?? null, + } +} + +function mapWorkExperienceAffiliation(a: IWorkExperienceAffiliation) { + return { + id: a.id, + organizationId: a.organizationId, + organizationName: a.organizationName, + organizationLogo: a.organizationLogo ?? null, + verified: a.verified ?? false, + verifiedBy: a.verifiedBy ?? null, + source: a.source ?? null, + startDate: a.dateStart ?? null, + endDate: a.dateEnd ?? null, + } +} + +export async function getProjectAffiliations(req: Request, res: Response): Promise { + const { memberId } = validateOrThrow(paramsSchema, req.params) + const qx = optionsQx(req) + + const member = await findMemberById(qx, memberId, [MemberField.ID]) + + if (!member) { + throw new NotFoundError('Member not found') + } + + const [projectSegments, maintainerRoles, segmentAffiliations, workExperiences] = + await Promise.all([ + fetchMemberProjectSegments(qx, memberId), + findMaintainerRoles(qx, [memberId]), + fetchMemberSegmentAffiliationsWithOrg(qx, memberId), + fetchMemberWorkExperienceAffiliations(qx, memberId), + ]) + + // Group maintainer roles by segmentId + const rolesBySegment = new Map() + for (const role of maintainerRoles) { + const existing = rolesBySegment.get(role.segmentId) ?? [] + existing.push(role) + rolesBySegment.set(role.segmentId, existing) + } + + // Group segment affiliations by segmentId + const affiliationsBySegment = new Map() + for (const aff of segmentAffiliations) { + const existing = affiliationsBySegment.get(aff.segmentId) ?? [] + existing.push(aff) + affiliationsBySegment.set(aff.segmentId, existing) + } + + const projectAffiliations = projectSegments.map((segment) => { + const roles = (rolesBySegment.get(segment.id) ?? []).map((r) => ({ + id: r.id, + role: r.role, + startDate: r.dateStart ?? null, + endDate: r.dateEnd ?? null, + repoUrl: r.url ?? null, + repoFileUrl: null, + })) + + // Use segment affiliations if they exist for this project, otherwise fall back to work experiences + const segmentAffs = affiliationsBySegment.get(segment.id) + const affiliations = segmentAffs + ? segmentAffs.map(mapSegmentAffiliation) + : workExperiences.map(mapWorkExperienceAffiliation) + + return { + id: segment.id, + projectSlug: segment.slug, + projectName: segment.name, + projectLogo: segment.url ?? null, + contributionCount: Number(segment.activityCount), + roles, + affiliations, + } + }) + + ok(res, { projectAffiliations }) +} diff --git a/backend/src/database/migrations/U1772799041__add-missing-indexes-for-project-affiliations.sql b/backend/src/database/migrations/U1772799041__add-missing-indexes-for-project-affiliations.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/U1773139177__add-verified-to-member-segment-affiliations.sql b/backend/src/database/migrations/U1773139177__add-verified-to-member-segment-affiliations.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1772799041__add-missing-indexes-for-project-affiliations.sql b/backend/src/database/migrations/V1772799041__add-missing-indexes-for-project-affiliations.sql new file mode 100644 index 0000000000..abbe3fa344 --- /dev/null +++ b/backend/src/database/migrations/V1772799041__add-missing-indexes-for-project-affiliations.sql @@ -0,0 +1,7 @@ +-- Add missing index on memberSegmentAffiliations for memberId lookups +create index concurrently if not exists "ix_memberSegmentAffiliations_memberId" + on "memberSegmentAffiliations" ("memberId"); + +-- Add missing index on mv_maintainer_roles materialized view for memberId lookups +create index concurrently if not exists "ix_mv_maintainer_roles_memberId" + on mv_maintainer_roles ("memberId"); diff --git a/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql b/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql new file mode 100644 index 0000000000..05055d7763 --- /dev/null +++ b/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql @@ -0,0 +1,3 @@ +alter table "memberSegmentAffiliations" + add column if not exists "verified" boolean not null default true, + add column if not exists "verifiedBy" varchar(255) default null; diff --git a/backend/src/utils/mapper.ts b/backend/src/utils/mapper.ts index 29195816a8..2ce358d1c2 100644 --- a/backend/src/utils/mapper.ts +++ b/backend/src/utils/mapper.ts @@ -8,6 +8,7 @@ export function toMemberWorkExperience(mo: IMemberRoleWithOrganization) { organizationLogo: mo.organizationLogo, jobTitle: mo.title ?? null, verified: mo.verified ?? false, + verifiedBy: mo.verifiedBy ?? null, source: mo.source ?? null, startDate: mo.dateStart ?? null, endDate: mo.dateEnd ?? null, diff --git a/services/apps/cron_service/src/jobs/refreshMvs.job.ts b/services/apps/cron_service/src/jobs/refreshMvs.job.ts new file mode 100644 index 0000000000..9c87ffdb5e --- /dev/null +++ b/services/apps/cron_service/src/jobs/refreshMvs.job.ts @@ -0,0 +1,29 @@ +import CronTime from 'cron-time-generator' + +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' + +import { IJobDefinition } from '../types' + +const MATERIALIZED_VIEWS = ['mv_maintainer_roles'] + +const job: IJobDefinition = { + name: 'refresh-mvs', + cronTime: CronTime.every(30).minutes(), + timeout: 10 * 60, // 10 minutes + process: async (ctx) => { + ctx.log.info('Starting materialized view refresh job!') + const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 1, 0) + + for (const mv of MATERIALIZED_VIEWS) { + ctx.log.info({ mv }, `Refreshing materialized view: ${mv}`) + const start = performance.now() + await dbConnection.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY "${mv}"`) + const duration = ((performance.now() - start) / 1000.0).toFixed(2) + ctx.log.info({ mv, duration }, `Refreshed materialized view ${mv} in ${duration}s`) + } + + ctx.log.info('Materialized view refresh job completed!') + }, +} + +export default job diff --git a/services/apps/cron_service/src/main.ts b/services/apps/cron_service/src/main.ts index 69a6dccc50..36e63408d6 100644 --- a/services/apps/cron_service/src/main.ts +++ b/services/apps/cron_service/src/main.ts @@ -4,6 +4,7 @@ import path from 'path' import pidusage from 'pidusage' import { getChildLogger, getServiceChildLogger, getServiceLogger } from '@crowd/logging' +import { SlackChannel, SlackPersona, sendSlackNotificationAsync } from '@crowd/slack' import { loadJobs } from './loader' import { IJobDefinition } from './types' @@ -145,6 +146,15 @@ const queueJob = async (job: IJobDefinition) => { } catch (err) { const diff = ((performance.now() - start) / 1000.0).toFixed(2) jobLogger.error(err, `Error while running a job! Job exited after ${diff} seconds!`) + + if (err instanceof Error && err.message.includes('did not finish in time')) { + await sendSlackNotificationAsync( + SlackChannel.ALERTS, + SlackPersona.CRITICAL_ALERTER, + `Cron job timed out: ${job.name}`, + `Job \`${job.name}\` was killed after exceeding its ${job.timeout}s timeout (ran for ${diff}s).`, + ) + } } finally { activeJobs.delete(job.name) } diff --git a/services/apps/members_enrichment_worker/src/workflows/lf-auth0/enrichMemberWithLFAuth0.ts b/services/apps/members_enrichment_worker/src/workflows/lf-auth0/enrichMemberWithLFAuth0.ts index b3a1478627..8b597eb9ef 100644 --- a/services/apps/members_enrichment_worker/src/workflows/lf-auth0/enrichMemberWithLFAuth0.ts +++ b/services/apps/members_enrichment_worker/src/workflows/lf-auth0/enrichMemberWithLFAuth0.ts @@ -60,6 +60,7 @@ export async function enrichMemberWithLFAuth0(member: IMember): Promise { platform: PlatformType.LFID, value: enriched.email.toLowerCase(), verified: true, + verifiedBy: 'lf-auth0', }) } @@ -78,6 +79,7 @@ export async function enrichMemberWithLFAuth0(member: IMember): Promise { type: MemberIdentityType.USERNAME, value: enriched.username.toLowerCase(), verified: true, + verifiedBy: 'lf-auth0', }) } @@ -100,6 +102,7 @@ export async function enrichMemberWithLFAuth0(member: IMember): Promise { platform: PlatformType.GITHUB, value: enrichmentGithub.profileData.nickname.toLowerCase(), verified: true, + verifiedBy: 'lf-auth0', }) } @@ -127,6 +130,7 @@ export async function enrichMemberWithLFAuth0(member: IMember): Promise { type: MemberIdentityType.EMAIL, value: githubEmail.email.toLowerCase(), verified: true, + verifiedBy: 'lf-auth0', }) } } diff --git a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts index e7334594c5..0b13f9bd5e 100644 --- a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts @@ -50,6 +50,7 @@ export class CventTransformer extends TransformerBase { value: email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.CVENT, sourceId, }, { @@ -57,6 +58,7 @@ export class CventTransformer extends TransformerBase { value: userName, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.CVENT, sourceId, }, ) @@ -66,6 +68,7 @@ export class CventTransformer extends TransformerBase { value: email, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.CVENT, sourceId, }) } @@ -76,6 +79,7 @@ export class CventTransformer extends TransformerBase { value: lfUsername, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.CVENT, sourceId, }) } diff --git a/services/libs/data-access-layer/src/members/index.ts b/services/libs/data-access-layer/src/members/index.ts index 346f476bb5..232f1619a1 100644 --- a/services/libs/data-access-layer/src/members/index.ts +++ b/services/libs/data-access-layer/src/members/index.ts @@ -8,4 +8,5 @@ export * from './attributes' export * from './dashboard' export * from './contributions' export * from './bot' +export * from './projectAffiliations' export { MemberQueryCache } from './queryCache' diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts new file mode 100644 index 0000000000..0e194dea7a --- /dev/null +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -0,0 +1,118 @@ +import { QueryExecutor } from '../queryExecutor' + +export interface IProjectAffiliationSegment { + id: string + slug: string + name: string + url: string | null + activityCount: number +} + +export interface ISegmentAffiliationWithOrg { + id: string + segmentId: string + organizationId: string + organizationName: string + organizationLogo: string | null + verified: boolean + verifiedBy: string | null + dateStart: string | null + dateEnd: string | null +} + +export interface IWorkExperienceAffiliation { + id: string + organizationId: string + organizationName: string + organizationLogo: string | null + verified: boolean + verifiedBy: string | null + source: string | null + dateStart: string | null + dateEnd: string | null +} + +/** + * Fetch all project-level segments a member has contributions in, + * along with contribution counts. + */ +export async function fetchMemberProjectSegments( + qx: QueryExecutor, + memberId: string, +): Promise { + return qx.select( + ` + SELECT + s.id, + s.slug, + s.name, + s.url, + msa."activityCount" + FROM "memberSegmentsAgg" msa + JOIN segments s ON msa."segmentId" = s.id + WHERE msa."memberId" = $(memberId) + AND s."parentSlug" IS NOT NULL + AND s."grandparentSlug" IS NULL + ORDER BY msa."activityCount" DESC + `, + { memberId }, + ) +} + +/** + * Fetch segment affiliations for a member with organization details. + * These are manual per-project overrides. + */ +export async function fetchMemberSegmentAffiliationsWithOrg( + qx: QueryExecutor, + memberId: string, +): Promise { + return qx.select( + ` + SELECT + msa.id, + msa."segmentId", + msa."organizationId", + o."displayName" AS "organizationName", + o.logo AS "organizationLogo", + msa.verified, + msa."verifiedBy", + msa."dateStart", + msa."dateEnd" + FROM "memberSegmentAffiliations" msa + JOIN organizations o ON msa."organizationId" = o.id + WHERE msa."memberId" = $(memberId) + `, + { memberId }, + ) +} + +/** + * Fetch work experiences for a member with organization details. + * Used as fallback affiliations when no segment affiliations exist for a project. + */ +export async function fetchMemberWorkExperienceAffiliations( + qx: QueryExecutor, + memberId: string, +): Promise { + return qx.select( + ` + SELECT + mo.id, + mo."organizationId", + o."displayName" AS "organizationName", + o.logo AS "organizationLogo", + mo.verified, + mo."verifiedBy", + mo.source, + mo."dateStart", + mo."dateEnd" + FROM "memberOrganizations" mo + JOIN organizations o ON mo."organizationId" = o.id + WHERE mo."memberId" = $(memberId) + AND mo."deletedAt" IS NULL + ORDER BY mo."dateStart" DESC NULLS LAST + `, + { memberId }, + ) +} diff --git a/services/libs/integrations/src/integrations/devto/processData.ts b/services/libs/integrations/src/integrations/devto/processData.ts index 09245ee30b..6749000721 100644 --- a/services/libs/integrations/src/integrations/devto/processData.ts +++ b/services/libs/integrations/src/integrations/devto/processData.ts @@ -23,6 +23,7 @@ const getMember = (comment: IDevToComment): IMemberData => { value: comment.user.username, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.DEVTO, }, ], attributes: { diff --git a/services/libs/integrations/src/integrations/discord/processData.ts b/services/libs/integrations/src/integrations/discord/processData.ts index 4aeb401de1..5b192cdbb6 100644 --- a/services/libs/integrations/src/integrations/discord/processData.ts +++ b/services/libs/integrations/src/integrations/discord/processData.ts @@ -73,6 +73,7 @@ const parseMembers = async (ctx: IProcessDataContext) => { value: username, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.DISCORD, }, ], attributes: { @@ -134,6 +135,7 @@ const parseMessage = async (ctx: IProcessDataContext) => { value: record.author.username, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.DISCORD, }, ], attributes: { diff --git a/services/libs/integrations/src/integrations/discourse/processData.ts b/services/libs/integrations/src/integrations/discourse/processData.ts index b236086c6b..768ac127d4 100644 --- a/services/libs/integrations/src/integrations/discourse/processData.ts +++ b/services/libs/integrations/src/integrations/discourse/processData.ts @@ -33,6 +33,7 @@ const parseUserIntoMember = (user: DiscourseUserResponse, forumHostname: string) type: MemberIdentityType.USERNAME, platform: PlatformType.DISCOURSE, verified: true, + verifiedBy: PlatformType.DISCOURSE, }, ], displayName: user.user.name, @@ -64,6 +65,7 @@ const parseUserIntoMember = (user: DiscourseUserResponse, forumHostname: string) type: MemberIdentityType.EMAIL, platform: PlatformType.DISCOURSE, verified: true, + verifiedBy: PlatformType.DISCOURSE, }) } diff --git a/services/libs/integrations/src/integrations/github/processData.ts b/services/libs/integrations/src/integrations/github/processData.ts index d3c5999467..4b927621b5 100644 --- a/services/libs/integrations/src/integrations/github/processData.ts +++ b/services/libs/integrations/src/integrations/github/processData.ts @@ -42,6 +42,7 @@ const parseBotMember = (memberData: GithubPrepareMemberOutput): IMemberData => { value: memberData.memberFromApi.login, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GITHUB, }, ], displayName: memberData.memberFromApi.login, @@ -72,6 +73,7 @@ const parseDeletedMember = (memberData: GithubPrepareMemberOutput): IMemberData value: memberData.memberFromApi.login, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GITHUB, }, ], displayName: 'Deleted User', @@ -115,6 +117,7 @@ const parseMember = (memberData: GithubPrepareMemberOutput): IMemberData => { type: MemberIdentityType.USERNAME, sourceId: memberFromApi.id.toString(), verified: true, + verifiedBy: PlatformType.GITHUB, }, ], displayName: memberFromApi?.name?.trim() || memberFromApi.login, @@ -146,6 +149,7 @@ const parseMember = (memberData: GithubPrepareMemberOutput): IMemberData => { value: email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GITHUB, }) } @@ -245,6 +249,7 @@ const parseOrgMember = (memberData: GithubPrepareOrgMemberOutput): IMemberData = value: orgFromApi.login, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GITHUB, }, ], displayName: orgFromApi?.name?.trim() || orgFromApi.login, @@ -270,6 +275,7 @@ const parseOrgMember = (memberData: GithubPrepareOrgMemberOutput): IMemberData = value: orgFromApi.email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GITHUB, }) } diff --git a/services/libs/integrations/src/integrations/gitlab/processData.ts b/services/libs/integrations/src/integrations/gitlab/processData.ts index 53dcfe017d..05557ba753 100644 --- a/services/libs/integrations/src/integrations/gitlab/processData.ts +++ b/services/libs/integrations/src/integrations/gitlab/processData.ts @@ -44,6 +44,7 @@ const parseUser = ({ data }: { data: UserSchema }): IMemberData | undefined => { value: data.username as string, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GITLAB, }, ...(data.public_email ? [ @@ -52,6 +53,7 @@ const parseUser = ({ data }: { data: UserSchema }): IMemberData | undefined => { value: data.public_email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GITLAB, }, ] : []), @@ -122,6 +124,7 @@ const parseUserFromCommit = ({ data }: { data: ExpandedCommitSchema }): IMemberD value: data.author_email as string, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GITLAB, }, ], displayName: data.author_name, diff --git a/services/libs/integrations/src/integrations/groupsio/processData.ts b/services/libs/integrations/src/integrations/groupsio/processData.ts index 9f156c5237..f83ad893ee 100644 --- a/services/libs/integrations/src/integrations/groupsio/processData.ts +++ b/services/libs/integrations/src/integrations/groupsio/processData.ts @@ -31,6 +31,7 @@ const processMemberJoin: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, { sourceId: memberData.user_id.toString(), @@ -38,6 +39,7 @@ const processMemberJoin: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, ], } @@ -72,6 +74,7 @@ const processMessage: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, { sourceId: memberData.user_id.toString(), @@ -79,6 +82,7 @@ const processMessage: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, ], } @@ -114,6 +118,7 @@ const processMemberLeft: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, { sourceId: memberData.user_id.toString(), @@ -121,6 +126,7 @@ const processMemberLeft: ProcessDataHandler = async (ctx) => { value: memberData.email, type: MemberIdentityType.EMAIL, verified: true, + verifiedBy: PlatformType.GROUPSIO, }, ], } diff --git a/services/libs/integrations/src/integrations/hackernews/processData.ts b/services/libs/integrations/src/integrations/hackernews/processData.ts index e038684a8f..9b8cbd19fe 100644 --- a/services/libs/integrations/src/integrations/hackernews/processData.ts +++ b/services/libs/integrations/src/integrations/hackernews/processData.ts @@ -31,6 +31,7 @@ async function parsePost(ctx: IProcessDataContext) { type: MemberIdentityType.USERNAME, sourceId: post.user.id, verified: true, + verifiedBy: PlatformType.HACKERNEWS, }, ], displayName: post.user.id, diff --git a/services/libs/integrations/src/integrations/linkedin/processData.ts b/services/libs/integrations/src/integrations/linkedin/processData.ts index a2eea4f749..879ff2c0c6 100644 --- a/services/libs/integrations/src/integrations/linkedin/processData.ts +++ b/services/libs/integrations/src/integrations/linkedin/processData.ts @@ -94,6 +94,7 @@ const getMember = async ( type: MemberIdentityType.USERNAME, sourceId, verified: true, + verifiedBy: PlatformType.LINKEDIN, }, ], displayName, diff --git a/services/libs/integrations/src/integrations/reddit/processData.ts b/services/libs/integrations/src/integrations/reddit/processData.ts index f11f9f953b..f346f5c7a1 100644 --- a/services/libs/integrations/src/integrations/reddit/processData.ts +++ b/services/libs/integrations/src/integrations/reddit/processData.ts @@ -130,6 +130,7 @@ function parseMember(activity: RedditPost | RedditComment): IMemberData { type: MemberIdentityType.USERNAME, sourceId: uniqueId, verified: true, + verifiedBy: PlatformType.REDDIT, }, ], displayName: 'Deleted User', @@ -143,6 +144,7 @@ function parseMember(activity: RedditPost | RedditComment): IMemberData { type: MemberIdentityType.USERNAME, sourceId: activity.author_fullname, verified: true, + verifiedBy: PlatformType.REDDIT, }, ], displayName: activity.author, diff --git a/services/libs/integrations/src/integrations/slack/processData.ts b/services/libs/integrations/src/integrations/slack/processData.ts index 028efc3d99..5ac6163ed6 100644 --- a/services/libs/integrations/src/integrations/slack/processData.ts +++ b/services/libs/integrations/src/integrations/slack/processData.ts @@ -34,6 +34,7 @@ function parseMember(record: SlackMember): IMemberData { type: MemberIdentityType.USERNAME, sourceId: record.id, verified: true, + verifiedBy: PlatformType.SLACK, }, ], attributes: { @@ -65,6 +66,7 @@ function parseMember(record: SlackMember): IMemberData { type: MemberIdentityType.EMAIL, sourceId: record.id, verified: true, + verifiedBy: PlatformType.SLACK, }) } diff --git a/services/libs/integrations/src/integrations/stackoverflow/processData.ts b/services/libs/integrations/src/integrations/stackoverflow/processData.ts index 45917d26b5..16996bc261 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/processData.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/processData.ts @@ -16,6 +16,7 @@ function parseMember(user: StackOverflowUser): IMemberData { type: MemberIdentityType.USERNAME, sourceId: user.user_id.toString(), verified: true, + verifiedBy: PlatformType.STACKOVERFLOW, }, ], attributes: { @@ -70,6 +71,7 @@ async function parseQuestion(ctx: IProcessDataContext) { value: question.owner.display_name, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.STACKOVERFLOW, }, ], } @@ -115,6 +117,7 @@ async function parseAnswer(ctx: IProcessDataContext) { value: answer.owner.display_name, type: MemberIdentityType.USERNAME, verified: true, + verifiedBy: PlatformType.STACKOVERFLOW, }, ], } diff --git a/services/libs/integrations/src/integrations/twitter/processData.ts b/services/libs/integrations/src/integrations/twitter/processData.ts index ca4d18073e..d7e0d50f81 100644 --- a/services/libs/integrations/src/integrations/twitter/processData.ts +++ b/services/libs/integrations/src/integrations/twitter/processData.ts @@ -24,6 +24,7 @@ const processTweetsWithMentions: ProcessDataHandler = async (ctx) => { platform: PlatformType.TWITTER, sourceId: data.member.id, verified: true, + verifiedBy: PlatformType.TWITTER, }, ], attributes: { @@ -83,6 +84,7 @@ const processTweetsWithHashtags: ProcessDataHandler = async (ctx) => { platform: PlatformType.TWITTER, sourceId: data.member.id, verified: true, + verifiedBy: PlatformType.TWITTER, }, ], attributes: { @@ -142,6 +144,7 @@ const processMemberReachUpdate: ProcessDataHandler = async (ctx) => { platform: PlatformType.TWITTER, sourceId: data.member.id, verified: true, + verifiedBy: PlatformType.TWITTER, }, ], attributes: { From cd6383c927909c9eca847aae271b41ba4181c69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 14:18:16 +0100 Subject: [PATCH 2/8] fix: fix for comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../getProjectAffiliations.ts | 2 +- .../cron_service/src/jobs/refreshMvs.job.ts | 20 +++++++++++-------- .../src/members/projectAffiliations.ts | 2 -- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts index 8e7aaa4d9b..1fd1fa8d47 100644 --- a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts +++ b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts @@ -105,7 +105,7 @@ export async function getProjectAffiliations(req: Request, res: Response): Promi id: segment.id, projectSlug: segment.slug, projectName: segment.name, - projectLogo: segment.url ?? null, + projectLogo: null, contributionCount: Number(segment.activityCount), roles, affiliations, diff --git a/services/apps/cron_service/src/jobs/refreshMvs.job.ts b/services/apps/cron_service/src/jobs/refreshMvs.job.ts index 9c87ffdb5e..8678dfce5b 100644 --- a/services/apps/cron_service/src/jobs/refreshMvs.job.ts +++ b/services/apps/cron_service/src/jobs/refreshMvs.job.ts @@ -14,15 +14,19 @@ const job: IJobDefinition = { ctx.log.info('Starting materialized view refresh job!') const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 1, 0) - for (const mv of MATERIALIZED_VIEWS) { - ctx.log.info({ mv }, `Refreshing materialized view: ${mv}`) - const start = performance.now() - await dbConnection.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY "${mv}"`) - const duration = ((performance.now() - start) / 1000.0).toFixed(2) - ctx.log.info({ mv, duration }, `Refreshed materialized view ${mv} in ${duration}s`) - } + try { + for (const mv of MATERIALIZED_VIEWS) { + ctx.log.info({ mv }, `Refreshing materialized view: ${mv}`) + const start = performance.now() + await dbConnection.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY "${mv}"`) + const duration = ((performance.now() - start) / 1000.0).toFixed(2) + ctx.log.info({ mv, duration }, `Refreshed materialized view ${mv} in ${duration}s`) + } - ctx.log.info('Materialized view refresh job completed!') + ctx.log.info('Materialized view refresh job completed!') + } finally { + await dbConnection.$pool.end() + } }, } diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts index 0e194dea7a..0976f087cb 100644 --- a/services/libs/data-access-layer/src/members/projectAffiliations.ts +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -4,7 +4,6 @@ export interface IProjectAffiliationSegment { id: string slug: string name: string - url: string | null activityCount: number } @@ -46,7 +45,6 @@ export async function fetchMemberProjectSegments( s.id, s.slug, s.name, - s.url, msa."activityCount" FROM "memberSegmentsAgg" msa JOIN segments s ON msa."segmentId" = s.id From 783e6401731c2a5c3e107a730d3230dd801859c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 20:36:42 +0100 Subject: [PATCH 3/8] fix: default to false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- ...V1773139177__add-verified-to-member-segment-affiliations.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql b/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql index 05055d7763..4b554f45ff 100644 --- a/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql +++ b/backend/src/database/migrations/V1773139177__add-verified-to-member-segment-affiliations.sql @@ -1,3 +1,3 @@ alter table "memberSegmentAffiliations" - add column if not exists "verified" boolean not null default true, + add column if not exists "verified" boolean not null default false, add column if not exists "verifiedBy" varchar(255) default null; From 511910e835ef2cf93df5713ff4ef62124a01192e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 20:52:12 +0100 Subject: [PATCH 4/8] fix: default to null for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../v1/members/project-affiliations/getProjectAffiliations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts index 1fd1fa8d47..88f1fc4304 100644 --- a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts +++ b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts @@ -31,7 +31,7 @@ function mapSegmentAffiliation(a: ISegmentAffiliationWithOrg) { organizationLogo: a.organizationLogo ?? null, verified: a.verified, verifiedBy: a.verifiedBy ?? null, - source: 'ui', + source: null, startDate: a.dateStart ?? null, endDate: a.dateEnd ?? null, } From 403fad7e6c0aec40a835d7c021024a9906f7b630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 21:34:15 +0100 Subject: [PATCH 5/8] fix: fixes for projectLogo and maintanerFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../project-affiliations/getProjectAffiliations.ts | 4 ++-- .../libs/data-access-layer/src/maintainers/index.ts | 10 ++++++++-- .../src/members/projectAffiliations.ts | 5 ++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts index 88f1fc4304..470da95a72 100644 --- a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts +++ b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts @@ -92,7 +92,7 @@ export async function getProjectAffiliations(req: Request, res: Response): Promi startDate: r.dateStart ?? null, endDate: r.dateEnd ?? null, repoUrl: r.url ?? null, - repoFileUrl: null, + repoFileUrl: r.maintainerFile ?? null, })) // Use segment affiliations if they exist for this project, otherwise fall back to work experiences @@ -105,7 +105,7 @@ export async function getProjectAffiliations(req: Request, res: Response): Promi id: segment.id, projectSlug: segment.slug, projectName: segment.name, - projectLogo: null, + projectLogo: segment.projectLogo ?? null, contributionCount: Number(segment.activityCount), roles, affiliations, diff --git a/services/libs/data-access-layer/src/maintainers/index.ts b/services/libs/data-access-layer/src/maintainers/index.ts index 3c20256ecc..0c2abe039c 100644 --- a/services/libs/data-access-layer/src/maintainers/index.ts +++ b/services/libs/data-access-layer/src/maintainers/index.ts @@ -16,6 +16,7 @@ export interface Maintainer { url: string repoType: MaintainerRepoType role: string + maintainerFile: string | null } export async function findMaintainerRoles( @@ -24,8 +25,13 @@ export async function findMaintainerRoles( ): Promise { return qx.select( ` - SELECT * FROM mv_maintainer_roles - WHERE "memberId" IN ($(memberIds:csv)) + SELECT + mmr.*, + rp."maintainerFile" + FROM mv_maintainer_roles mmr + LEFT JOIN public.repositories r ON r.url = mmr.url + LEFT JOIN git."repositoryProcessing" rp ON rp."repositoryId" = r.id + WHERE mmr."memberId" IN ($(memberIds:csv)) `, { memberIds, diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts index 0976f087cb..e3423a5b37 100644 --- a/services/libs/data-access-layer/src/members/projectAffiliations.ts +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -5,6 +5,7 @@ export interface IProjectAffiliationSegment { slug: string name: string activityCount: number + projectLogo: string | null } export interface ISegmentAffiliationWithOrg { @@ -45,9 +46,11 @@ export async function fetchMemberProjectSegments( s.id, s.slug, s.name, - msa."activityCount" + msa."activityCount", + ip."logoUrl" AS "projectLogo" FROM "memberSegmentsAgg" msa JOIN segments s ON msa."segmentId" = s.id + LEFT JOIN "insightsProjects" ip ON ip."segmentId" = s.id AND ip."deletedAt" IS NULL WHERE msa."memberId" = $(memberId) AND s."parentSlug" IS NOT NULL AND s."grandparentSlug" IS NULL From d60f1b83136b655ea2f8bac69ecd6328f45ace66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Tue, 10 Mar 2026 21:45:10 +0100 Subject: [PATCH 6/8] fix: fixes from bot reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../cron_service/src/jobs/refreshMvs.job.ts | 20 ++++++++----------- services/apps/cron_service/src/main.ts | 4 +++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/services/apps/cron_service/src/jobs/refreshMvs.job.ts b/services/apps/cron_service/src/jobs/refreshMvs.job.ts index 8678dfce5b..9c87ffdb5e 100644 --- a/services/apps/cron_service/src/jobs/refreshMvs.job.ts +++ b/services/apps/cron_service/src/jobs/refreshMvs.job.ts @@ -14,19 +14,15 @@ const job: IJobDefinition = { ctx.log.info('Starting materialized view refresh job!') const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 1, 0) - try { - for (const mv of MATERIALIZED_VIEWS) { - ctx.log.info({ mv }, `Refreshing materialized view: ${mv}`) - const start = performance.now() - await dbConnection.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY "${mv}"`) - const duration = ((performance.now() - start) / 1000.0).toFixed(2) - ctx.log.info({ mv, duration }, `Refreshed materialized view ${mv} in ${duration}s`) - } - - ctx.log.info('Materialized view refresh job completed!') - } finally { - await dbConnection.$pool.end() + for (const mv of MATERIALIZED_VIEWS) { + ctx.log.info({ mv }, `Refreshing materialized view: ${mv}`) + const start = performance.now() + await dbConnection.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY "${mv}"`) + const duration = ((performance.now() - start) / 1000.0).toFixed(2) + ctx.log.info({ mv, duration }, `Refreshed materialized view ${mv} in ${duration}s`) } + + ctx.log.info('Materialized view refresh job completed!') }, } diff --git a/services/apps/cron_service/src/main.ts b/services/apps/cron_service/src/main.ts index 36e63408d6..756233c63e 100644 --- a/services/apps/cron_service/src/main.ts +++ b/services/apps/cron_service/src/main.ts @@ -148,11 +148,13 @@ const queueJob = async (job: IJobDefinition) => { jobLogger.error(err, `Error while running a job! Job exited after ${diff} seconds!`) if (err instanceof Error && err.message.includes('did not finish in time')) { - await sendSlackNotificationAsync( + sendSlackNotificationAsync( SlackChannel.ALERTS, SlackPersona.CRITICAL_ALERTER, `Cron job timed out: ${job.name}`, `Job \`${job.name}\` was killed after exceeding its ${job.timeout}s timeout (ran for ${diff}s).`, + ).catch((slackErr) => + jobLogger.error(slackErr, 'Failed to send Slack timeout notification'), ) } } finally { From 51f4d3c1a8f8eced9758046b2614339106c2d7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 11 Mar 2026 09:22:11 +0100 Subject: [PATCH 7/8] fix: fixed missing verifiedBy on insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- services/libs/data-access-layer/src/members/identities.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/libs/data-access-layer/src/members/identities.ts b/services/libs/data-access-layer/src/members/identities.ts index 7cb77e1ee5..561d336137 100644 --- a/services/libs/data-access-layer/src/members/identities.ts +++ b/services/libs/data-access-layer/src/members/identities.ts @@ -208,6 +208,7 @@ export async function insertManyMemberIdentities( 'value', 'type', 'verified', + 'verifiedBy', ], identities.map((i) => { return { From 5a01991e35ab1468585e4cd64c4bc41cbc615e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Wed, 11 Mar 2026 11:05:22 +0100 Subject: [PATCH 8/8] fix: removed source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- .../v1/members/project-affiliations/getProjectAffiliations.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts index 470da95a72..e890a8826f 100644 --- a/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts +++ b/backend/src/api/public/v1/members/project-affiliations/getProjectAffiliations.ts @@ -31,7 +31,6 @@ function mapSegmentAffiliation(a: ISegmentAffiliationWithOrg) { organizationLogo: a.organizationLogo ?? null, verified: a.verified, verifiedBy: a.verifiedBy ?? null, - source: null, startDate: a.dateStart ?? null, endDate: a.dateEnd ?? null, }