diff --git a/.env b/.env index 32e64b6d44..5be823eaa9 100644 --- a/.env +++ b/.env @@ -28,8 +28,8 @@ REDIS_PORT=6379 ENABLE_SETTINGS_EXPERIMENT= -INTERNAL_FEED=http://localhost:6000/feed.json -PERSONALIZED_DIGEST_FEED=http://localhost:6000/feed.json +INTERNAL_FEED=http://localhost:6000 +PERSONALIZED_DIGEST_FEED=http://localhost:6000 LOFN_ORIGIN=http://localhost:6002 BRIEFING_FEED=http://api diff --git a/.infra/Pulumi.prod.yaml b/.infra/Pulumi.prod.yaml index c818e5a410..1020c0dd7a 100644 --- a/.infra/Pulumi.prod.yaml +++ b/.infra/Pulumi.prod.yaml @@ -72,7 +72,7 @@ config: secure: AAABAJEgS6b8xZv0j06KOawyNMdkTpGmzKs7Ryed5SofiLp3vSTjxhqQuKSVsaHjR5s= growthbookClientKey: secure: AAABAIVkkeJGes/CVRjekXu8GfXXhC9FihKkcb2C4peMA7yS7HbO52uViOrP5uIFmgAQ2A== - internalFeed: http://feed.feed.svc.cluster.local/api/feed? + internalFeed: http://feed.feed.svc.cluster.local jwtAudience: secure: AAABABYdafVA0XsTaLOHd1ROWJc6ggvXKA+e+lOjg3jduCMeGQ== jwtIssuer: @@ -106,7 +106,7 @@ config: paddleEnvironment: production paddleWebhookSecret: secure: AAABAMlKmiJ3FUQOJKHlOEbgpKnYDi6z62jEVcWDLdvhWuqRAah/dUZrgF9aISC86B56vR9M0wbp2/C7gJVy5if2n+O25cz6VJtJpSvgkSkfua6kDHX1qzNe1ewqxYh6VOBz1zLH - personalizedDigestFeed: http://feed-digest-server.feed/api/personalised + personalizedDigestFeed: http://feed-digest-server.feed personalizedDigestSecret: secure: AAABAN1TYaqznVMVbKDZtYEnLGB0VIzXDrpcmB6rBsImneobrDOS12T4XwbVvnexdZt1NBCnZRBwH4AL7OgpZkj/2UgT8Lzj0o7TdVSBeIeVDzKa/V6jlvJ4AskEC9rg postScraperOrigin: http://post-scraper-one-ai-server.content.svc.cluster.local diff --git a/__tests__/feeds.ts b/__tests__/feeds.ts index 9df9af5665..a0b949735a 100644 --- a/__tests__/feeds.ts +++ b/__tests__/feeds.ts @@ -67,6 +67,7 @@ import { base64 } from 'graphql-relay/utils/base64'; import { maxFeedsPerUser, UserVote } from '../src/types'; import { SubmissionFailErrorMessage } from '../src/errors'; import { baseFeedConfig, FeedConfigName } from '../src/integrations/feed'; +import { recswipeClient } from '../src/integrations/recswipe/clients'; import { ContentPreferenceStatus, ContentPreferenceType, @@ -77,6 +78,17 @@ import { ContentPreferenceWord } from '../src/entity/contentPreference/ContentPr import { ContentPreferenceUser } from '../src/entity/contentPreference/ContentPreferenceUser'; import { SubscriptionCycles } from '../src/paddle'; +jest.mock('../src/integrations/recswipe/clients', () => ({ + ...jest.requireActual('../src/integrations/recswipe/clients'), + recswipeClient: { + recommendTags: jest.fn(), + }, +})); + +const recommendTagsMock = recswipeClient.recommendTags as jest.MockedFunction< + typeof recswipeClient.recommendTags +>; + let app: FastifyInstance; let con: DataSource; let state: GraphQLTestingState; @@ -440,7 +452,7 @@ describe('query anonymousFeed', () => { loggedUser = '1'; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { total_pages: 1, page_size: 10, fresh_page_size: '4', @@ -470,7 +482,7 @@ describe('query anonymousFeed', () => { it('should safetly handle a case where the feed is empty', async () => { loggedUser = '1'; - nock('http://localhost:6000').post('/feed.json').reply(200, { + nock('http://localhost:6000').post('/api/feed').reply(200, { data: [], }); const res = await client.query(QUERY, { @@ -535,7 +547,7 @@ describe('query anonymousFeed', () => { ]); nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { total_pages: 1, page_size: 10, fresh_page_size: '4', @@ -626,7 +638,7 @@ describe('query anonymousFeed by time', () => { loggedUser = '1'; nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { // Verify the request includes order_by: 'date' for TIME ranking expect(body.order_by).toBe(FeedOrderBy.Date); expect(body.feed_config_name).toBe('for_you_by_date'); @@ -651,7 +663,7 @@ describe('query anonymousFeed by time', () => { loggedUser = '1'; nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body.order_by).toBe(FeedOrderBy.Date); return true; }) @@ -948,7 +960,7 @@ describe('query feed', () => { }, }); nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { total_pages: 1, page_size: 10, offset: 0, @@ -980,7 +992,7 @@ describe('query feed', () => { }, }); nock('http://localhost:6000') - .post('/feed.json', (body) => body.cursor === 'a') + .post('/api/feed', (body) => body.cursor === 'a') .reply(200, { data: [{ post_id: 'p1' }, { post_id: 'p4' }], cursor: 'b', @@ -1004,7 +1016,7 @@ describe('query feed', () => { }, }); nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body.allowed_post_types).toEqual(['article']); return true; }) @@ -1034,7 +1046,7 @@ describe('query feed', () => { }, }); nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body.allowed_post_types).toEqual(['article', 'collections']); return true; }) @@ -1064,7 +1076,7 @@ describe('query feed', () => { }, }); nock('http://localhost:6000') - .post('/feed.json') + .post('/api/feed') .reply(200, { data: [{ post_id: 'yt2' }], cursor: 'b', @@ -1079,7 +1091,7 @@ describe('query feed', () => { it('should return feed v2 with TIME ranking using chronological config', async () => { loggedUser = '1'; nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body.feed_config_name).toBe('for_you_by_date'); expect(body.order_by).toBe(FeedOrderBy.Date); expect(body.disable_engagement_filter).toBe(true); @@ -1180,7 +1192,7 @@ describe('query feedV2', () => { }, }); nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body.allowed_post_types).toEqual(['article']); expect(body.highlights_limit).toEqual(4); return true; @@ -1231,7 +1243,7 @@ describe('query feedV2', () => { }, }); nock('http://localhost:6000') - .post('/feed.json') + .post('/api/feed') .reply(200, { data: [ { post_id: 'p1', metadata: { p: 'post' } }, @@ -1330,7 +1342,7 @@ describe('query feedV2', () => { }, }); nock('http://localhost:6000') - .post('/feed.json') + .post('/api/feed') .reply(200, { data: [{ post_id: 'p1' }, { post_id: 'p4' }], cursor: 'next-cursor', @@ -1430,6 +1442,93 @@ describe('query feedV2', () => { }); }); +describe('query feedByTags', () => { + const QUERY = ` + query FeedByTags($tags: [String!]!, $ranking: Ranking, $first: Int, $version: Int) { + feedByTags(tags: $tags, ranking: $ranking, first: $first, version: $version) { + edges { + node { + id + } + } + } + } + `; + + it('should not authorize when not logged-in', () => + testQueryErrorCode( + client, + { query: QUERY, variables: { tags: ['javascript'] } }, + 'UNAUTHENTICATED', + )); + + it('should override allowed_tags with the provided tags', async () => { + loggedUser = '1'; + await saveFeedFixtures(); + + nock('http://localhost:6002') + .post('/config') + .reply(200, { + user_id: '1', + config: { providers: {} }, + }); + + let capturedBody: Record = {}; + nock('http://localhost:6000') + .post('/api/feed', (body) => { + capturedBody = body; + return true; + }) + .reply(200, { data: [{ post_id: 'p1' }] }); + + const res = await client.query(QUERY, { + variables: { + tags: ['rust', 'webdev'], + ranking: Ranking.POPULARITY, + first: 10, + version: 20, + }, + }); + + expect(res.errors).toBeFalsy(); + expect(capturedBody.allowed_tags).toEqual(['rust', 'webdev']); + expect(capturedBody.blocked_tags).toEqual( + expect.arrayContaining(['golang']), + ); + expect(capturedBody.blocked_sources).toEqual( + expect.arrayContaining(['b', 'c']), + ); + }); + + it('should reject empty tags array', async () => { + loggedUser = '1'; + await testQueryErrorCode( + client, + { query: QUERY, variables: { tags: [], version: 20 } }, + 'ZOD_VALIDATION_ERROR', + ); + }); + + it('should reject more than 20 tags', async () => { + loggedUser = '1'; + const tags = Array.from({ length: 21 }, (_, i) => `tag-${i}`); + await testQueryErrorCode( + client, + { query: QUERY, variables: { tags, version: 20 } }, + 'ZOD_VALIDATION_ERROR', + ); + }); + + it('should reject empty-string tags', async () => { + loggedUser = '1'; + await testQueryErrorCode( + client, + { query: QUERY, variables: { tags: ['rust', ''], version: 20 } }, + 'ZOD_VALIDATION_ERROR', + ); + }); +}); + describe('query feedByConfig', () => { const variables = { first: 10, @@ -1459,7 +1558,7 @@ describe('query feedByConfig', () => { it('should send provided config to feed service', async () => { nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { key: 'value', total_pages: 1, page_size: 10, @@ -2659,7 +2758,7 @@ describe('query channelFeed', () => { const cursor = base64('10'); nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body).toMatchObject({ feed_config_name: FeedConfigName.Channel, channel: 'devops', @@ -2707,7 +2806,7 @@ describe('query channelFeed', () => { 'should $name', async ({ contentCuration, expectedAllowedContentCurations }) => { nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { expect(body).toMatchObject({ feed_config_name: FeedConfigName.Channel, channel, @@ -2745,7 +2844,7 @@ describe('query similarPostsFeed', () => { it('should return posts from feed service', async () => { nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { feed_config_name: 'post_similarity', total_pages: 1, page_size: 30, @@ -2779,7 +2878,7 @@ describe('query randomSimilarPostsByTags', () => { // it('should return posts from feed service', async () => { // nock('http://localhost:6000') - // .post('/feed.json', { + // .post('/api/feed', { // feed_config_name: 'post_similarity', // total_pages: 1, // page_size: 3, @@ -2801,7 +2900,7 @@ describe('query randomSimilarPostsByTags', () => { it('should fallback to old algorithm', async () => { nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { feed_config_name: 'post_similarity', total_pages: 1, page_size: 3, @@ -2845,7 +2944,7 @@ describe('query randomSimilarPostsByTags', () => { it('should fallback to old algorithm even when tags not provided', async () => { nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { feed_config_name: 'post_similarity', total_pages: 1, page_size: 3, @@ -4196,7 +4295,7 @@ describe('query feedPreview', () => { }, ]); nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { feed_config_name: 'onboarding', total_pages: 1, page_size: 20, @@ -4219,7 +4318,7 @@ describe('query feedPreview', () => { loggedUser = '1'; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 20, offset: 0, @@ -5297,7 +5396,7 @@ describe('query customFeed', () => { isPlus = true; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 10, offset: 0, @@ -5357,7 +5456,7 @@ describe('query customFeed', () => { isPlus = true; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 10, offset: 0, @@ -5393,7 +5492,7 @@ describe('query customFeed', () => { isPlus = true; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 10, offset: 0, @@ -5441,7 +5540,7 @@ describe('query customFeed', () => { isPlus = true; nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 10, offset: 0, @@ -5502,7 +5601,7 @@ describe('query customFeed', () => { ]); nock('http://localhost:6000') - .post('/feed.json', { + .post('/api/feed', { user_id: '1', page_size: 10, offset: 0, @@ -5692,3 +5791,263 @@ describe('poll options ordering in feeds', () => { }); }); }); + +describe('query feedTagsList', () => { + const QUERY = /* GraphQL */ ` + query FeedTagsList($limit: Int) { + feedTagsList(limit: $limit) { + tags + } + } + `; + + beforeEach(() => { + recommendTagsMock.mockReset(); + }); + + it('should require auth', async () => { + return testQueryErrorCode( + client, + { query: QUERY, variables: { limit: 5 } }, + 'UNAUTHENTICATED', + ); + }); + + it('should fetch tags from feed service and cache them in user flags', async () => { + loggedUser = '1'; + let capturedBody: Record = {}; + nock('http://localhost:6000') + .post('/api/user_tags', (body) => { + capturedBody = body; + return true; + }) + .reply(200, { + data: ['ai-coding', 'llm', 'claude-code', 'ai-agents', 'anthropic'], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'ai-coding', + 'llm', + 'claude-code', + 'ai-agents', + 'anthropic', + ]); + expect(capturedBody).toEqual({ user_id: '1', limit: 5 }); + expect(recommendTagsMock).not.toHaveBeenCalled(); + + const user = await con.getRepository(User).findOneBy({ id: '1' }); + expect(user?.flags?.feedTagsList?.tags).toEqual([ + 'ai-coding', + 'llm', + 'claude-code', + 'ai-agents', + 'anthropic', + ]); + }); + + it('should return cached tags when fresh', async () => { + loggedUser = '1'; + await con.getRepository(User).update( + { id: '1' }, + { + flags: { + feedTagsList: { + tags: ['cached-tag-1', 'cached-tag-2'], + updatedAt: new Date().toISOString(), + }, + }, + }, + ); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'cached-tag-1', + 'cached-tag-2', + ]); + // No nock interceptor set up — if getUserTags were called, the request would error. + expect(nock.pendingMocks()).toEqual([]); + }); + + it('should re-fetch when cache is older than 24h', async () => { + loggedUser = '1'; + const stale = new Date(); + stale.setDate(stale.getDate() - 2); + await con.getRepository(User).update( + { id: '1' }, + { + flags: { + feedTagsList: { + tags: ['stale-tag'], + updatedAt: stale.toISOString(), + }, + }, + }, + ); + + const userTagsScope = nock('http://localhost:6000') + .post('/api/user_tags') + .reply(200, { + data: [ + 'fresh-tag-1', + 'fresh-tag-2', + 'fresh-tag-3', + 'fresh-tag-4', + 'fresh-tag-5', + ], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'fresh-tag-1', + 'fresh-tag-2', + 'fresh-tag-3', + 'fresh-tag-4', + 'fresh-tag-5', + ]); + expect(userTagsScope.isDone()).toBe(true); + }); + + it('should reject limit greater than 10', async () => { + loggedUser = '1'; + + await testQueryErrorCode( + client, + { query: QUERY, variables: { limit: 11 } }, + 'ZOD_VALIDATION_ERROR', + ); + expect(nock.pendingMocks()).toEqual([]); + }); + + it('should backfill with recswipe when feed returns fewer than limit', async () => { + loggedUser = '1'; + nock('http://localhost:6000') + .post('/api/user_tags') + .reply(200, { data: ['ai-coding', 'llm'] }); + + recommendTagsMock.mockResolvedValue({ + recommended_tags: [ + { tag: 'machine-learning', score: 0.9 }, + { tag: 'pytorch', score: 0.8 }, + { tag: 'tensorflow', score: 0.7 }, + ], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'ai-coding', + 'llm', + 'machine-learning', + 'pytorch', + 'tensorflow', + ]); + expect(recommendTagsMock).toHaveBeenCalledWith('1', { + selectedTags: ['ai-coding', 'llm'], + n: 3, + }); + }); + + it('should cache empty and return empty when feed service fails', async () => { + loggedUser = '1'; + nock('http://localhost:6000') + .post('/api/user_tags') + .replyWithError('feed service down'); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([]); + expect(recommendTagsMock).not.toHaveBeenCalled(); + + const user = await con.getRepository(User).findOneBy({ id: '1' }); + expect(user?.flags?.feedTagsList?.tags).toEqual([]); + expect(user?.flags?.feedTagsList?.updatedAt).toEqual(expect.any(String)); + }); + + it('should dedupe overlap between feed and recswipe results', async () => { + loggedUser = '1'; + nock('http://localhost:6000') + .post('/api/user_tags') + .reply(200, { data: ['ai-coding', 'llm'] }); + + recommendTagsMock.mockResolvedValue({ + recommended_tags: [ + { tag: 'llm', score: 0.95 }, + { tag: 'machine-learning', score: 0.9 }, + { tag: 'pytorch', score: 0.8 }, + { tag: 'tensorflow', score: 0.7 }, + ], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'ai-coding', + 'llm', + 'machine-learning', + 'pytorch', + 'tensorflow', + ]); + }); + + it('should dedupe duplicates within the feed-service response', async () => { + loggedUser = '1'; + nock('http://localhost:6000') + .post('/api/user_tags') + .reply(200, { data: ['rust', 'rust', 'golang', 'rust'] }); + + recommendTagsMock.mockResolvedValue({ + recommended_tags: [ + { tag: 'docker', score: 0.9 }, + { tag: 'kubernetes', score: 0.8 }, + { tag: 'python', score: 0.7 }, + ], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'rust', + 'golang', + 'docker', + 'kubernetes', + 'python', + ]); + }); + + it('should re-fetch when cached updatedAt is more than 24h in the future', async () => { + loggedUser = '1'; + const farFuture = new Date(Date.now() + 2 * 24 * 60 * 60 * 1000); + await con.getRepository(User).update( + { id: '1' }, + { + flags: { + feedTagsList: { + tags: ['stale-future-tag'], + updatedAt: farFuture.toISOString(), + }, + }, + }, + ); + + const userTagsScope = nock('http://localhost:6000') + .post('/api/user_tags') + .reply(200, { + data: ['fresh-1', 'fresh-2', 'fresh-3', 'fresh-4', 'fresh-5'], + }); + + const res = await client.query(QUERY, { variables: { limit: 5 } }); + expect(res.errors).toBeFalsy(); + expect(res.data.feedTagsList.tags).toEqual([ + 'fresh-1', + 'fresh-2', + 'fresh-3', + 'fresh-4', + 'fresh-5', + ]); + expect(userTagsScope.isDone()).toBe(true); + }); +}); diff --git a/__tests__/integrations/feed.ts b/__tests__/integrations/feed.ts index a71792be82..64d063cacb 100644 --- a/__tests__/integrations/feed.ts +++ b/__tests__/integrations/feed.ts @@ -47,7 +47,7 @@ import { ContentPreferenceUser } from '../../src/entity/contentPreference/Conten let con: DataSource; let ctx: Context; -const url = 'http://localhost:3000/feed.json'; +const url = 'http://localhost:3000'; const config: FeedConfig = { page_size: 2, offset: 0, @@ -108,22 +108,22 @@ describe('FeedClient', () => { it('should parse feed service response', async () => { nock(url) // eslint-disable-next-line @typescript-eslint/no-explicit-any - .post('', config as any) + .post('/api/feed', config as any) .reply(200, rawFeedResponse); const feedClient = new FeedClient(url); - const feed = await feedClient.fetchFeed(ctx, 'id', config); + const feed = await feedClient.fetchFeed(ctx, '/api/feed', config); expect(feed).toEqual(feedResponse); }); it('should merge tyr metadata with feed metadata', async () => { nock(url) // eslint-disable-next-line @typescript-eslint/no-explicit-any - .post('', config as any) + .post('/api/feed', config as any) .reply(200, rawFeedResponse); const feedClient = new FeedClient(url); - const feed = await feedClient.fetchFeed(ctx, 'id', config, { + const feed = await feedClient.fetchFeed(ctx, '/api/feed', config, { mab: { test: 'da' }, }); expect(feed).toEqual({ @@ -165,7 +165,7 @@ describe('FeedClient', () => { it('should preserve highlight items from the feed service response', async () => { nock(url) // eslint-disable-next-line @typescript-eslint/no-explicit-any - .post('', config as any) + .post('/api/feed', config as any) .reply(200, { data: [ { post_id: '1', metadata: { p: 'a' } }, @@ -179,7 +179,7 @@ describe('FeedClient', () => { }); const feedClient = new FeedClient(url); - const feed = await feedClient.fetchFeed(ctx, 'id', config); + const feed = await feedClient.fetchFeed(ctx, '/api/feed', config); expect(feed).toEqual({ data: [ @@ -202,11 +202,11 @@ describe('FeedClient', () => { nock(url) // eslint-disable-next-line @typescript-eslint/no-explicit-any - .post('', config as any) + .post('/api/feed', config as any) .reply(200, responseWithStaleCursor); const feedClient = new FeedClient(url); - const feed = await feedClient.fetchFeed(ctx, 'id', config); + const feed = await feedClient.fetchFeed(ctx, '/api/feed', config); expect(feed).toMatchObject({ cursor: 'abc123', staleCursor: true, @@ -216,13 +216,35 @@ describe('FeedClient', () => { it('should not include staleCursor when not present in response', async () => { nock(url) // eslint-disable-next-line @typescript-eslint/no-explicit-any - .post('', config as any) + .post('/api/feed', config as any) .reply(200, rawFeedResponse); const feedClient = new FeedClient(url); - const feed = await feedClient.fetchFeed(ctx, 'id', config); + const feed = await feedClient.fetchFeed(ctx, '/api/feed', config); expect(feed.staleCursor).toBeUndefined(); }); + + describe('getUserTags', () => { + const expectedBody = { user_id: 'u1', limit: 10 }; + + it('should return the data array from the feed service', async () => { + nock(url) + .post('/api/user_tags', expectedBody) + .reply(200, { data: ['ai-coding', 'llm'] }); + + const feedClient = new FeedClient(url); + const tags = await feedClient.getUserTags('u1', 10); + expect(tags).toEqual(['ai-coding', 'llm']); + }); + + it('should return an empty array when data is missing', async () => { + nock(url).post('/api/user_tags', expectedBody).reply(200, {}); + + const feedClient = new FeedClient(url); + const tags = await feedClient.getUserTags('u1', 10); + expect(tags).toEqual([]); + }); + }); }); describe('connectionFromNodes with staleCursor', () => { @@ -1081,7 +1103,7 @@ describe('versionToTimeFeedGenerator', () => { it('should generate config with chronological settings and no lofn', async () => { let capturedBody: Record = {}; nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/feed', (body) => { capturedBody = body; return true; }) diff --git a/__tests__/routes/public/customFeeds.ts b/__tests__/routes/public/customFeeds.ts index 9c9266e864..95ab6305f5 100644 --- a/__tests__/routes/public/customFeeds.ts +++ b/__tests__/routes/public/customFeeds.ts @@ -85,7 +85,7 @@ describe('GET /public/v1/feeds/custom/:feedId', () => { // Mock the feed service nock('http://localhost:6000') - .post('/feed.json') + .post('/api/feed') .reply(200, { data: [{ post_id: 'p1' }, { post_id: 'p2' }], cursor: null, diff --git a/__tests__/workers/personalizedDigestEmail.ts b/__tests__/workers/personalizedDigestEmail.ts index 1b3b81026c..093de3fe3f 100644 --- a/__tests__/workers/personalizedDigestEmail.ts +++ b/__tests__/workers/personalizedDigestEmail.ts @@ -186,7 +186,7 @@ beforeEach(async () => { .map((post) => ({ post_id: post.id })); nockScope = nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/personalised', (body) => { nockBody = body; return true; @@ -610,7 +610,7 @@ describe('personalizedDigestEmail worker', () => { nock.cleanAll(); nockScope = nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/personalised', (body) => { nockBody = body; return true; @@ -834,7 +834,7 @@ describe('personalizedDigestEmail worker', () => { .slice(0, 5) .map((post) => ({ post_id: post.id })); - nock('http://localhost:6000').post('/feed.json').reply(200, { + nock('http://localhost:6000').post('/api/personalised').reply(200, { data: mockedPostIds, rows: mockedPostIds.length, }); @@ -920,7 +920,7 @@ describe('personalizedDigestEmail worker', () => { nock.cleanAll(); nockScope = nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/personalised', (body) => { nockBody = body; return true; }) @@ -1005,7 +1005,7 @@ describe('personalizedDigestEmail worker', () => { .slice(0, 3) .map((post) => ({ post_id: post.id })); - nock('http://localhost:6000').post('/feed.json').reply(200, { + nock('http://localhost:6000').post('/api/personalised').reply(200, { data: newPostIds, rows: newPostIds.length, }); @@ -1578,7 +1578,7 @@ describe('personalizedDigestEmail worker', () => { nock.cleanAll(); nockScope = nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/personalised', (body) => { nockBody = body; return true; @@ -1644,7 +1644,7 @@ describe('personalizedDigestEmail worker', () => { nock.cleanAll(); nockScope = nock('http://localhost:6000') - .post('/feed.json', (body) => { + .post('/api/personalised', (body) => { nockBody = body; return true; diff --git a/src/common/feedTagsList.ts b/src/common/feedTagsList.ts new file mode 100644 index 0000000000..996026054f --- /dev/null +++ b/src/common/feedTagsList.ts @@ -0,0 +1,106 @@ +import type { DataSource } from 'typeorm'; +import { User } from '../entity/user/User'; +import { feedClient } from '../integrations/feed/generators'; +import { recswipeClient } from '../integrations/recswipe/clients'; +import { queryReadReplica } from './queryReadReplica'; +import { updateFlagsStatement } from './utils'; +import { ONE_DAY_IN_SECONDS } from './constants'; +import { logger } from '../logger'; + +export type FeedTagsList = { + tags: string[]; +}; + +const CACHE_TTL_MS = ONE_DAY_IN_SECONDS * 1000; + +const isFresh = (updatedAt: string): boolean => { + const ts = Date.parse(updatedAt); + if (Number.isNaN(ts)) { + return false; + } + return Math.abs(Date.now() - ts) < CACHE_TTL_MS; +}; + +const dedupeKeepOrder = (tags: string[]): string[] => { + const seen = new Set(); + const result: string[] = []; + for (const tag of tags) { + if (!seen.has(tag)) { + seen.add(tag); + result.push(tag); + } + } + return result; +}; + +const writeCache = async ({ + con, + userId, + tags, +}: { + con: DataSource; + userId: string; + tags: string[]; +}): Promise => { + await con.getRepository(User).update( + { id: userId }, + { + flags: updateFlagsStatement({ + feedTagsList: { + tags, + updatedAt: new Date().toISOString(), + }, + }), + }, + ); +}; + +export const getFeedTagsList = async ({ + con, + userId, + limit, +}: { + con: DataSource; + userId: string; + limit: number; +}): Promise => { + const user = await queryReadReplica(con, ({ queryRunner }) => + queryRunner.manager + .getRepository(User) + .findOne({ where: { id: userId }, select: ['id', 'flags'] }), + ); + + const cached = user?.flags?.feedTagsList; + if (cached && isFresh(cached.updatedAt)) { + return { tags: cached.tags.slice(0, limit) }; + } + + let tags: string[]; + try { + tags = await feedClient.getUserTags(userId, limit); + } catch (err) { + logger.error( + { err, userId }, + 'feedClient.getUserTags failed; caching empty feedTagsList', + ); + await writeCache({ con, userId, tags: [] }); + return { tags: [] }; + } + + tags = dedupeKeepOrder(tags); + + if (tags.length < limit) { + const supplement = await recswipeClient.recommendTags(userId, { + selectedTags: tags, + n: limit - tags.length, + }); + const supplementTags = (supplement.recommended_tags ?? []).map( + (t) => t.tag, + ); + tags = dedupeKeepOrder([...tags, ...supplementTags]).slice(0, limit); + } + + await writeCache({ con, userId, tags }); + + return { tags }; +}; diff --git a/src/common/personalizedDigest.ts b/src/common/personalizedDigest.ts index e2de8bf167..a5e3028fe1 100644 --- a/src/common/personalizedDigest.ts +++ b/src/common/personalizedDigest.ts @@ -430,7 +430,7 @@ export const getPersonalizedDigestEmailPayload = async ({ }; const feedResponse = await personalizedDigestFeedClient.fetchFeed( { log: logger }, - personalizedDigest.userId, + '/api/personalised', feedConfigPayload, ); diff --git a/src/common/schema/feedByTags.ts b/src/common/schema/feedByTags.ts new file mode 100644 index 0000000000..c2c6fc5a6f --- /dev/null +++ b/src/common/schema/feedByTags.ts @@ -0,0 +1,7 @@ +import { z } from 'zod'; + +export const FEED_BY_TAGS_MAX_TAGS = 20; + +export const feedByTagsInputSchema = z.object({ + tags: z.array(z.string().min(1)).min(1).max(FEED_BY_TAGS_MAX_TAGS), +}); diff --git a/src/common/schema/feedTagsList.ts b/src/common/schema/feedTagsList.ts new file mode 100644 index 0000000000..ca8c600416 --- /dev/null +++ b/src/common/schema/feedTagsList.ts @@ -0,0 +1,12 @@ +import { z } from 'zod'; + +export const FEED_TAGS_LIST_MAX_LIMIT = 10; + +export const feedTagsListInputSchema = z.object({ + limit: z + .number() + .int() + .min(1) + .max(FEED_TAGS_LIST_MAX_LIMIT) + .default(FEED_TAGS_LIST_MAX_LIMIT), +}); diff --git a/src/entity/user/User.ts b/src/entity/user/User.ts index a655242ebc..faa11addf1 100644 --- a/src/entity/user/User.ts +++ b/src/entity/user/User.ts @@ -51,6 +51,10 @@ export type UserFlags = Partial<{ lastExtensionUse: Date | null; inDeletion: boolean; hackathonParticipant: boolean; + feedTagsList: { + tags: string[]; + updatedAt: string; + }; }>; export type UserFlagsPublic = Pick; diff --git a/src/integrations/feed/clients.ts b/src/integrations/feed/clients.ts index c0020d7f9c..19dffed37e 100644 --- a/src/integrations/feed/clients.ts +++ b/src/integrations/feed/clients.ts @@ -7,6 +7,8 @@ import { GarmrNoopService, IGarmrClient, IGarmrService } from '../garmr'; import { Briefing, UserBriefingRequest } from '@dailydotdev/schema'; import type { JsonValue } from '@bufbuild/protobuf'; import { ServiceError } from '../../errors'; +import { isMockEnabled } from '../../mocks/common'; +import { mockUserTagsResponse } from '../../mocks/feed/userTags'; type RawFeedDataItem = { post_id: string; @@ -52,12 +54,12 @@ export class FeedClient implements IFeedClient, IGarmrClient { async fetchFeed( ctx: unknown, - feedId: string, + path: string, config: FeedConfig, extraMetadata?: GenericMetadata, ): Promise { const res = await this.garmr.execute(() => { - return fetchParse(this.url, { + return fetchParse(`${this.url}${path}`, { ...this.fetchOptions, method: 'POST', body: JSON.stringify(config), @@ -172,4 +174,23 @@ export class FeedClient implements IFeedClient, IGarmrClient { updatedAt, }; } + + async getUserTags(userId: string, limit: number): Promise { + if (isMockEnabled()) { + return mockUserTagsResponse(limit); + } + + const result = await this.garmr.execute(() => + fetchParse<{ data: string[] }>(`${this.url}/api/user_tags`, { + ...this.fetchOptions, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ user_id: userId, limit }), + }), + ); + + return result?.data ?? []; + } } diff --git a/src/integrations/feed/generators.ts b/src/integrations/feed/generators.ts index fc300a38c1..60f9919143 100644 --- a/src/integrations/feed/generators.ts +++ b/src/integrations/feed/generators.ts @@ -40,13 +40,7 @@ export class FeedGenerator { async generate(ctx: Context, opts: DynamicConfig): Promise { const { config, extraMetadata } = await this.config.generate(ctx, opts); - const userId = opts.user_id; - return this.client.fetchFeed( - ctx, - this.feedId ?? userId!, - config, - extraMetadata, - ); + return this.client.fetchFeed(ctx, '/api/feed', config, extraMetadata); } withConfigTransform( diff --git a/src/integrations/feed/types.ts b/src/integrations/feed/types.ts index 9db26c66e4..fd20c1c42a 100644 --- a/src/integrations/feed/types.ts +++ b/src/integrations/feed/types.ts @@ -125,12 +125,13 @@ export interface IFeedClient { /** * Fetches the feed from the service * @param ctx GraphQL context - * @param feedId The feed ID (used for caching primarily) + * @param path The feed-service request path (e.g. `/api/feed`, `/api/personalised`) * @param config The feed config + * @param extraMetadata Metadata merged into every item's feedMeta */ fetchFeed( ctx: Context, - feedId: string, + path: string, config: FeedConfig, extraMetadata?: GenericMetadata, ): Promise; diff --git a/src/mocks/common.ts b/src/mocks/common.ts new file mode 100644 index 0000000000..cb2ae39e54 --- /dev/null +++ b/src/mocks/common.ts @@ -0,0 +1,5 @@ +/** + * Shared helper used by domain-specific mock modules under `src/mocks/`. + */ +export const isMockEnabled = (): boolean => + process.env.MOCK_EXTERNAL_SERVICES === 'true'; diff --git a/src/mocks/feed/userTags.ts b/src/mocks/feed/userTags.ts new file mode 100644 index 0000000000..d0e06d0e40 --- /dev/null +++ b/src/mocks/feed/userTags.ts @@ -0,0 +1,19 @@ +/** + * Mock response for the feed service `/api/user_tags` endpoint. + * Returns a static list when `MOCK_EXTERNAL_SERVICES=true` is set in env. + */ +const MOCK_USER_TAGS = [ + 'javascript', + 'typescript', + 'react', + 'nodejs', + 'python', + 'golang', + 'rust', + 'docker', + 'kubernetes', + 'webdev', +]; + +export const mockUserTagsResponse = (limit: number): string[] => + MOCK_USER_TAGS.slice(0, limit); diff --git a/src/mocks/opportunity/services.ts b/src/mocks/opportunity/services.ts index 1e0804529d..1fc83b42e3 100644 --- a/src/mocks/opportunity/services.ts +++ b/src/mocks/opportunity/services.ts @@ -11,11 +11,7 @@ import { } from '@dailydotdev/schema'; import { PersonaliseState } from '../../integrations/snotra'; -/** - * Check if external services should be mocked - */ -export const isMockEnabled = (): boolean => - process.env.MOCK_EXTERNAL_SERVICES === 'true'; +export { isMockEnabled } from '../common'; /** * Mock response for Brokkr parseOpportunity service call diff --git a/src/schema/feeds.ts b/src/schema/feeds.ts index 4a384216b4..dcef91e68f 100644 --- a/src/schema/feeds.ts +++ b/src/schema/feeds.ts @@ -101,6 +101,7 @@ import { getForYouFeedGenerator, toFeedV2PostConnection, } from './feedV2'; +import { feedByTagsInputSchema } from '../common/schema/feedByTags'; interface GQLTagsCategory { id: string; @@ -576,6 +577,48 @@ export const typeDefs = /* GraphQL */ ` supportedTypes: [String!] ): PostConnection! + """ + Get a personalized feed limited to one or more tags. Reuses the standard feed pipeline + (blocked tags/sources/users, content prefs, etc.) but overrides allowed_tags with the + supplied list — the user's followed tags are not included. + """ + feedByTags( + """ + Tags to allow in the feed (overrides the user's followed tags) + """ + tags: [String!]! + + """ + Time the pagination started to ignore new items + """ + now: DateTime + + """ + Paginate after opaque cursor + """ + after: String + + """ + Paginate first + """ + first: Int + + """ + Ranking criteria for the feed + """ + ranking: Ranking = POPULARITY + + """ + Version of the feed algorithm + """ + version: Int = 1 + + """ + Array of supported post types + """ + supportedTypes: [String!] + ): PostConnection! @auth + """ Get a single tag feed """ @@ -1170,6 +1213,11 @@ interface TagFeedArgs extends FeedArgs { tag: string; } +interface FeedByTagsArgs extends FeedArgs { + tags: string[]; + version: number; +} + interface KeywordFeedArgs extends FeedArgs { keyword: string; } @@ -1702,6 +1750,27 @@ export const resolvers: IResolvers = { } return feedResolverV1(source, args, ctx, info); }, + feedByTags: (source, args: FeedByTagsArgs, ctx: AuthContext, info) => { + const { tags } = feedByTagsInputSchema.parse(args); + const generator = getForYouFeedGenerator(args).withConfigTransform( + (result) => ({ + ...result, + config: { + ...result.config, + allowed_tags: tags, + }, + }), + ); + return feedResolverCursor( + source, + { + ...(args as FeedArgs), + generator, + }, + ctx, + info, + ); + }, feedV2: (source, args: FeedV2Args, ctx: AuthContext, info) => shouldUseFeedGenerator(args) ? feedV2QueryResolver(source, args, ctx, info) diff --git a/src/schema/sources.ts b/src/schema/sources.ts index 1bc6223a94..93df5429ab 100644 --- a/src/schema/sources.ts +++ b/src/schema/sources.ts @@ -67,7 +67,7 @@ import { import { validateAndTransformHandle } from '../common/handles'; import { QueryBuilder } from '../graphorm/graphorm'; import type { GQLTagResults } from './tags'; -import { MIN_SEARCH_QUERY_LENGTH } from './tags'; +import { MIN_SEARCH_QUERY_LENGTH } from '../types'; import { SourceSimilarityView } from '../entity/SourceSimilarityView'; import { SourceTagView } from '../entity/SourceTagView'; import { TrendingSource } from '../entity/TrendingSource'; diff --git a/src/schema/tags.ts b/src/schema/tags.ts index 04bde710b8..bcb30db86a 100644 --- a/src/schema/tags.ts +++ b/src/schema/tags.ts @@ -1,5 +1,5 @@ import { IResolvers } from '@graphql-tools/utils'; -import { BaseContext, Context } from '../Context'; +import { AuthContext, BaseContext, Context } from '../Context'; import { Keyword } from '../entity'; import { TagRecommendation } from '../entity/TagRecommendation'; import { In, Not, ObjectType } from 'typeorm'; @@ -9,6 +9,14 @@ import graphorm from '../graphorm'; import { GQLKeyword } from './keywords'; import { TrendingTag } from '../entity/TrendingTag'; import { PopularTag } from '../entity/PopularTag'; +import { getFeedTagsList } from '../common/feedTagsList'; +import { feedTagsListInputSchema } from '../common/schema/feedTagsList'; +import { z } from 'zod'; +import { + MIN_SEARCH_QUERY_LENGTH, + RECOMMENDED_TAGS_LIMIT, + SEARCH_TAGS_LIMIT, +} from '../types'; interface GQLTag { name: string; @@ -21,12 +29,6 @@ interface GQLTagSearchResults { export type GQLTagResults = Pick; -export const RECOMMENDED_TAGS_LIMIT = 5; - -export const MIN_SEARCH_QUERY_LENGTH = 2; - -export const SEARCH_TAGS_LIMIT = 100; - export const typeDefs = /* GraphQL */ ` """ Post tag @@ -59,6 +61,16 @@ export const typeDefs = /* GraphQL */ ` hits: [Tag]! } + """ + Personalized feed tag list returned for the signed-in user. + """ + type FeedTagsList { + """ + Tags to render as chips + """ + tags: [String!]! + } + extend type Query { """ Get all tags @@ -92,6 +104,13 @@ export const typeDefs = /* GraphQL */ ` """ onboardingTags: TagResults! + """ + Tag chips to render in the unified feed nav for the signed-in user. + Cached in user.flags.feedTagsList for 24 hours; backfilled with + onboardingRecommendTags when the feed service returns fewer than the requested limit. + """ + feedTagsList(limit: Int = 10): FeedTagsList! @auth + """ Get recommended tags based on current selected and shown tags """ @@ -192,6 +211,14 @@ export const resolvers: IResolvers = { hits: hits.map((hit: Keyword) => ({ name: hit.value })), }; }, + feedTagsList: ( + _, + args: z.input, + ctx: AuthContext, + ) => { + const { limit } = feedTagsListInputSchema.parse(args); + return getFeedTagsList({ con: ctx.con, userId: ctx.userId, limit }); + }, recommendedTags: async ( source, { tags, excludedTags }, diff --git a/src/types.ts b/src/types.ts index 98e48d4ec1..16c7902cae 100644 --- a/src/types.ts +++ b/src/types.ts @@ -198,6 +198,12 @@ export const maxFeedsPerUser = 20; export const maxBookmarksPerMutation = 10; +export const RECOMMENDED_TAGS_LIMIT = 5; + +export const MIN_SEARCH_QUERY_LENGTH = 2; + +export const SEARCH_TAGS_LIMIT = 100; + export enum BookmarkListCountLimit { Free = 0, Plus = 50,