diff --git a/services/libs/integrations/src/integrations/stackoverflow/api/getQuestions.ts b/services/libs/integrations/src/integrations/stackoverflow/api/getQuestions.ts index d17a4f5723..e476292530 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/api/getQuestions.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/api/getQuestions.ts @@ -48,7 +48,21 @@ async function getQuestions( } const platformSettings = ctx.platformSettings as StackOverflowPlatformSettings - const key = platformSettings.key + const key = platformSettings?.key + + ctx.log.info( + { + hasPlatformSettings: !!platformSettings, + hasKey: !!key, + keyPrefix: key ? key.substring(0, 5) : 'undefined', + }, + 'StackOverflow: platformSettings check (tags)', + ) + + if (!key) { + ctx.log.error('StackOverflow: No API key found in platformSettings!') + throw new Error('StackOverflow API key not configured') + } // Get access token from Nango let accessToken = null @@ -83,7 +97,22 @@ async function getQuestions( params, } + ctx.log.info( + { url: config.url, params: { ...config.params, key: '***' } }, + 'StackOverflow: Making API call (tags)', + ) + const response: StackOverflowQuestionsResponse = (await axios(config)).data + + ctx.log.info( + { + itemsCount: response.items?.length ?? 0, + hasMore: response.has_more, + quotaRemaining: response.quota_remaining, + }, + 'StackOverflow: API response received (tags)', + ) + const backoff = response.backoff if (backoff) { if (backoff <= 2) { diff --git a/services/libs/integrations/src/integrations/stackoverflow/api/getQuestionsByKeywords.ts b/services/libs/integrations/src/integrations/stackoverflow/api/getQuestionsByKeywords.ts index b3b9ecc72d..3f96e38788 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/api/getQuestionsByKeywords.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/api/getQuestionsByKeywords.ts @@ -59,7 +59,21 @@ async function getQuestions( } const platformSettings = ctx.platformSettings as StackOverflowPlatformSettings - const key = platformSettings.key + const key = platformSettings?.key + + ctx.log.info( + { + hasPlatformSettings: !!platformSettings, + hasKey: !!key, + keyPrefix: key ? key.substring(0, 5) : 'undefined', + }, + 'StackOverflow: platformSettings check', + ) + + if (!key) { + ctx.log.error('StackOverflow: No API key found in platformSettings!') + throw new Error('StackOverflow API key not configured') + } const params: Record = { page: input.page, @@ -83,7 +97,22 @@ async function getQuestions( params, } + ctx.log.info( + { url: config.url, params: { ...config.params, key: '***' } }, + 'StackOverflow: Making API call', + ) + const response: StackOverflowQuestionsResponse = (await axios(config)).data + + ctx.log.info( + { + itemsCount: response.items?.length ?? 0, + hasMore: response.has_more, + quotaRemaining: response.quota_remaining, + }, + 'StackOverflow: API response received', + ) + const backoff = response.backoff if (backoff) { if (backoff <= 2) { diff --git a/services/libs/integrations/src/integrations/stackoverflow/generateStreams.ts b/services/libs/integrations/src/integrations/stackoverflow/generateStreams.ts index 2f3cb59e23..82464d41c8 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/generateStreams.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/generateStreams.ts @@ -10,13 +10,27 @@ import { const handler: GenerateStreamsHandler = async (ctx) => { const settings = ctx.integration.settings as IStackOverflowIntegrationSettings + ctx.log.info( + { + integrationId: ctx.integration.id, + tagsCount: settings?.tags?.length ?? 0, + keywordsCount: settings?.keywords?.length ?? 0, + tags: settings?.tags, + keywords: settings?.keywords, + }, + 'StackOverflow generateStreams: starting', + ) + if (settings.keywords.length === 0 && settings.tags.length === 0) { await ctx.abortRunWithError('No keywords or tags configured!') return } + let streamsPublished = 0 + if (settings.tags.length > 0) { for (const tag of settings.tags) { + ctx.log.info({ tag }, 'StackOverflow: publishing tag stream') await ctx.publishStream( `${StackOverflowRootStream.QUESTIONS_BY_TAG}:${tag}:${1}`, { @@ -24,10 +38,12 @@ const handler: GenerateStreamsHandler = async (ctx) => { page: 1, }, ) + streamsPublished++ } } if (settings.keywords.length > 0) { for (const keyword of settings.keywords) { + ctx.log.info({ keyword }, 'StackOverflow: publishing keyword stream') await ctx.publishStream( `${StackOverflowRootStream.QUESTIONS_BY_KEYWORD}:${keyword}:${1}`, { @@ -35,8 +51,14 @@ const handler: GenerateStreamsHandler = async (ctx) => { page: 1, }, ) + streamsPublished++ } } + + ctx.log.info( + { streamsPublished, integrationId: ctx.integration.id }, + 'StackOverflow generateStreams: completed', + ) } export default handler diff --git a/services/libs/integrations/src/integrations/stackoverflow/index.ts b/services/libs/integrations/src/integrations/stackoverflow/index.ts index d96cdca1ac..7851669a8f 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/index.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/index.ts @@ -10,7 +10,7 @@ import processStream from './processStream' const descriptor: IIntegrationDescriptor = { type: PlatformType.STACKOVERFLOW, memberAttributes: STACKOVERFLOW_MEMBER_ATTRIBUTES, - checkEvery: 360, // 6 hours + checkEvery: 60, // 1 hour generateStreams, processStream, processData, diff --git a/services/libs/integrations/src/integrations/stackoverflow/processStream.ts b/services/libs/integrations/src/integrations/stackoverflow/processStream.ts index b2cdb1bebf..b15017e090 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/processStream.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/processStream.ts @@ -42,6 +42,11 @@ const processTagStream: ProcessStreamHandler = async (ctx) => { const questions = response.items + ctx.log.info( + { tags, page, questionsCount: questions.length, hasMore: response.has_more }, + 'StackOverflow: tag stream API response', + ) + if (questions.length === 0) { return } @@ -110,6 +115,11 @@ const processKeywordStream: ProcessStreamHandler = async (ctx) => { const questions = response.items + ctx.log.info( + { keyword, page, questionsCount: questions.length, hasMore: response.has_more }, + 'StackOverflow: keyword stream API response', + ) + if (questions.length === 0) { return } @@ -179,6 +189,11 @@ const processAnswerStream: ProcessStreamHandler = async (ctx) => { const answers = response.items + ctx.log.info( + { questionId, page, answersCount: answers.length, hasMore: response.has_more, tag, keyword }, + 'StackOverflow: answer stream API response', + ) + if (answers.length === 0) { return } diff --git a/services/libs/integrations/src/integrations/stackoverflow/types.ts b/services/libs/integrations/src/integrations/stackoverflow/types.ts index fb752fa5be..0e9f3c5548 100644 --- a/services/libs/integrations/src/integrations/stackoverflow/types.ts +++ b/services/libs/integrations/src/integrations/stackoverflow/types.ts @@ -1,4 +1,4 @@ -export const STACKOVERFLOW_MAX_RETROSPECT_IN_HOURS = 7 +export const STACKOVERFLOW_MAX_RETROSPECT_IN_HOURS = 5 export const STACKOVERFLOW_LAST_MAX_PAGES = 20 export enum StackOverflowActivityType {