feat(worker): Add sync all option for bitbucket server (#927)

* Update bitbucket json schema

* Add generated schema files

* Implement getting all repos

* Add documentation

* Add changelog

---------

Co-authored-by: Christian Svedin <christian.svedin@idainfront.se>
Co-authored-by: Brendan Kellam <brendan@sourcebot.dev>

Author: 3 people

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `wa_user_created` PostHog event fired on successful user sign-up. [#933](https://github.com/sourcebot-dev/sourcebot/pull/933)
 - Added `wa_askgh_login_wall_prompted` PostHog event fired when an unauthenticated user attempts to ask a question on Ask GitHub. [#933](https://github.com/sourcebot-dev/sourcebot/pull/933)
 - Added Bitbucket Server (Data Center) OAuth 2.0 SSO identity provider support (`provider: "bitbucket-server"`). [#934](https://github.com/sourcebot-dev/sourcebot/pull/934)
+- Added Bitbucket Server (Data Center) sync all repositories support. [#927](https://github.com/sourcebot-dev/sourcebot/pull/927)
 
 ### Changed
 - Hide version upgrade toast for askgithub deployment (`EXPERIMENT_ASK_GH_ENABLED`). [#931](https://github.com/sourcebot-dev/sourcebot/pull/931)

docs/docs/connections/bitbucket-data-center.mdx

@@ -39,6 +39,18 @@ If you're not familiar with Sourcebot [connections](/docs/connections/overview),
         }
         ```
     </Accordion>
+    <Accordion title="Sync all repos">
+        <Note>Requires a `token` to be set in order to access private repositories.</Note>
+        ```json
+        {
+            "type": "bitbucket",
+            "deploymentType": "server",
+            "url": "https://mybitbucketdeployment.com",
+            // Index all repos visible to the provided token
+            "all": true
+        }
+        ```
+    </Accordion>
     <Accordion title="Exclude repos from syncing">
         ```json
         {

docs/snippets/schemas/v3/bitbucket.schema.mdx

@@ -67,6 +67,11 @@
       "default": "cloud",
       "description": "The type of Bitbucket deployment"
     },
+    "all": {
+      "type": "boolean",
+      "default": false,
+      "description": "Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`."
+    },
     "workspaces": {
       "type": "array",
       "items": {

docs/snippets/schemas/v3/connection.schema.mdx

@@ -744,6 +744,11 @@
           "default": "cloud",
           "description": "The type of Bitbucket deployment"
         },
+        "all": {
+          "type": "boolean",
+          "default": false,
+          "description": "Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`."
+        },
         "workspaces": {
           "type": "array",
           "items": {

docs/snippets/schemas/v3/index.schema.mdx

@@ -1159,6 +1159,11 @@
                   "default": "cloud",
                   "description": "The type of Bitbucket deployment"
                 },
+                "all": {
+                  "type": "boolean",
+                  "default": false,
+                  "description": "Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`."
+                },
                 "workspaces": {
                   "type": "array",
                   "items": {

packages/backend/src/bitbucket.ts

@@ -76,6 +76,18 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon
     let allRepos: BitbucketRepository[] = [];
     let allWarnings: string[] = [];
 
+    if (config.all === true) {
+        if (client.deploymentType === BITBUCKET_SERVER) {
+            const { repos, warnings } = await serverGetAllRepos(client);
+            allRepos = allRepos.concat(repos);
+            allWarnings = allWarnings.concat(warnings);
+        } else {
+            const warning = `Ignoring option all:true in config: not supported for Bitbucket Cloud`;
+            logger.warn(warning);
+            allWarnings = allWarnings.concat(warning);
+        }
+    }
+
     if (config.workspaces) {
         const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces);
         allRepos = allRepos.concat(repos);
@@ -554,6 +566,26 @@ async function serverGetRepos(client: BitbucketClient, repoList: string[]): Prom
     };
 }
 
+async function serverGetAllRepos(client: BitbucketClient): Promise<{repos: ServerRepository[], warnings: string[]}> {
+    logger.debug(`Fetching all repos from Bitbucket Server...`);
+    const path = `/rest/api/1.0/repos` as ServerGetRequestPath;
+    const { durationMs, data } = await measure(async () => {
+        const fetchFn = () => getPaginatedServer<ServerRepository>(path, async (url, start) => {
+            const response = await client.apiClient.GET(url, {
+                params: { query: { start } }
+            });
+            const { data, error } = response;
+            if (error) {
+                throw new Error(`Failed to fetch all repos: ${JSON.stringify(error)}`);
+            }
+            return data;
+        });
+        return fetchWithRetry(fetchFn, `all repos`, logger);
+    });
+    logger.debug(`Found ${data.length} total repos in ${durationMs}ms.`);
+    return { repos: data, warnings: [] };
+}
+
 export function serverShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean {
     const serverRepo = repo as ServerRepository;
 

packages/schemas/src/v3/bitbucket.schema.ts

@@ -66,6 +66,11 @@ const schema = {
       "default": "cloud",
       "description": "The type of Bitbucket deployment"
     },
+    "all": {
+      "type": "boolean",
+      "default": false,
+      "description": "Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`."
+    },
     "workspaces": {
       "type": "array",
       "items": {

packages/schemas/src/v3/bitbucket.type.ts

@@ -37,6 +37,10 @@ export interface BitbucketConnectionConfig {
    * The type of Bitbucket deployment
    */
   deploymentType?: "cloud" | "server";
+  /**
+   * Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`.
+   */
+  all?: boolean;
   /**
    * List of workspaces to sync. Ignored if deploymentType is server.
    */

packages/schemas/src/v3/connection.schema.ts

@@ -743,6 +743,11 @@ const schema = {
           "default": "cloud",
           "description": "The type of Bitbucket deployment"
         },
+        "all": {
+          "type": "boolean",
+          "default": false,
+          "description": "Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`."
+        },
         "workspaces": {
           "type": "array",
           "items": {

packages/schemas/src/v3/connection.type.ts

@@ -288,6 +288,10 @@ export interface BitbucketConnectionConfig {
    * The type of Bitbucket deployment
    */
   deploymentType?: "cloud" | "server";
+  /**
+   * Sync all repositories visible to the provided `token` (if any) in the Bitbucket Server instance. This option is ignored if `deploymentType` is `cloud`.
+   */
+  all?: boolean;
   /**
    * List of workspaces to sync. Ignored if deploymentType is server.
    */