Git commits/PRs w/pagination

requirements.txt

@@ -4,6 +4,6 @@ pygsheets==2.0.6
 pandas==2.2.3
 pytz==2024.2
 requests==2.32.3
-pyforgejo==2.0.0
+pyforgejo==2.0.7
 isodate==0.7.2
 unittest-parametrize==1.6.0

src/ForgejoRepoAPI.py

@@ -67,9 +67,7 @@ def get_collaborator_permission(self, repo: Repository, user: User) -> str:
 
     @log_exceptions(default_return=[], message="Failed to get commits from Forgejo")
     def get_commits(self, repo: Repository, files: bool = True) -> list[Commit]:
-        commits = self.client.repository.repo_get_all_commits(
-            repo.owner.login, repo.name
-        )
+        commits = self.get_all_data_from_pages(self.client.repository.repo_get_all_commits, repo.owner.login, repo.name)
         return [
             Commit(
                 _id=c.sha,
@@ -121,9 +119,8 @@ def get_issues(self, repo: Repository) -> list[Issue]:
 
     @log_exceptions(default_return=[], message="Failed to get pull requests from Forgejo")
     def get_pull_requests(self, repo: Repository) -> list[PullRequest]:
-        pulls = self.client.repository.repo_list_pull_requests(
-            repo.owner.login, repo.name
-        )
+        pulls = self.get_all_data_from_pages(self.client.repository.repo_list_pull_requests,
+                                             repo.owner.login, repo.name)
         return [
             PullRequest(
                 _id=p.number,
@@ -171,6 +168,8 @@ def get_branches(self, repo: Repository) -> list[Branch]:
                 author=contributor,
                 date=commit.timestamp,
                 files=files,
+                additions=None,  # TODO
+                deletions=None,  # TODO
             )
 
             result.append(Branch(name=branch.name, last_commit=commit_obj))
@@ -269,6 +268,17 @@ def get_invites(self, repo: Repository, users: list[User] = None) -> list[Invite
                 )
         return invites
 
+    @log_exceptions(default_return=[], message="Failed to get_all_data_from_pages")
+    def get_all_data_from_pages(self, method, *method_args, **kw_method_args):
+        all_data = []
+        page_index = 1
+        data = method(*method_args, page=page_index, **kw_method_args)
+        while data:
+            page_index += 1
+            all_data += data
+            data = method(*method_args, page=page_index, **kw_method_args)
+        return all_data
+
     def get_rate_limiting(self) -> tuple[int, int]:
         return sys.maxsize, sys.maxsize
 

src/graphql/pull_request_parser.py

@@ -19,26 +19,27 @@ def log_repositories_pr_by_graphql(owner, repo_name, token, csv_name, first_n=10
     query GetPRData($owner: String!, $repo: String!, $first: Int!, $after: String) {
         repository(owner: $owner, name: $repo) {
             nameWithOwner
-            pullRequests(first: $first, after: $after, states: [OPEN, CLOSED, MERGED], orderBy: {field: CREATED_AT, direction: DESC}) {
+            pullRequests(first: $first, after: $after, states: [OPEN, CLOSED, MERGED],
+                            orderBy: {field: CREATED_AT, direction: DESC}) {
                 totalCount
-                pageInfo {		
-                    hasNextPage		
-                    endCursor		
+                pageInfo {
+                    hasNextPage
+                    endCursor
                 }
                 nodes {
                     title
                     number
                     state
                     createdAt
-                
+
                     author {
                         login
                         ... on User {
                         name
                         email
                         }
                     }
-                
+
                     baseRef {
                         name
                         target {
@@ -52,11 +53,11 @@ def log_repositories_pr_by_graphql(owner, repo_name, token, csv_name, first_n=10
                         oid
                         }
                     }
-                
+
                     changedFiles
                     additions
                     deletions
-                
+
                     mergedAt
                     mergedBy {
                         login
@@ -65,14 +66,14 @@ def log_repositories_pr_by_graphql(owner, repo_name, token, csv_name, first_n=10
                         email
                         }
                     }
-                    
+
                     assignees(first: 10) {
                         nodes {
                         login
                         name
                         }
                     }
-                    
+
                     labels(first: 20) {
                         nodes {
                         name

src/pull_requests_parser.py

@@ -1,5 +1,5 @@
 import json
-from dataclasses import asdict, dataclass
+from dataclasses import asdict
 from datetime import datetime
 from time import sleep
 from typing import Generator
@@ -11,7 +11,6 @@
 from src.git_logger import get_assignee_story
 from src.interface_wrapper import IRepositoryAPI, Repository
 from src.utils import logger
-from src.graphql.pull_request_parser import log_pull_requests_by_graphql    # for using in main.py
 from src.repo_dataclasses import PullRequestData
 
 

src/repo_dataclasses.py

@@ -27,4 +27,4 @@ class PullRequestData:
     assignee_story: str = ''
     related_issues: str = ''
     labels: str = ''
-    milestone: str = ''
+    milestone: str = ''