buildersOpenAlex.py

import pandas as pd
import traceback

def buildPaperMetadata(data, debug_mode = False):
    """
    Extracts metadata from a JSON (transformed in dict) data object containing paper information and constructs a DataFrame.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - dfMeta (DataFrame): A pandas DataFrame containing the extracted metadata.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts various metadata attributes such as paper ID, DOI, publication date, journal ID,
    number of citations, language, fees (in USD), and whether the paper is open access.

    If a metadata attribute is missing for a paper, the corresponding field in the DataFrame will be filled with an empty string ('').

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'doi': '10.1234/example', ...}, {...}, ...]}
    >>> metadata_df = buildPaperMetadata(data)
    """
    pId=[]
    pDate=[]
    pNCit=[]
    pJId=[]
    pDoi=[]
    pLang=[]
    pFees=[]
    pIsOA=[]

    for paper in data['results']:
        pId.append(paper['id'].replace('https://openalex.org/',''))

        try:
            pDoi.append(paper['doi'])
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pDoi.append('')

        try:
            pDate.append(paper['publication_date'])
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pDate.append('')

        try:
            pNCit.append(paper['cited_by_count'])
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pNCit.append('')

        try:
            pJId.append(paper['primary_location']['source']['id'].replace('https://openalex.org/',''))
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pJId.append('')

        try:
            pLang.append(paper['language'])
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pLang.append('')
            
        try:
            pFees.append(paper['apc_paid']["value_usd"])
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pFees.append('')
            
        try:
            pIsOA.append(paper['open_access']["is_oa"])
        except Exception as e:
            print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pIsOA.append('')
            
    dfMeta=pd.DataFrame()
    dfMeta['paper_id']=pId
    dfMeta['paper_doi']=pDoi
    dfMeta['pulication_date']=pDate
    dfMeta['journal_id']=pJId
    dfMeta['n_cit']=pNCit
    dfMeta['language']=pLang
    dfMeta['fees_usd']=pFees
    dfMeta['is_oa']=pIsOA
    
    return dfMeta


def rebuild_abstract(word_dict, debug_mode = False):
    """
    Rebuilds abstract text from an inverted index of words to positions, using the 'abstract_inverted_index' key per paper.

    Parameters:
    - word_dict (dict): A dictionary containing word positions as keys and words as values.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - abstract (str): The reconstructed abstract text.

    The function iterates through the keys (word positions) in the input dictionary and reconstructs
    the abstract text by placing the corresponding word at its position in the abstract. The reconstructed abstract is
    then returned as a single string.

    Example usage:
    >>> word_dict = {'This':[0], 'is':[1], 'an':[2], 'abstract'[3]}
    >>> abstract = rebuild_abstract(word_dict)
    >>> print(abstract)
    This is an abstract
    """
    
    # find max index in inverted abstract (word_dict)
    max_index = 0
    for word, list_indexes in word_dict.items():
        max_index = max([max_index]+list_indexes)
    # create a list of elements long as the number of total words used
    abstract = [0]*(max_index+1)
    for i in word_dict.keys():
        for j in word_dict[i]:
            abstract[j] = i
    abstract = ' '.join(abstract)
    return abstract

def buildTitleAbstractDataframe(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper title and abstract information from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - p2Ab (DataFrame): A pandas DataFrame containing paper titles and abstracts.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts the paper ID, title, and abstract information. If any attribute is missing,
    it fills the corresponding field in the DataFrame with an empty string ('').

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'title': 'Example Title', ...}, {...}, ...]}
    >>> title_abstract_df = buildTitleAbstractDataframe(data)
    """
    pTitle=[]
    p_abstr=[]
    pId=[]
    for paper in data['results']:
        pId.append(paper['id'].replace('https://openalex.org/',''))
        try:
            pTitle.append(paper['title'].replace(';','.'))
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            pTitle.append('')

        try:
            # every ";" is substituted with a "." so that ";" can be used as a separator in the DataFrame.
            p_abstr.append(rebuild_abstract(paper['abstract_inverted_index'], debug_mode = debug_mode).replace(';','.'))
        except Exception as e:
            if debug_mode:
                print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            p_abstr.append('')
        
    p2Ab=pd.DataFrame()
    p2Ab['paper_id']=pId
    p2Ab['title']=pTitle
    p2Ab['abstract']=p_abstr
    return p2Ab

def buildPaper2Author(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper-to-author relationships from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - p2Au (DataFrame): A pandas DataFrame containing paper-to-author relationships.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts the paper ID and author ID information and constructs a DataFrame representing
    the relationship between papers and their authors.
    For each paper, if multiple authors are present, then multiple lines are created for that paper, one for each author.

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'authorships': [{'author': {...}}, ...]}, {...}, ...]}
    >>> paper_author_df = buildPaper2Author(data)
    """
    pp=[]
    aa=[]
    for paper in data['results']:
        try:
            authors=paper['authorships']
            for au in authors:
                try:
                    aa.append(au['author']['id'].replace('https://openalex.org/',''))
                    pp.append(paper['id'].replace('https://openalex.org/',''))

                except Exception as e:
                    if debug_mode:
                        print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                    continue
        except Exception as e:
            print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
            continue
            
    p2Au=pd.DataFrame()
    p2Au['paper_id']=pp
    p2Au['author_id']=aa
    return p2Au

def buildAuthorInformation(data, debug_mode = False):
    """
    Constructs a DataFrame containing author information from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - auMD (DataFrame): A pandas DataFrame containing author information.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts author-related attributes such as ID, name, ORCID, institution ID, institution name,
    and country. 
    For each paper, if multiple authors are present, then multiple lines are created for that paper, one for each author.

    Example usage:
    >>> data = {'results': [{'authorships': [{'author': {'id': '123', ...}}, ...]}, ...]}
    >>> author_info_df = buildAuthorInformation(data)
    """
    authorId=[]
    authorName=[]
    authorOrcid=[]
    authorInstitutionId=[]
    authorInstitutionName=[]
    authorInstitutionCountry=[]

    for paper in data['results']:
        authors=paper['authorships']
        for au in authors:
            try:
                authorId.append(au['author']['id'].replace('https://openalex.org/',''))
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                authorId.append('')

            try:
                authorName.append(au['author']['display_name'])
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                authorName.append('')

            try:
                authorOrcid.append(au['author']['orcid'])
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                authorOrcid.append('')
            
            tmp_authorInstitutionId = ""
            tmp_authorInstitutionName = ""
            tmp_authorInstitutionCountry = ""
            for institution in au['institutions']:
                try:
                    tmp_authorInstitutionId += institution['id'].replace('https://openalex.org/','') + "_"
                except Exception as e:
                    if debug_mode:
                        print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                    tmp_authorInstitutionId += '' + "_"

                try:
                    tmp_authorInstitutionName += institution['display_name'] + "_"
                except Exception as e:
                    if debug_mode:
                        print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                    tmp_authorInstitutionName += '' + "_"

                try:
                    tmp_authorInstitutionCountry += institution['country_code'] + "_"
                except Exception as e:
                    if debug_mode:
                        print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                    tmp_authorInstitutionId += '' + "_"
            # remove last /
            if len(au['institutions']) > 0:
                tmp_authorInstitutionId = tmp_authorInstitutionId[:-1]
                tmp_authorInstitutionName = tmp_authorInstitutionName[:-1]
                tmp_authorInstitutionCountry = tmp_authorInstitutionCountry[:-1]
                
            authorInstitutionId.append(tmp_authorInstitutionId)
            authorInstitutionName.append(tmp_authorInstitutionName)
            authorInstitutionCountry.append(tmp_authorInstitutionCountry)
    
    auMD=pd.DataFrame()
    auMD['author_id']=authorId
    auMD['author_name']=authorName
    auMD['author_orcid']=authorOrcid
    auMD['institution_id']=authorInstitutionId
    auMD['institution_name']=authorInstitutionName
    auMD['country']=authorInstitutionCountry
    return auMD

def buildPaperConcepts(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper concepts from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - concDf (DataFrame): A pandas DataFrame containing paper concepts.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts concept-related attributes such as paper ID, concept ID, and concept score. 
    For each paper, if multiple concepts are present, then multiple lines are created for that paper, one for each concept.

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'concepts': [{'id': '123', ...}}, ...], ...}
    >>> paper_concepts_df = buildPaperConcepts(data)
    """
    pList=[]
    pConcepts=[]
    pScore=[]
    for paper in data['results']:
        try:
            klist=[(paper['id'].replace('https://openalex.org/',''),i['id'].replace('https://openalex.org/',''),i['score']) for i in paper['concepts']]
        except KeyError as e: # some recent papers do not yet have topics field in the json
            klist=[(paper['id'].replace('https://openalex.org/',''),"","")]
        for (u,v,w) in klist:
            pList.append(u)
            pConcepts.append(v)
            pScore.append(w)
    concDf=pd.DataFrame()
    concDf['paper_id']=pList
    concDf['concept_id']=pConcepts
    concDf['concept_score']=pScore
    return concDf


def buildPaperKeywords(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper keywords from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - keywordDf (DataFrame): A pandas DataFrame containing paper keywords.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts keyword-related attributes such as paper ID, keyword, and keyword score. 
    For each paper, if multiple keywords are present, then multiple lines are created for that paper, one for each keyword.

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'keywords': [{'keyword': 'machine learning', 'score': 0.8}, ...]}, ...]}
    >>> paper_keywords_df = buildPaperKeywords(data)
    """
    pList=[]
    pKeyword=[]
    pScore=[]
    for paper in data['results']:
        try:
            klist=[(paper['id'].replace('https://openalex.org/',''),i['display_name'],i['score']) for i in paper['keywords']]
        except KeyError as e: # some recent papers do not yet have topics field in the json
            klist=[(paper['id'].replace('https://openalex.org/',''),"","")]
        for (u,v,w) in klist:
            pList.append(u)
            pKeyword.append(v)
            pScore.append(w)
    keywordDf=pd.DataFrame()
    keywordDf['paper_id']=pList
    keywordDf['keyword']=pKeyword
    keywordDf['keyword_score']=pScore
    return keywordDf


def buildPaperTopics(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper topics from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - topicDf (DataFrame): A pandas DataFrame containing paper topics.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts topic-related attributes such as paper ID, topic ID, and topic score. 
    For each paper, if multiple topics are present, then multiple lines are created for that paper, one for each topic.
    For each paper, there should always be three topics

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'topics': [{'id': '123', 'score': 0.8}, ...]}, ...]}
    >>> paper_topics_df = buildPaperTopics(data)
    """
    pList=[]
    pTopic=[]
    pScore=[]
    for paper in data['results']:
        try:
            klist=[(paper['id'].replace('https://openalex.org/',''),i['id'].replace('https://openalex.org/',''),i['score']) for i in paper['topics']]
        except KeyError as e: # some recent papers do not yet have topics field in the json
            klist=[(paper['id'].replace('https://openalex.org/',''),"","")]
        for _ in klist:
            for i,l in enumerate([pList,pTopic,pScore]):
                l.append(_[i])
    topicDf=pd.DataFrame()
    topicDf['paper_id']=pList
    topicDf['topic_id']=pTopic
    topicDf['topic_score']=pScore
    return topicDf


def buildPaperReferences(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper references from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.

    Returns:
    - referenceDf (DataFrame): A pandas DataFrame containing paper references.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts reference-related attributes such as paper ID and referenced work IDs. 
    For each paper, if multiple references are present, then multiple lines are created for that paper, one for each reference.

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'referenced_works': ['https://openalex.org/5678', ...]}, ...]}
    >>> paper_references_df = buildPaperReferences(data)
    """
    pList=[]
    pReferences=[]
    for paper in data['results']:
        pList.append(paper['id'].replace('https://openalex.org/',''))
        tmp_references = ""
        for reference_url in paper['referenced_works']:
            try:
                tmp_references += reference_url.replace('https://openalex.org/','') + "_"
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                tmp_references += '' + "_"
        # remove last /
        if len(paper['referenced_works']) > 0:
            tmp_references = tmp_references[:-1]
        
        pReferences.append(tmp_references)
    referenceDf=pd.DataFrame()
    referenceDf['paper_id']=pList
    referenceDf['reference_ids']=pReferences
    return referenceDf

def buildPaperGrants(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper grants from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - grantDf (DataFrame): A pandas DataFrame containing paper grants.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts grant-related attributes such as paper ID, funder ID, funder name, and award ID. 
    For each paper, if multiple grants are present, then multiple lines are created for that paper, one for each grant.

    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'grants': [{'funder': 'https://openalex.org/funder1', 'funder_display_name': 'Funder 1', 'pAwardID': '12345'}, ...]}, ...]}
    >>> paper_grants_df = buildPaperGrants(data)
    """
    pList=[]
    pFunder=[]
    pFname=[]
    pAwardID=[]
    for paper in data['results']:
        for grant in paper['grants']:
            try:
                pList.append(paper['id'].replace('https://openalex.org/',''))
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                pList.append('')
                
            try:
                pFunder.append(grant['funder'].replace('https://openalex.org/',''))
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                pFunder.append('')
                
            try:
                pFname.append(grant['funder_display_name'])
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                pFname.append('')
                
            try:
                pAwardID.append(grant['award_id'])
            except Exception as e:
                if debug_mode:
                    print(paper['id'].replace('https://openalex.org/',''), e, "\n\n", traceback.format_exc(), flush=True)
                pAwardID.append('')
        
    grantDf=pd.DataFrame()
    grantDf['paper_id']=pList
    grantDf['funder_id']=pFunder
    grantDf['funder_name']=pFname
    grantDf['award_id']=pAwardID
    return grantDf

def buildPaperSustainableDevelopmentGoals(data, debug_mode = False):
    """
    Constructs a DataFrame containing paper sustainable development goals (SDGs) from a JSON data object.

    Parameters:
    - data (dict): A dictionary containing paper information.
    - debug_mode (bool): if True it prints errors caught with try and except.

    Returns:
    - sdgDf (DataFrame): A pandas DataFrame containing paper SDGs.

    The function iterates through each paper in the 'results' list of the input data dictionary.
    It extracts SDG-related attributes such as paper ID, SDG ID, and SDG score. 
    For each paper, if multiple SDGs are present, then multiple lines are created for that paper, one for each SDG.
    
    Example usage:
    >>> data = {'results': [{'id': 'https://openalex.org/1234', 'sustainable_development_goals': [{'id': '1', 'score': 0.8}, ...]}, ...]}
    >>> paper_sdgs_df = buildPaperSustainableDevelopmentGoals(data)
    """
    pList=[]
    pSdgIdList=[]
    pSdgIdScore=[]
    for paper in data['results']:
        try:
            klist=[(paper['id'].replace('https://openalex.org/',''),i['id'].replace('https://metadata.un.org/sdg/',''),i['score']) for i in paper['sustainable_development_goals']]
        except KeyError as e: # some recent papers do not yet have topics field in the json
            klist = [(paper['id'].replace('https://openalex.org/',''),"","")]
        for _ in klist:
            for i,l in enumerate([pList,pSdgIdList,pSdgIdScore]):
                l.append(_[i])
    
    sdgDf=pd.DataFrame()
    sdgDf['paper_id']=pList
    sdgDf['sdg_id']=pSdgIdList
    sdgDf['sdg_score']=pSdgIdScore
    
    return sdgDf