File size: 4,443 Bytes
4596869
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
import pandas as pd
import arxiv
import requests
from pinecone import Pinecone, ServerlessSpec
import logging
import os

script_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(script_dir) 

def get_zotero_ids(api_key, library_id, tag):

    base_url = 'https://api.zotero.org'
    suffix = '/users/'+ library_id +'/items?tag='+ tag

    header = {'Authorization': 'Bearer '+ api_key}
    request = requests.get(base_url + suffix, headers= header)
    
    return [data['data']['archiveID'].replace('arXiv:', '') for data in request.json()]

def get_arxiv_papers(ids = None, category = None, comment = None):

    logging.getLogger('arxiv').setLevel(logging.WARNING)

    client = arxiv.Client()

    if category is None:
        search = arxiv.Search(
            id_list= ids,
            max_results= len(ids),
        )
    else :
        if comment is None:
            custom_query = f'cat:{category}'
        else:
            custom_query = f'cat:{category} AND co:{comment}'

        search = arxiv.Search(
            query = custom_query,
            max_results= 15,
            sort_by= arxiv.SortCriterion.SubmittedDate
        )
    if ids is None and category is None:
        raise ValueError('not a valid query')

    df = pd.DataFrame({'Title': [result.title for result in client.results(search)],
                'Abstract': [result.summary.replace('\n', ' ') for result in client.results(search)],
                'Date': [result.published.date().strftime('%Y-%m-%d') for result in client.results(search)],
                'id': [result.entry_id for result in client.results(search)]})
    
    if ids:
        df.to_csv('arxiv-scrape.csv', index = False)
    return df

def get_hf_embeddings(api_key, df):

    title_abs = [title + '[SEP]' + abstract for title,abstract in zip(df['Title'], df['Abstract'])]

    API_URL = "https://api-inference.huggingface.co/models/malteos/scincl"
    headers = {"Authorization": f"Bearer {api_key}"}

    response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": False})
    print(str(response.status_code) + 'This part needs an update, causing KeyError 0')
    if response.status_code == 503:
        response = requests.post(API_URL, headers=headers, json={"inputs": title_abs, "wait_for_model": True})

    embeddings = response.json()

    return embeddings, len(embeddings[0])


def upload_to_pinecone(api_key, index, namespace, embeddings, dim, df):
    input = [{'id': df['id'][i], 'values': embeddings[i]} for i in range(len(embeddings))]

    pc = Pinecone(api_key = api_key)
    if index in pc.list_indexes().names():
        while True:
            logging.warning(f'Index name : {index} already exists.')
            return f'Index name : {index} already exists'
                
    pc.create_index(
        name=index,
        dimension=dim,
        metric="cosine",
        spec=ServerlessSpec(
            cloud='aws', 
            region='us-east-1'
        ) 
    ) 

    index = pc.Index(index)
    return index.upsert(vectors=input, namespace=namespace)


def get_new_papers(df):
    df_main = pd.read_csv('arxiv-scrape.csv')
    df.reset_index(inplace=True)
    df.drop(columns=['index'], inplace=True)
    union_df = df.merge(df_main, how='left', indicator=True)
    df = union_df[union_df['_merge'] == 'left_only'].drop(columns=['_merge'])
    if df.empty:
        return 'No New Papers Found'
    else:
        df_main = pd.concat([df_main, df], ignore_index= True)
        df_main.drop_duplicates(inplace= True)
        df_main.to_csv('arxiv-scrape.csv', index = False)
        return df

def recommend_papers(api_key, index, namespace, embeddings, df, threshold):

    pc = Pinecone(api_key = api_key)
    if index in pc.list_indexes().names():
        index = pc.Index(index)
    else:
        raise ValueError(f"{index} doesnt exist. Project isnt initialized properly")
    
    results = []
    score_threshold = threshold
    for i,embedding in enumerate(embeddings):
        query = embedding
        result = index.query(namespace=namespace,vector=query,top_k=3,include_values=False)
        sum_score = sum(match['score'] for match in result['matches'])
        if sum_score > score_threshold:
            results.append(f"Paper-URL : [{df['id'][i]}]({df['id'][i]}) with score: {sum_score / 3} <br />")

    if results:
        return '\n'.join(results)
    else:
        return 'No Interesting Paper'