| import json | |
| import random | |
| from bs4 import BeautifulSoup | |
| import utils | |
| def fetch_new_page(category): | |
| url = f'https://arxiv.org/list/{category}/new' | |
| return utils.fetch_page(url) | |
| def fetch_recent_page(category): | |
| url = f'https://arxiv.org/list/{category}/recent' | |
| return utils.fetch_page(url) | |
| def extract_new_data(category): | |
| paper_ids = [] | |
| page_content = fetch_new_page(category) | |
| lists = BeautifulSoup(page_content, 'html.parser').find_all('dl') | |
| for list in lists: | |
| papers = list.find_all('dt') | |
| paper_contents = list.find_all('dd') | |
| titles = [paper_content.find('div', class_='list-title').text.strip().split('Title:')[-1].strip() for paper_content in paper_contents] | |
| for paper, title in zip(papers, titles): | |
| if not utils.verify_simple_title(title): | |
| continue | |
| paper_link = paper.find('a', href=True) | |
| if paper_link: | |
| paper_id = paper_link.text.strip().split(':')[1] | |
| paper_ids.append(paper_id) | |
| return paper_ids | |
| def extract_recent_data(category): | |
| paper_ids = [] | |
| page_content = fetch_recent_page(category) | |
| lists = BeautifulSoup(page_content, 'html.parser').find_all('dl') | |
| for list in lists: | |
| papers = list.find_all('dt') | |
| for paper in papers: | |
| paper_link = paper.find('a', href=True) | |
| if paper_link: | |
| paper_id = paper_link.text.strip().split(':')[1] | |
| paper_ids.append(paper_id) | |
| return paper_ids | |
| def extract_data(category): | |
| all_ids = set() | |
| new_data = extract_new_data(category) | |
| recent_data = extract_recent_data(category) | |
| combined_data = new_data + recent_data | |
| for paper_id in combined_data: | |
| if not utils.check_data_in_file(paper_id, 'arxiv.txt'): | |
| utils.write_data_to_file(paper_id, 'arxiv.txt') | |
| all_ids.add(paper_id) | |
| if len(all_ids) >= 2: | |
| break | |
| return list(all_ids) | |
| def extract_arxiv_data(): | |
| if not utils.download_datafile('arxiv.txt'): | |
| raise Exception("Failed to download datafile") | |
| categories = { | |
| "Astrophysics": ["astro-ph"], | |
| "Condensed Matter": ["cond-mat"], | |
| "General Relativity and Quantum Cosmology": ["gr-qc"], | |
| "High Energy Physics": ["hep-ex", "hep-lat", "hep-ph", "hep-th"], | |
| "Mathematical Physics": ["math-ph"], | |
| "Nonlinear Sciences": ["nlin"], | |
| "Nuclear Experiment": ["nucl-ex"], | |
| "Nuclear Theory": ["nucl-th"], | |
| "Physics": ["physics"], | |
| "Quantum Physics": ["quant-ph"], | |
| "Mathematics": ["math"], | |
| "Computer Science": ["cs"], | |
| "Quantitative Biology": ["q-bio"], | |
| "Quantitative Finance": ["q-fin"], | |
| "Statistics": ["stat"], | |
| "Electrical Engineering and Systems Science": ["eess"], | |
| "Economics": ["econ"] | |
| } | |
| data = {} | |
| used_ids = set() | |
| for category, subcategories in categories.items(): | |
| category_ids = set() | |
| for subcategory in subcategories: | |
| ids = extract_data(subcategory) | |
| for paper_id in ids: | |
| if paper_id not in used_ids: | |
| category_ids.add(paper_id) | |
| used_ids.add(paper_id) | |
| if len(category_ids) == 2: | |
| break | |
| if len(category_ids) == 2: | |
| break | |
| while len(category_ids) < 2: | |
| category_ids.add(random.choice(list(used_ids))) | |
| data[category] = {"ids": list(category_ids), "count": len(category_ids)} | |
| if not utils.upload_datafile('arxiv.txt'): | |
| raise Exception("Failed to upload datafile") | |
| return json.dumps(data, indent=4, ensure_ascii=False) | |
| if __name__ == '__main__': | |
| data = extract_arxiv_data() | |
| with open('arxiv_data.json', 'w') as f: | |
| f.write(data) | |