"""
regex_arxiv.py
author: Matt Bierbaum
date: 2019-03-14
RegEx patterns for finding arXiv id citations in fulltext articles.
"""
import re
# These are all the primary categories present in the OAI ArXiv metadata
CATEGORIES = [
"acc-phys", "adap-org", "alg-geom", "ao-sci", "astro-ph", "atom-ph",
"bayes-an", "chao-dyn", "chem-ph", "cmp-lg", "comp-gas", "cond-mat", "cs",
"dg-ga", "funct-an", "gr-qc", "hep-ex", "hep-lat", "hep-ph", "hep-th",
"math", "math-ph", "mtrl-th", "nlin", "nucl-ex", "nucl-th", "patt-sol",
"physics", "plasm-ph", "q-alg", "q-bio", "quant-ph", "solv-int",
"supr-con", "eess", "econ", "q-fin", "stat"
]
# All subcategories with more than 2 capital letters (not SG, SI, SP, etc)
SUB_CATEGORIES = [
'acc-ph', 'ao-ph', 'app-ph', 'atm-clus', 'atom-ph', 'bio-ph', 'chem-ph',
'class-ph', 'comp-ph', 'data-an', 'dis-nn', 'ed-ph', 'flu-dyn', 'gen-ph',
'geo-ph', 'hist-ph', 'ins-det', 'med-ph', 'mes-hall', 'mtrl-sci', 'optics',
'other', 'plasm-ph', 'pop-ph', 'quant-gas', 'soc-ph', 'soft', 'space-ph',
'stat-mech', 'str-el', 'supr-con'
]
__all__ = (
'REGEX_ARXIV_SIMPLE',
'REGEX_ARXIV_STRICT',
'REGEX_ARXIV_FLEXIBLE'
)
dashdict = {c.replace('-', ''): c for c in CATEGORIES if '-' in c}
dashdict.update({c.replace('-', ''): c for c in SUB_CATEGORIES if '-' in c})
REGEX_VERSION_SPLITTER = re.compile(r'([vV][1-9]\d*)')
def strip_version(name):
""" 1501.21981v1 -> 1501.21981 """
return REGEX_VERSION_SPLITTER.split(name)[0]
def format_cat(name):
""" Strip subcategory, add hyphen to category name if missing """
if '/' in name: # OLD ID, names contains subcategory
catsubcat, aid = name.split('/')
cat = catsubcat.split('.')[0]
return dashdict.get(cat, cat) + "/" + aid
else:
return name
def zeropad_1501(name):
""" Arxiv IDs after yymm=1501 are padded to 5 zeros """
if not '/' in name: # new ID
yymm, num = name.split('.')
if int(yymm) > 1500 and len(num) < 5:
return yymm + ".0" + num
return name
def clean(name):
""" Correct common errors in ArXiv IDs to improve matching """
funcs = [strip_version, format_cat, zeropad_1501]
for func in funcs:
name = func(name)
return name
# A common typo is to exclude the hyphen in the category.
categories = list(set(CATEGORIES + [cat.replace('-', '') for cat in
CATEGORIES]))
subcategories = list(set(SUB_CATEGORIES + [cat.replace('-', '') for cat in
SUB_CATEGORIES]))
# capture possible minor categories
RE_CATEGORIES = r'(?:{})(?:(?:[.][A-Z]{{2}})|(?:{}))?'.format(
r'|'.join(categories), r'|'.join(subcategories)
)
# valid YYMM date, NOT preceded by any digits
# NOTE: at the date of writing, it is 2019, so we do not allow
# proper dates for YY 20 or larger
RE_DATE = r'(?:(?:[0-1][0-9])|(?:9[1-9]))(?:0[1-9]|1[0-2])'
RE_VERSION = r'(?:[vV][1-9]\d*)?'
# =============================================================================
RE_NUM_NEW = RE_DATE + r'(?:[.]\d{4,5})' + RE_VERSION
RE_NUM_OLD = RE_DATE + r'(?:\d{3})' + RE_VERSION
# matches: 1612.00001 1203.0023v2
RE_ID_NEW = r'(?:{})'.format(RE_NUM_NEW)
# matches: hep-th/11030234 cs/0112345v2 cs.AI/0112345v2
RE_ID_OLD = r'(?:{}/{})'.format(RE_CATEGORIES, RE_NUM_OLD)
# =============================================================================
# matches: https://arxiv.org/abs/ abs/ arxiv.org/abs/
# 3. e-print: eprints
RE_PREFIX_URL = (
r'(?:'
r'(?i:http[s]?\://)?' # we could have a url prefix
r'(?i:arxiv\.org/)?' # maybe with the arxiv.org bit
r'(?i:abs/|pdf/)' # at least it has the abs/ part
r')'
)
# matches: arXiv: arxiv/ arxiv
RE_PREFIX_ARXIV = r'(?i:arxiv\s*[:/\s,.]*\s*)'
# matches: cs.AI/ cs.AI nucl-th
RE_PREFIX_CATEGORIES = r'(?i:{})'.format(RE_CATEGORIES)
# matches: e-prints: e-print eprints:
RE_PREFIX_EPRINT = r'(?i:e[-]?print[s]?.{1,3})'
# =============================================================================
# matches simple old or new identifiers, no fancy business
REGEX_ARXIV_SIMPLE = r'(?:{}|{})'.format(RE_ID_OLD, RE_ID_NEW)
# this one follows the guide set forth by:
# https://arxiv.org/help/arxiv_identifier
REGEX_ARXIV_STRICT = (
r'(?:{})'.format(RE_PREFIX_ARXIV) +
r'(?:'
r'({})'.format(RE_ID_OLD) +
r'|'
r'({})'.format(RE_ID_NEW) +
r')'
)
# this regex essentially accepts anything that looks like an arxiv id and has
# the slightest smell of being one as well. that is, if it is an id and
# mentions anything about the arxiv before hand, then it is an id.
REGEX_ARXIV_FLEXIBLE = (
r'(?:'
r'({})'.format(REGEX_ARXIV_SIMPLE) + # capture
r')|(?:'
r'(?:'
r'(?:{})?'.format(RE_PREFIX_URL) +
r'(?:{})?'.format(RE_PREFIX_EPRINT) +
r'(?:'
r'(?:{})?'.format(RE_PREFIX_ARXIV) +
r'({})'.format(RE_ID_OLD) + # capture
r'|'
r'(?:{})'.format(RE_PREFIX_ARXIV) +
r'(?:{}/)?'.format(RE_CATEGORIES) +
r'({})'.format(RE_ID_NEW) + # capture
r')'
r')'
r'|'
r'(?:'
r'(?:{})|'.format(RE_PREFIX_URL) +
r'(?:{})|'.format(RE_PREFIX_EPRINT) +
r'(?:{})|'.format(RE_PREFIX_CATEGORIES) +
r'(?:{})'.format(RE_PREFIX_ARXIV) +
r')'
r'.*?'
r'({})'.format(REGEX_ARXIV_SIMPLE) + # capture
r')|(?:'
r'(?:[\[\(]\s*)'
r'({})'.format(REGEX_ARXIV_SIMPLE) + # capture
r'(?:\s*[\]\)])'
r')'
)
TEST_POSITIVE = [
'arXiv:quant-ph 1503.01017v3',
'math. RT/0903.2992',
'arXiv, 1511.03262',
'tions. arXiv preprint arXiv:1607.00021, 2016',
'Math. Phys. 255, 577 (2005), hep-th/0306165',
'Kuzovlev, arXiv:cond-mat/9903350 ',
'arXiv:math.RT/1206.5933,',
'arXiv e-prints 1306.1595',
'ays, JHEP 07 (2009) 055, [ 0903.0883]',
' Rev. D71 (2005) 063534, [ astro-ph/0501562]',
'e-print arXiv:1506.02215v1',
'available at: http://arxiv.org/abs/1511.08977',
'arXiv e-print: 1306.2144',
'Preprint arXiv:math/0612139',
'Vertices in a Digraph. arXiv preprint 1602.02129 ',
'cond-mat/0309488.'
'decays, 1701.01871 LHCB-PAPE',
'Distribution. In: 1404.2485v3 (2015)',
'113005 (2013), 1307.4331,',
'scalar quantum 1610.07877v1',
'cond-mat/0309488.'
'cond-mat/0309488.8383'
]
TEST_NEGATIVE = [
'doi: 10.1145/ 321105.321114 ',
'doi: 10.1145/ 1105.321114 ',
]