Archives
/
python-readability
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Convert tabs to spaces; put article in body

Browse Source
0.3.0.dev
Jerry Charumilind 13 years ago
parent 01247903b8
commit ac517834e6
1 changed files with 475 additions and 472 deletions
Show Stats Download Patch File Download Diff File

947
readability/readability.py
Unescape Escape View File

@ -4,6 +4,7 @@ from collections import defaultdict
from htmls import build_doc, get_body, get_title, shorten_title from htmls import build_doc, get_body, get_title, shorten_title
from lxml.etree import tostring, tounicode from lxml.etree import tostring, tounicode
from lxml.html import fragment_fromstring, document_fromstring from lxml.html import fragment_fromstring, document_fromstring
from lxml.html import builder as B
import logging import logging
import re import re
import sys import sys
@ -11,495 +12,497 @@ import sys
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
REGEXES = { REGEXES = {
'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I),
'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I),
'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I),
'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I),
'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I),
#'replaceBrsRe': re.compile('(<br[^>]*>[ \n\r\t]*){2,}',re.I), #'replaceBrsRe': re.compile('(<br[^>]*>[ \n\r\t]*){2,}',re.I),
#'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
#'trimRe': re.compile('^\s+|\s+$/'), #'trimRe': re.compile('^\s+|\s+$/'),
#'normalizeRe': re.compile('\s{2,}/'), #'normalizeRe': re.compile('\s{2,}/'),
#'killBreaksRe': re.compile('(<br\s*\/?>(\s|&nbsp;?)*){1,}/'), #'killBreaksRe': re.compile('(<br\s*\/?>(\s|&nbsp;?)*){1,}/'),
#'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I),
#skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
} }
def describe(node, depth=1): def describe(node, depth=1):
if not hasattr(node, 'tag'): if not hasattr(node, 'tag'):
return "[%s]" % type(node) return "[%s]" % type(node)
name = node.tag name = node.tag
if node.get('id', ''): name += '#'+node.get('id') if node.get('id', ''): name += '#'+node.get('id')
if node.get('class', ''): if node.get('class', ''):
name += '.' + node.get('class').replace(' ','.') name += '.' + node.get('class').replace(' ','.')
if name[:4] in ['div#', 'div.']: if name[:4] in ['div#', 'div.']:
name = name[3:] name = name[3:]
if depth and node.getparent() is not None: if depth and node.getparent() is not None:
return name+' - '+describe(node.getparent(), depth-1) return name+' - '+describe(node.getparent(), depth-1)
return name return name
def to_int(x): def to_int(x):
if not x: return None if not x: return None
x = x.strip() x = x.strip()
if x.endswith('px'): if x.endswith('px'):
return int(x[:-2]) return int(x[:-2])
if x.endswith('em'): if x.endswith('em'):
return int(x[:-2]) * 12 return int(x[:-2]) * 12
return int(x) return int(x)
def clean(text): def clean(text):
text = re.sub('\s*\n\s*', '\n', text) text = re.sub('\s*\n\s*', '\n', text)
text = re.sub('[ \t]{2,}', ' ', text) text = re.sub('[ \t]{2,}', ' ', text)
return text.strip() return text.strip()
def text_length(i): def text_length(i):
return len(clean(i.text_content() or "")) return len(clean(i.text_content() or ""))
class Unparseable(ValueError): class Unparseable(ValueError):
pass pass
class Document: class Document:
TEXT_LENGTH_THRESHOLD = 25 TEXT_LENGTH_THRESHOLD = 25
RETRY_LENGTH = 250 RETRY_LENGTH = 250
def __init__(self, input, **options): def __init__(self, input, **options):
self.input = input self.input = input
self.options = defaultdict(lambda: None) self.options = defaultdict(lambda: None)
for k, v in options.items(): for k, v in options.items():
self.options[k] = v self.options[k] = v
self.html = None self.html = None
def _html(self, force=False): def _html(self, force=False):
if force or self.html is None: if force or self.html is None:
self.html = self._parse(self.input) self.html = self._parse(self.input)
return self.html return self.html
def _parse(self, input): def _parse(self, input):
doc = build_doc(input) doc = build_doc(input)
doc = html_cleaner.clean_html(doc) doc = html_cleaner.clean_html(doc)
base_href = self.options['url'] base_href = self.options['url']
if base_href: if base_href:
doc.make_links_absolute(base_href, resolve_base_href=True) doc.make_links_absolute(base_href, resolve_base_href=True)
else: else:
doc.resolve_base_href() doc.resolve_base_href()
return doc return doc
def content(self): def content(self):
return get_body(self._html(True)) return get_body(self._html(True))
def title(self): def title(self):
return get_title(self._html(True)) return get_title(self._html(True))
def short_title(self): def short_title(self):
return shorten_title(self._html(True)) return shorten_title(self._html(True))
def summary(self): def summary(self):
try: try:
ruthless = True ruthless = True
while True: while True:
self._html(True) self._html(True)
for i in self.tags(self.html, 'script', 'style'): for i in self.tags(self.html, 'script', 'style'):
i.drop_tree() i.drop_tree()
for i in self.tags(self.html, 'body'): for i in self.tags(self.html, 'body'):
i.set('id', 'readabilityBody') i.set('id', 'readabilityBody')
if ruthless: if ruthless:
self.remove_unlikely_candidates() self.remove_unlikely_candidates()
self.transform_misused_divs_into_paragraphs() self.transform_misused_divs_into_paragraphs()
candidates = self.score_paragraphs() candidates = self.score_paragraphs()
best_candidate = self.select_best_candidate(candidates) best_candidate = self.select_best_candidate(candidates)
if best_candidate: if best_candidate:
article = self.get_article(candidates, best_candidate) article = self.get_article(candidates, best_candidate)
else: else:
if ruthless: if ruthless:
logging.debug("ruthless removal did not work. ") logging.debug("ruthless removal did not work. ")
ruthless = False ruthless = False
self.debug("ended up stripping too much - going for a safer _parse") self.debug("ended up stripping too much - going for a safer _parse")
# try again # try again
continue continue
else: else:
logging.debug("Ruthless and lenient parsing did not work. Returning raw html") logging.debug("Ruthless and lenient parsing did not work. Returning raw html")
article = self.html.find('body') or self.html article = self.html.find('body') or self.html
cleaned_article = self.sanitize(article, candidates) cleaned_article = self.sanitize(article, candidates)
of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH)
if ruthless and not of_acceptable_length: if ruthless and not of_acceptable_length:
ruthless = False ruthless = False
continue # try again continue # try again
else: else:
return cleaned_article return cleaned_article
except StandardError, e: except StandardError, e:
#logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info())))
logging.exception('error getting summary: ' ) logging.exception('error getting summary: ' )
raise Unparseable(str(e)), None, sys.exc_info()[2] raise Unparseable(str(e)), None, sys.exc_info()[2]
def get_article(self, candidates, best_candidate): def get_article(self, candidates, best_candidate):
# Now that we have the top candidate, look through its siblings for content that might also be related. # Now that we have the top candidate, look through its siblings for content that might also be related.
# Things like preambles, content split by ads that we removed, etc. # Things like preambles, content split by ads that we removed, etc.
sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2])
output = document_fromstring('<div/>') body = B.BODY()
best_elem = best_candidate['elem'] html = B.HTML(body)
for sibling in best_elem.getparent().getchildren(): best_elem = best_candidate['elem']
#if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text for sibling in best_elem.getparent().getchildren():
append = False #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text
if sibling is best_elem: append = False
append = True if sibling is best_elem:
sibling_key = sibling #HashableElement(sibling) append = True
if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: sibling_key = sibling #HashableElement(sibling)
append = True if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold:
append = True
if sibling.tag == "p":
link_density = self.get_link_density(sibling) if sibling.tag == "p":
node_content = sibling.text or "" link_density = self.get_link_density(sibling)
node_length = len(node_content) node_content = sibling.text or ""
node_length = len(node_content)
if node_length > 80 and link_density < 0.25:
append = True if node_length > 80 and link_density < 0.25:
elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content): append = True
append = True elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content):
append = True
if append:
output.append(sibling) if append:
#if output is not None: body.append(sibling)
# output.append(best_elem)
return output #if body is not None:
# body.append(best_elem)
def select_best_candidate(self, candidates): return html
sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True)
for candidate in sorted_candidates[:5]: def select_best_candidate(self, candidates):
elem = candidate['elem'] sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True)
self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) for candidate in sorted_candidates[:5]:
elem = candidate['elem']
if len(sorted_candidates) == 0: self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem)))
return None
if len(sorted_candidates) == 0:
best_candidate = sorted_candidates[0] return None
return best_candidate
best_candidate = sorted_candidates[0]
return best_candidate
def get_link_density(self, elem):
link_length = 0
for i in elem.findall(".//a"): def get_link_density(self, elem):
link_length += text_length(i) link_length = 0
#if len(elem.findall(".//div") or elem.findall(".//p")): for i in elem.findall(".//a"):
# link_length = link_length link_length += text_length(i)
total_length = text_length(elem) #if len(elem.findall(".//div") or elem.findall(".//p")):
return float(link_length) / max(total_length, 1) # link_length = link_length
total_length = text_length(elem)
def score_paragraphs(self, ): return float(link_length) / max(total_length, 1)
MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)
candidates = {} def score_paragraphs(self, ):
#self.debug(str([describe(node) for node in self.tags(self.html, "div")])) MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)
candidates = {}
ordered = [] #self.debug(str([describe(node) for node in self.tags(self.html, "div")]))
for elem in self.tags(self.html, "p", "pre", "td"):
parent_node = elem.getparent() ordered = []
if parent_node is None: for elem in self.tags(self.html, "p", "pre", "td"):
continue parent_node = elem.getparent()
grand_parent_node = parent_node.getparent() if parent_node is None:
continue
inner_text = clean(elem.text_content() or "") grand_parent_node = parent_node.getparent()
inner_text_len = len(inner_text)
inner_text = clean(elem.text_content() or "")
# If this paragraph is less than 25 characters, don't even count it. inner_text_len = len(inner_text)
if inner_text_len < MIN_LEN:
continue # If this paragraph is less than 25 characters, don't even count it.
if inner_text_len < MIN_LEN:
if parent_node not in candidates: continue
candidates[parent_node] = self.score_node(parent_node)
ordered.append(parent_node) if parent_node not in candidates:
candidates[parent_node] = self.score_node(parent_node)
if grand_parent_node is not None and grand_parent_node not in candidates: ordered.append(parent_node)
candidates[grand_parent_node] = self.score_node(grand_parent_node)
ordered.append(grand_parent_node) if grand_parent_node is not None and grand_parent_node not in candidates:
candidates[grand_parent_node] = self.score_node(grand_parent_node)
content_score = 1 ordered.append(grand_parent_node)
content_score += len(inner_text.split(','))
content_score += min((inner_text_len / 100), 3) content_score = 1
#if elem not in candidates: content_score += len(inner_text.split(','))
# candidates[elem] = self.score_node(elem) content_score += min((inner_text_len / 100), 3)
#if elem not in candidates:
#WTF? candidates[elem]['content_score'] += content_score # candidates[elem] = self.score_node(elem)
candidates[parent_node]['content_score'] += content_score
if grand_parent_node is not None: #WTF? candidates[elem]['content_score'] += content_score
candidates[grand_parent_node]['content_score'] += content_score / 2.0 candidates[parent_node]['content_score'] += content_score
if grand_parent_node is not None:
# Scale the final candidates score based on link density. Good content should have a candidates[grand_parent_node]['content_score'] += content_score / 2.0
# relatively small link density (5% or less) and be mostly unaffected by this operation.
for elem in ordered: # Scale the final candidates score based on link density. Good content should have a
candidate = candidates[elem] # relatively small link density (5% or less) and be mostly unaffected by this operation.
ld = self.get_link_density(elem) for elem in ordered:
score = candidate['content_score'] candidate = candidates[elem]
self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) ld = self.get_link_density(elem)
candidate['content_score'] *= (1 - ld) score = candidate['content_score']
self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld)))
return candidates candidate['content_score'] *= (1 - ld)
def class_weight(self, e): return candidates
weight = 0
if e.get('class', None): def class_weight(self, e):
if REGEXES['negativeRe'].search(e.get('class')): weight = 0
weight -= 25 if e.get('class', None):
if REGEXES['negativeRe'].search(e.get('class')):
if REGEXES['positiveRe'].search(e.get('class')): weight -= 25
weight += 25
if REGEXES['positiveRe'].search(e.get('class')):
if e.get('id', None): weight += 25
if REGEXES['negativeRe'].search(e.get('id')):
weight -= 25 if e.get('id', None):
if REGEXES['negativeRe'].search(e.get('id')):
if REGEXES['positiveRe'].search(e.get('id')): weight -= 25
weight += 25
if REGEXES['positiveRe'].search(e.get('id')):
return weight weight += 25
def score_node(self, elem): return weight
content_score = self.class_weight(elem)
name = elem.tag.lower() def score_node(self, elem):
if name == "div": content_score = self.class_weight(elem)
content_score += 5 name = elem.tag.lower()
elif name in ["pre", "td", "blockquote"]: if name == "div":
content_score += 3 content_score += 5
elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: elif name in ["pre", "td", "blockquote"]:
content_score -= 3 content_score += 3
elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]:
content_score -= 5 content_score -= 3
return { elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]:
'content_score': content_score, content_score -= 5
'elem': elem return {
} 'content_score': content_score,
'elem': elem
def debug(self, *a): }
#if self.options['debug']:
logging.debug(*a) def debug(self, *a):
#if self.options['debug']:
def remove_unlikely_candidates(self): logging.debug(*a)
for elem in self.html.iter():
s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) def remove_unlikely_candidates(self):
#self.debug(s) for elem in self.html.iter():
if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': s = "%s %s" % (elem.get('class', ''), elem.get('id', ''))
self.debug("Removing unlikely candidate - %s" % describe(elem)) #self.debug(s)
elem.drop_tree() if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body':
self.debug("Removing unlikely candidate - %s" % describe(elem))
def transform_misused_divs_into_paragraphs(self): elem.drop_tree()
for elem in self.tags(self.html, 'div'):
# transform <div>s that do not contain other block elements into <p>s def transform_misused_divs_into_paragraphs(self):
if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): for elem in self.tags(self.html, 'div'):
#self.debug("Altering %s to p" % (describe(elem))) # transform <div>s that do not contain other block elements into <p>s
elem.tag = "p" if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))):
#print "Fixed element "+describe(elem) #self.debug("Altering %s to p" % (describe(elem)))
elem.tag = "p"
for elem in self.tags(self.html, 'div'): #print "Fixed element "+describe(elem)
if elem.text and elem.text.strip():
p = fragment_fromstring('<p/>') for elem in self.tags(self.html, 'div'):
p.text = elem.text if elem.text and elem.text.strip():
elem.text = None p = fragment_fromstring('<p/>')
elem.insert(0, p) p.text = elem.text
#print "Appended "+tounicode(p)+" to "+describe(elem) elem.text = None
elem.insert(0, p)
for pos, child in reversed(list(enumerate(elem))): #print "Appended "+tounicode(p)+" to "+describe(elem)
if child.tail and child.tail.strip():
p = fragment_fromstring('<p/>') for pos, child in reversed(list(enumerate(elem))):
p.text = child.tail if child.tail and child.tail.strip():
child.tail = None p = fragment_fromstring('<p/>')
elem.insert(pos + 1, p) p.text = child.tail
#print "Inserted "+tounicode(p)+" to "+describe(elem) child.tail = None
if child.tag == 'br': elem.insert(pos + 1, p)
#print 'Dropped <br> at '+describe(elem) #print "Inserted "+tounicode(p)+" to "+describe(elem)
child.drop_tree() if child.tag == 'br':
#print 'Dropped <br> at '+describe(elem)
def tags(self, node, *tag_names): child.drop_tree()
for tag_name in tag_names:
for e in node.findall('.//%s' % tag_name): def tags(self, node, *tag_names):
yield e for tag_name in tag_names:
for e in node.findall('.//%s' % tag_name):
def reverse_tags(self, node, *tag_names): yield e
for tag_name in tag_names:
for e in reversed(node.findall('.//%s' % tag_name)): def reverse_tags(self, node, *tag_names):
yield e for tag_name in tag_names:
for e in reversed(node.findall('.//%s' % tag_name)):
def sanitize(self, node, candidates): yield e
MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)
for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): def sanitize(self, node, candidates):
if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)
header.drop_tree() for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33:
for elem in self.tags(node, "form", "iframe", "textarea"): header.drop_tree()
elem.drop_tree()
allowed = {} for elem in self.tags(node, "form", "iframe", "textarea"):
# Conditionally clean <table>s, <ul>s, and <div>s elem.drop_tree()
for el in self.reverse_tags(node, "table", "ul", "div"): allowed = {}
if el in allowed: # Conditionally clean <table>s, <ul>s, and <div>s
continue for el in self.reverse_tags(node, "table", "ul", "div"):
weight = self.class_weight(el) if el in allowed:
if el in candidates: continue
content_score = candidates[el]['content_score'] weight = self.class_weight(el)
#print '!',el, '-> %6.3f' % content_score if el in candidates:
else: content_score = candidates[el]['content_score']
content_score = 0 #print '!',el, '-> %6.3f' % content_score
tag = el.tag else:
content_score = 0
if weight + content_score < 0: tag = el.tag
self.debug("Cleaned %s with score %6.3f and weight %-3s" %
(describe(el), content_score, weight, )) if weight + content_score < 0:
el.drop_tree() self.debug("Cleaned %s with score %6.3f and weight %-3s" %
elif el.text_content().count(",") < 10: (describe(el), content_score, weight, ))
counts = {} el.drop_tree()
for kind in ['p', 'img', 'li', 'a', 'embed', 'input']: elif el.text_content().count(",") < 10:
counts[kind] = len(el.findall('.//%s' %kind)) counts = {}
counts["li"] -= 100 for kind in ['p', 'img', 'li', 'a', 'embed', 'input']:
counts[kind] = len(el.findall('.//%s' %kind))
content_length = text_length(el) # Count the text length excluding any surrounding whitespace counts["li"] -= 100
link_density = self.get_link_density(el)
parent_node = el.getparent() content_length = text_length(el) # Count the text length excluding any surrounding whitespace
if parent_node is not None: link_density = self.get_link_density(el)
if parent_node in candidates: parent_node = el.getparent()
content_score = candidates[parent_node]['content_score'] if parent_node is not None:
else: if parent_node in candidates:
content_score = 0 content_score = candidates[parent_node]['content_score']
#if parent_node is not None: else:
#pweight = self.class_weight(parent_node) + content_score content_score = 0
#pname = describe(parent_node) #if parent_node is not None:
#else: #pweight = self.class_weight(parent_node) + content_score
#pweight = 0 #pname = describe(parent_node)
#pname = "no parent" #else:
to_remove = False #pweight = 0
reason = "" #pname = "no parent"
to_remove = False
#if el.tag == 'div' and counts["img"] >= 1: reason = ""
# continue
if counts["p"] and counts["img"] > counts["p"]: #if el.tag == 'div' and counts["img"] >= 1:
reason = "too many images (%s)" % counts["img"] # continue
to_remove = True if counts["p"] and counts["img"] > counts["p"]:
elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol": reason = "too many images (%s)" % counts["img"]
reason = "more <li>s than <p>s" to_remove = True
to_remove = True elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol":
elif counts["input"] > (counts["p"] / 3): reason = "more <li>s than <p>s"
reason = "less than 3x <p>s than <input>s" to_remove = True
to_remove = True elif counts["input"] > (counts["p"] / 3):
elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2): reason = "less than 3x <p>s than <input>s"
reason = "too short content length %s without a single image" % content_length to_remove = True
to_remove = True elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2):
elif weight < 25 and link_density > 0.2: reason = "too short content length %s without a single image" % content_length
reason = "too many links %.3f for its weight %s" % (link_density, weight) to_remove = True
to_remove = True elif weight < 25 and link_density > 0.2:
elif weight >= 25 and link_density > 0.5: reason = "too many links %.3f for its weight %s" % (link_density, weight)
reason = "too many links %.3f for its weight %s" % (link_density, weight) to_remove = True
to_remove = True elif weight >= 25 and link_density > 0.5:
elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: reason = "too many links %.3f for its weight %s" % (link_density, weight)
reason = "<embed>s with too short content length, or too many <embed>s" to_remove = True
to_remove = True elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1:
# if el.tag == 'div' and counts['img'] >= 1 and to_remove: reason = "<embed>s with too short content length, or too many <embed>s"
# imgs = el.findall('.//img') to_remove = True
# valid_img = False # if el.tag == 'div' and counts['img'] >= 1 and to_remove:
# self.debug(tounicode(el)) # imgs = el.findall('.//img')
# for img in imgs: # valid_img = False
# self.debug(tounicode(el))
# for img in imgs:
# #
# height = img.get('height') # height = img.get('height')
# text_length = img.get('text_length') # text_length = img.get('text_length')
# self.debug ("height %s text_length %s" %(repr(height), repr(text_length))) # self.debug ("height %s text_length %s" %(repr(height), repr(text_length)))
# if to_int(height) >= 100 or to_int(text_length) >= 100: # if to_int(height) >= 100 or to_int(text_length) >= 100:
# valid_img = True # valid_img = True
# self.debug("valid image" + tounicode(img)) # self.debug("valid image" + tounicode(img))
# break # break
# if valid_img: # if valid_img:
# to_remove = False # to_remove = False
# self.debug("Allowing %s" %el.text_content()) # self.debug("Allowing %s" %el.text_content())
# for desnode in self.tags(el, "table", "ul", "div"): # for desnode in self.tags(el, "table", "ul", "div"):
# allowed[desnode] = True # allowed[desnode] = True
#find x non empty preceding and succeeding siblings #find x non empty preceding and succeeding siblings
i, j = 0, 0 i, j = 0, 0
x = 1 x = 1
siblings = [] siblings = []
for sib in el.itersiblings(): for sib in el.itersiblings():
#self.debug(sib.text_content()) #self.debug(sib.text_content())
sib_content_length = text_length(sib) sib_content_length = text_length(sib)
if sib_content_length: if sib_content_length:
i =+ 1 i =+ 1
siblings.append(sib_content_length) siblings.append(sib_content_length)
if i == x: if i == x:
break break
for sib in el.itersiblings(preceding=True): for sib in el.itersiblings(preceding=True):
#self.debug(sib.text_content()) #self.debug(sib.text_content())
sib_content_length = text_length(sib) sib_content_length = text_length(sib)
if sib_content_length: if sib_content_length:
j =+ 1 j =+ 1
siblings.append(sib_content_length) siblings.append(sib_content_length)
if j == x: if j == x:
break break
#self.debug(str(siblings)) #self.debug(str(siblings))
if siblings and sum(siblings) > 1000 : if siblings and sum(siblings) > 1000 :
to_remove = False to_remove = False
self.debug("Allowing %s" % describe(el)) self.debug("Allowing %s" % describe(el))
for desnode in self.tags(el, "table", "ul", "div"): for desnode in self.tags(el, "table", "ul", "div"):
allowed[desnode] = True allowed[desnode] = True
if to_remove: if to_remove:
self.debug("Cleaned %6.3f %s with weight %s cause it has %s." % self.debug("Cleaned %6.3f %s with weight %s cause it has %s." %
(content_score, describe(el), weight, reason)) (content_score, describe(el), weight, reason))
#print tounicode(el) #print tounicode(el)
#self.debug("pname %s pweight %.3f" %(pname, pweight)) #self.debug("pname %s pweight %.3f" %(pname, pweight))
el.drop_tree() el.drop_tree()
for el in ([node] + [n for n in node.iter()]): for el in ([node] + [n for n in node.iter()]):
if not (self.options['attributes']): if not (self.options['attributes']):
#el.attrib = {} #FIXME:Checkout the effects of disabling this #el.attrib = {} #FIXME:Checkout the effects of disabling this
pass pass
return clean_attributes(tounicode(node)) return clean_attributes(tounicode(node))
class HashableElement(): class HashableElement():
def __init__(self, node): def __init__(self, node):
self.node = node self.node = node
self._path = None self._path = None
def _get_path(self): def _get_path(self):
if self._path is None: if self._path is None:
reverse_path = [] reverse_path = []
node = self.node node = self.node
while node is not None: while node is not None:
node_id = (node.tag, tuple(node.attrib.items()), node.text) node_id = (node.tag, tuple(node.attrib.items()), node.text)
reverse_path.append(node_id) reverse_path.append(node_id)
node = node.getparent() node = node.getparent()
self._path = tuple(reverse_path) self._path = tuple(reverse_path)
return self._path return self._path
path = property(_get_path) path = property(_get_path)
def __hash__(self): def __hash__(self):
return hash(self.path) return hash(self.path)
def __eq__(self, other): def __eq__(self, other):
return self.path == other.path return self.path == other.path
def __getattr__(self, tag): def __getattr__(self, tag):
return getattr(self.node, tag) return getattr(self.node, tag)
def main(): def main():
from optparse import OptionParser from optparse import OptionParser
parser = OptionParser(usage="%prog: [options] [file]") parser = OptionParser(usage="%prog: [options] [file]")
parser.add_option('-v', '--verbose', action='store_true') parser.add_option('-v', '--verbose', action='store_true')
parser.add_option('-u', '--url', help="use URL instead of a local file") parser.add_option('-u', '--url', help="use URL instead of a local file")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if not (len(args) == 1 or options.url): if not (len(args) == 1 or options.url):
parser.print_help() parser.print_help()
sys.exit(1) sys.exit(1)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
file = None file = None
if options.url: if options.url:
import urllib import urllib
file = urllib.urlopen(options.url) file = urllib.urlopen(options.url)
else: else:
file = open(args[0]) file = open(args[0])
try: try:
print Document(file.read(), debug=options.verbose).summary().encode('ascii','ignore') print Document(file.read(), debug=options.verbose).summary().encode('ascii','ignore')
finally: finally:
file.close() file.close()
if __name__ == '__main__': if __name__ == '__main__':
main() main()

Write Preview
Loading…
Cancel
Save