From edccec5d3b4cecee3fdccff7667dd81bb3ed6258 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Mon, 16 Apr 2012 17:13:24 -0400 Subject: [PATCH 1/4] Work on why we have an empty tag - Seems to come because the sanitizer ends up with two nodes, not one. The first is an empty body, the second is the article div. - Fix up the tabs so we can work with the file. Needs lots of pep8 love. - Implement an initial hack that at least gets it working atm. - Start to add test cases, sample html files we can test against, etc. --- readability/readability.py | 964 +++++++++++++++--------------- tests/__init__.py | 0 tests/samples/si-game.sample.html | 762 +++++++++++++++++++++++ tests/test_article_only.py | 39 ++ 4 files changed, 1286 insertions(+), 479 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/samples/si-game.sample.html create mode 100644 tests/test_article_only.py diff --git a/readability/readability.py b/readability/readability.py index 9029a2f..b409c59 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -11,502 +11,508 @@ import sys logging.basicConfig(level=logging.INFO) REGEXES = { - 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), - 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), - 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), - 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), - 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), - #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), - #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), - #'trimRe': re.compile('^\s+|\s+$/'), - #'normalizeRe': re.compile('\s{2,}/'), - #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), - #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), - #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, + 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), + 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), + 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), + 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), + 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), + #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), + #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), + #'trimRe': re.compile('^\s+|\s+$/'), + #'normalizeRe': re.compile('\s{2,}/'), + #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), + #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), + #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } def describe(node, depth=1): - if not hasattr(node, 'tag'): - return "[%s]" % type(node) - name = node.tag - if node.get('id', ''): name += '#'+node.get('id') - if node.get('class', ''): - name += '.' + node.get('class').replace(' ','.') - if name[:4] in ['div#', 'div.']: - name = name[3:] - if depth and node.getparent() is not None: - return name+' - '+describe(node.getparent(), depth-1) - return name + if not hasattr(node, 'tag'): + return "[%s]" % type(node) + name = node.tag + if node.get('id', ''): name += '#'+node.get('id') + if node.get('class', ''): + name += '.' + node.get('class').replace(' ','.') + if name[:4] in ['div#', 'div.']: + name = name[3:] + if depth and node.getparent() is not None: + return name+' - '+describe(node.getparent(), depth-1) + return name def to_int(x): - if not x: return None - x = x.strip() - if x.endswith('px'): - return int(x[:-2]) - if x.endswith('em'): - return int(x[:-2]) * 12 - return int(x) + if not x: return None + x = x.strip() + if x.endswith('px'): + return int(x[:-2]) + if x.endswith('em'): + return int(x[:-2]) * 12 + return int(x) def clean(text): - text = re.sub('\s*\n\s*', '\n', text) - text = re.sub('[ \t]{2,}', ' ', text) - return text.strip() + text = re.sub('\s*\n\s*', '\n', text) + text = re.sub('[ \t]{2,}', ' ', text) + return text.strip() def text_length(i): - return len(clean(i.text_content() or "")) + return len(clean(i.text_content() or "")) class Unparseable(ValueError): - pass + pass class Document: - TEXT_LENGTH_THRESHOLD = 25 - RETRY_LENGTH = 250 - - def __init__(self, input, **options): - self.input = input - self.options = defaultdict(lambda: None) - for k, v in options.items(): - self.options[k] = v - self.html = None - - def _html(self, force=False): - if force or self.html is None: - self.html = self._parse(self.input) - return self.html - - def _parse(self, input): - doc = build_doc(input) - doc = html_cleaner.clean_html(doc) - base_href = self.options['url'] - if base_href: - doc.make_links_absolute(base_href, resolve_base_href=True) - else: - doc.resolve_base_href() - return doc - - def content(self): - return get_body(self._html(True)) - - def title(self): - return get_title(self._html(True)) - - def short_title(self): - return shorten_title(self._html(True)) - - def summary(self): - try: - ruthless = True - while True: - self._html(True) - - for i in self.tags(self.html, 'script', 'style'): - i.drop_tree() - for i in self.tags(self.html, 'body'): - i.set('id', 'readabilityBody') - if ruthless: - self.remove_unlikely_candidates() - self.transform_misused_divs_into_paragraphs() - candidates = self.score_paragraphs() - - best_candidate = self.select_best_candidate(candidates) - if best_candidate: - article = self.get_article(candidates, best_candidate) - else: - if ruthless: - logging.debug("ruthless removal did not work. ") - ruthless = False - self.debug("ended up stripping too much - going for a safer _parse") - # try again - continue - else: - logging.debug("Ruthless and lenient parsing did not work. Returning raw html") - article = self.html.find('body') - if article is None: - article = self.html - - cleaned_article = self.sanitize(article, candidates) - of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) - if ruthless and not of_acceptable_length: - ruthless = False - continue # try again - else: - return cleaned_article - except StandardError, e: - #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) - logging.exception('error getting summary: ' ) - raise Unparseable(str(e)), None, sys.exc_info()[2] - - def get_article(self, candidates, best_candidate): - # Now that we have the top candidate, look through its siblings for content that might also be related. - # Things like preambles, content split by ads that we removed, etc. - - sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) - output = document_fromstring('
') - best_elem = best_candidate['elem'] - for sibling in best_elem.getparent().getchildren(): - #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text - append = False - if sibling is best_elem: - append = True - sibling_key = sibling #HashableElement(sibling) - if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: - append = True - - if sibling.tag == "p": - link_density = self.get_link_density(sibling) - node_content = sibling.text or "" - node_length = len(node_content) - - if node_length > 80 and link_density < 0.25: - append = True - elif node_length <= 80 and link_density == 0 and re.search('\.( |$)', node_content): - append = True - - if append: - output.append(sibling) - #if output is not None: - # output.append(best_elem) - return output - - def select_best_candidate(self, candidates): - sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) - for candidate in sorted_candidates[:5]: - elem = candidate['elem'] - self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) - - if len(sorted_candidates) == 0: - return None - - best_candidate = sorted_candidates[0] - return best_candidate - - - def get_link_density(self, elem): - link_length = 0 - for i in elem.findall(".//a"): - link_length += text_length(i) - #if len(elem.findall(".//div") or elem.findall(".//p")): - # link_length = link_length - total_length = text_length(elem) - return float(link_length) / max(total_length, 1) - - def score_paragraphs(self, ): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) - candidates = {} - #self.debug(str([describe(node) for node in self.tags(self.html, "div")])) - - ordered = [] - for elem in self.tags(self._html(), "p", "pre", "td"): - parent_node = elem.getparent() - if parent_node is None: - continue - grand_parent_node = parent_node.getparent() - - inner_text = clean(elem.text_content() or "") - inner_text_len = len(inner_text) - - # If this paragraph is less than 25 characters, don't even count it. - if inner_text_len < MIN_LEN: - continue - - if parent_node not in candidates: - candidates[parent_node] = self.score_node(parent_node) - ordered.append(parent_node) - - if grand_parent_node is not None and grand_parent_node not in candidates: - candidates[grand_parent_node] = self.score_node(grand_parent_node) - ordered.append(grand_parent_node) - - content_score = 1 - content_score += len(inner_text.split(',')) - content_score += min((inner_text_len / 100), 3) - #if elem not in candidates: - # candidates[elem] = self.score_node(elem) - - #WTF? candidates[elem]['content_score'] += content_score - candidates[parent_node]['content_score'] += content_score - if grand_parent_node is not None: - candidates[grand_parent_node]['content_score'] += content_score / 2.0 - - # Scale the final candidates score based on link density. Good content should have a - # relatively small link density (5% or less) and be mostly unaffected by this operation. - for elem in ordered: - candidate = candidates[elem] - ld = self.get_link_density(elem) - score = candidate['content_score'] - self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) - candidate['content_score'] *= (1 - ld) - - return candidates - - def class_weight(self, e): - weight = 0 - if e.get('class', None): - if REGEXES['negativeRe'].search(e.get('class')): - weight -= 25 - - if REGEXES['positiveRe'].search(e.get('class')): - weight += 25 - - if e.get('id', None): - if REGEXES['negativeRe'].search(e.get('id')): - weight -= 25 - - if REGEXES['positiveRe'].search(e.get('id')): - weight += 25 - - return weight - - def score_node(self, elem): - content_score = self.class_weight(elem) - name = elem.tag.lower() - if name == "div": - content_score += 5 - elif name in ["pre", "td", "blockquote"]: - content_score += 3 - elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: - content_score -= 3 - elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: - content_score -= 5 - return { - 'content_score': content_score, - 'elem': elem - } - - def debug(self, *a): - #if self.options['debug']: - logging.debug(*a) - - def remove_unlikely_candidates(self): - for elem in self.html.iter(): - s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) - if len(s) < 2: - continue - #self.debug(s) - if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': - self.debug("Removing unlikely candidate - %s" % describe(elem)) - elem.drop_tree() - - def transform_misused_divs_into_paragraphs(self): - for elem in self.tags(self.html, 'div'): - # transform
s that do not contain other block elements into

s - #FIXME: The current implementation ignores all descendants that are not direct children of elem - # This results in incorrect results in case there is an buried within an for example - if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): - #self.debug("Altering %s to p" % (describe(elem))) - elem.tag = "p" - #print "Fixed element "+describe(elem) - - for elem in self.tags(self.html, 'div'): - if elem.text and elem.text.strip(): - p = fragment_fromstring('

') - p.text = elem.text - elem.text = None - elem.insert(0, p) - #print "Appended "+tounicode(p)+" to "+describe(elem) - - for pos, child in reversed(list(enumerate(elem))): - if child.tail and child.tail.strip(): - p = fragment_fromstring('

') - p.text = child.tail - child.tail = None - elem.insert(pos + 1, p) - #print "Inserted "+tounicode(p)+" to "+describe(elem) - if child.tag == 'br': - #print 'Dropped
at '+describe(elem) - child.drop_tree() - - def tags(self, node, *tag_names): - for tag_name in tag_names: - for e in node.findall('.//%s' % tag_name): - yield e - - def reverse_tags(self, node, *tag_names): - for tag_name in tag_names: - for e in reversed(node.findall('.//%s' % tag_name)): - yield e - - def sanitize(self, node, candidates): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) - for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): - if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: - header.drop_tree() - - for elem in self.tags(node, "form", "iframe", "textarea"): - elem.drop_tree() - allowed = {} - # Conditionally clean s,
    s, and
    s - for el in self.reverse_tags(node, "table", "ul", "div"): - if el in allowed: - continue - weight = self.class_weight(el) - if el in candidates: - content_score = candidates[el]['content_score'] - #print '!',el, '-> %6.3f' % content_score - else: - content_score = 0 - tag = el.tag - - if weight + content_score < 0: - self.debug("Cleaned %s with score %6.3f and weight %-3s" % - (describe(el), content_score, weight, )) - el.drop_tree() - elif el.text_content().count(",") < 10: - counts = {} - for kind in ['p', 'img', 'li', 'a', 'embed', 'input']: - counts[kind] = len(el.findall('.//%s' %kind)) - counts["li"] -= 100 - - content_length = text_length(el) # Count the text length excluding any surrounding whitespace - link_density = self.get_link_density(el) - parent_node = el.getparent() - if parent_node is not None: - if parent_node in candidates: - content_score = candidates[parent_node]['content_score'] - else: - content_score = 0 - #if parent_node is not None: - #pweight = self.class_weight(parent_node) + content_score - #pname = describe(parent_node) - #else: - #pweight = 0 - #pname = "no parent" - to_remove = False - reason = "" - - #if el.tag == 'div' and counts["img"] >= 1: - # continue - if counts["p"] and counts["img"] > counts["p"]: - reason = "too many images (%s)" % counts["img"] - to_remove = True - elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol": - reason = "more
  • s than

    s" - to_remove = True - elif counts["input"] > (counts["p"] / 3): - reason = "less than 3x

    s than s" - to_remove = True - elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2): - reason = "too short content length %s without a single image" % content_length - to_remove = True - elif weight < 25 and link_density > 0.2: - reason = "too many links %.3f for its weight %s" % (link_density, weight) - to_remove = True - elif weight >= 25 and link_density > 0.5: - reason = "too many links %.3f for its weight %s" % (link_density, weight) - to_remove = True - elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: - reason = "s with too short content length, or too many s" - to_remove = True -# if el.tag == 'div' and counts['img'] >= 1 and to_remove: -# imgs = el.findall('.//img') -# valid_img = False -# self.debug(tounicode(el)) -# for img in imgs: + TEXT_LENGTH_THRESHOLD = 25 + RETRY_LENGTH = 250 + + def __init__(self, input, **options): + self.input = input + self.options = defaultdict(lambda: None) + for k, v in options.items(): + self.options[k] = v + self.html = None + + def _html(self, force=False): + if force or self.html is None: + self.html = self._parse(self.input) + return self.html + + def _parse(self, input): + doc = build_doc(input) + doc = html_cleaner.clean_html(doc) + base_href = self.options['url'] + if base_href: + doc.make_links_absolute(base_href, resolve_base_href=True) + else: + doc.resolve_base_href() + return doc + + def content(self): + return get_body(self._html(True)) + + def title(self): + return get_title(self._html(True)) + + def short_title(self): + return shorten_title(self._html(True)) + + def summary(self, document_only=False): + try: + ruthless = True + while True: + self._html(True) + + for i in self.tags(self.html, 'script', 'style'): + i.drop_tree() + for i in self.tags(self.html, 'body'): + i.set('id', 'readabilityBody') + if ruthless: + self.remove_unlikely_candidates() + self.transform_misused_divs_into_paragraphs() + candidates = self.score_paragraphs() + + best_candidate = self.select_best_candidate(candidates) + + if best_candidate: + article = self.get_article(candidates, best_candidate) + else: + if ruthless: + logging.debug("ruthless removal did not work. ") + ruthless = False + self.debug("ended up stripping too much - going for a safer _parse") + # try again + continue + else: + logging.debug("Ruthless and lenient parsing did not work. Returning raw html") + article = self.html.find('body') + if article is None: + article = self.html + cleaned_article = self.sanitize(article, candidates) + of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) + if ruthless and not of_acceptable_length: + ruthless = False + continue # try again + else: + return cleaned_article + except StandardError, e: + #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) + logging.exception('error getting summary: ' ) + raise Unparseable(str(e)), None, sys.exc_info()[2] + + def get_article(self, candidates, best_candidate): + # Now that we have the top candidate, look through its siblings for content that might also be related. + # Things like preambles, content split by ads that we removed, etc. + + sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) + output = document_fromstring('

    ') + best_elem = best_candidate['elem'] + for sibling in best_elem.getparent().getchildren(): + #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text + append = False + if sibling is best_elem: + append = True + sibling_key = sibling #HashableElement(sibling) + if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: + append = True + + if sibling.tag == "p": + link_density = self.get_link_density(sibling) + node_content = sibling.text or "" + node_length = len(node_content) + + if node_length > 80 and link_density < 0.25: + append = True + elif node_length <= 80 and link_density == 0 and re.search('\.( |$)', node_content): + append = True + + if append: + output.append(sibling) + #if output is not None: + # output.append(best_elem) + return output + + def select_best_candidate(self, candidates): + sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) + for candidate in sorted_candidates[:5]: + elem = candidate['elem'] + self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) + + if len(sorted_candidates) == 0: + return None + + best_candidate = sorted_candidates[0] + return best_candidate + + + def get_link_density(self, elem): + link_length = 0 + for i in elem.findall(".//a"): + link_length += text_length(i) + #if len(elem.findall(".//div") or elem.findall(".//p")): + # link_length = link_length + total_length = text_length(elem) + return float(link_length) / max(total_length, 1) + + def score_paragraphs(self, ): + MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + candidates = {} + #self.debug(str([describe(node) for node in self.tags(self.html, "div")])) + + ordered = [] + for elem in self.tags(self._html(), "p", "pre", "td"): + parent_node = elem.getparent() + if parent_node is None: + continue + grand_parent_node = parent_node.getparent() + + inner_text = clean(elem.text_content() or "") + inner_text_len = len(inner_text) + + # If this paragraph is less than 25 characters, don't even count it. + if inner_text_len < MIN_LEN: + continue + + if parent_node not in candidates: + candidates[parent_node] = self.score_node(parent_node) + ordered.append(parent_node) + + if grand_parent_node is not None and grand_parent_node not in candidates: + candidates[grand_parent_node] = self.score_node(grand_parent_node) + ordered.append(grand_parent_node) + + content_score = 1 + content_score += len(inner_text.split(',')) + content_score += min((inner_text_len / 100), 3) + #if elem not in candidates: + # candidates[elem] = self.score_node(elem) + + #WTF? candidates[elem]['content_score'] += content_score + candidates[parent_node]['content_score'] += content_score + if grand_parent_node is not None: + candidates[grand_parent_node]['content_score'] += content_score / 2.0 + + # Scale the final candidates score based on link density. Good content should have a + # relatively small link density (5% or less) and be mostly unaffected by this operation. + for elem in ordered: + candidate = candidates[elem] + ld = self.get_link_density(elem) + score = candidate['content_score'] + self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) + candidate['content_score'] *= (1 - ld) + + return candidates + + def class_weight(self, e): + weight = 0 + if e.get('class', None): + if REGEXES['negativeRe'].search(e.get('class')): + weight -= 25 + + if REGEXES['positiveRe'].search(e.get('class')): + weight += 25 + + if e.get('id', None): + if REGEXES['negativeRe'].search(e.get('id')): + weight -= 25 + + if REGEXES['positiveRe'].search(e.get('id')): + weight += 25 + + return weight + + def score_node(self, elem): + content_score = self.class_weight(elem) + name = elem.tag.lower() + if name == "div": + content_score += 5 + elif name in ["pre", "td", "blockquote"]: + content_score += 3 + elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: + content_score -= 3 + elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: + content_score -= 5 + return { + 'content_score': content_score, + 'elem': elem + } + + def debug(self, *a): + #if self.options['debug']: + logging.debug(*a) + + def remove_unlikely_candidates(self): + for elem in self.html.iter(): + s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) + if len(s) < 2: + continue + #self.debug(s) + if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': + self.debug("Removing unlikely candidate - %s" % describe(elem)) + elem.drop_tree() + + def transform_misused_divs_into_paragraphs(self): + for elem in self.tags(self.html, 'div'): + # transform
    s that do not contain other block elements into

    s + #FIXME: The current implementation ignores all descendants that are not direct children of elem + # This results in incorrect results in case there is an buried within an for example + if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): + #self.debug("Altering %s to p" % (describe(elem))) + elem.tag = "p" + #print "Fixed element "+describe(elem) + + for elem in self.tags(self.html, 'div'): + if elem.text and elem.text.strip(): + p = fragment_fromstring('

    ') + p.text = elem.text + elem.text = None + elem.insert(0, p) + #print "Appended "+tounicode(p)+" to "+describe(elem) + + for pos, child in reversed(list(enumerate(elem))): + if child.tail and child.tail.strip(): + p = fragment_fromstring('

    ') + p.text = child.tail + child.tail = None + elem.insert(pos + 1, p) + #print "Inserted "+tounicode(p)+" to "+describe(elem) + if child.tag == 'br': + #print 'Dropped
    at '+describe(elem) + child.drop_tree() + + def tags(self, node, *tag_names): + for tag_name in tag_names: + for e in node.findall('.//%s' % tag_name): + yield e + + def reverse_tags(self, node, *tag_names): + for tag_name in tag_names: + for e in reversed(node.findall('.//%s' % tag_name)): + yield e + + def sanitize(self, node, candidates): + MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): + if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: + header.drop_tree() + + for elem in self.tags(node, "form", "iframe", "textarea"): + elem.drop_tree() + allowed = {} + # Conditionally clean

s,
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 123456789RHE
TIGERS            
ROYALS            
+

+
+
    +
  1. 8:10 PM ET + +
    2. +
+ +
+
+ + + + +
+ + + + + + +
PREVIEWMATCHUPFAN COMMENTS
+
+ + + +
+
+
+ + +

Tigers-Royals Preview

+

+ + Justin Verlander + has pitched well in each of his first two starts, though he doesn't have a win to show for those efforts. + +

+

+ He hasn't had much trouble earning victories against the + Kansas City Royals + . + +

+

+ Verlander looks to continue his mastery of the Royals when the + Detroit Tigers + visit Kauffman Stadium in the opener of a three-game series Monday night. + +

+

+ The reigning AL + Cy Young + winner and MVP had a 2-0 lead through eight innings in both of his outings, but the Tigers weren't able to hold the lead. + +

+

Verlander (0-1, 2.20 ERA) allowed two hits before running into trouble in the ninth against Tampa Bay on Wednesday, getting + charged with four runs in 8 1-3 innings of a 4-2 defeat. +

"Once a couple guys got on, really the first time I've cranked it up like that - and lost a little bit of my consistency that + I'd had all day," Verlander said. "It's inexcusable. This loss rests solely on my shoulders." +

The right-hander did his part in his opening-day start against Boston on April 5, allowing two hits before the bullpen faltered. + Detroit ended up winning 3-2 with a run in the bottom of the ninth, though Verlander didn't earn a decision. +

+

That hasn't been the case in his last four starts against the Royals, winning each with a 1.82 ERA. Verlander is 13-2 with + a 2.40 ERA in 19 career starts versus Kansas City, and another win will give him more victories than he has against any other + team. He's also beaten Cleveland 13 times. +

+

Verlander is 8-2 with a 1.82 ERA lifetime at Kauffman Stadium, where the Royals (3-6) were swept in a three-game series against + the Indians with Sunday's 13-7 loss. +

+

+ + Billy Butler + , who is 14 for 39 (.359) with two homers off Verlander, had an RBI single and is hitting .364 with four doubles and a homer + during a five-game hitting streak. + +

+

+ Royals pitchers allowed seven home runs, 17 extra-base hits and 32 runs in the series, and manager + Ned Yost + turned to outfielder + Mitch Maier + in the ninth to pitched a scoreless inning Sunday. + +

"Let's hope it doesn't happen again," Maier said. "I don't like to be put in that situation, but we needed an inning." +

+ Kansas City will look to bounce back with the help of another solid outing from + Danny Duffy + (1-0, 0.00), who allowed one hit and struck out eight in six innings of a 3-0 win over Oakland on Tuesday. + +

+

The left-hander will be seeking his first win against Detroit after going 0-2 with a 5.63 ERA in three starts versus the Tigers + as a rookie. +

+

+ + Gerald Laird + was a triple short of the cycle and helped the Tigers (6-3) salvage the finale of a three-game series with a 5-2 victory over + Chicago on Sunday. + +

+

+ + Rick Porcello + allowed one run in 7 2-3 innings to give Detroit's starting rotation its first victory. + +

"All the other starters have pitched well," Porcello said. "It's just the way it's happened so far." +

Verlander allowed three runs in seven innings of a 4-3 win over the Royals on Aug. 6, beating Duffy, who gave up three runs + over five. +

+ +

+ © 2011 STATS LLC STATS, Inc + +

+
+
+ +
+
+
+
+ +
+
+
+
+
TURN COMMENTS OFF
+
TURN COMMENTS ON
+
+
+
+
+
+ + + +
+ +
+
+
+
+
SI.com
+
Hot Topics: Peter King: MMQB NHL Playoffs Bobby Petrino Bobby Valentine Roger Clemens MLB Power Rankings Jackie Robinson
+
+
+ +
+
+ + Turner - SI Digital + +
Terms under which this service is provided to you. Read our privacy guidelines, your California privacy rights, and ad choices. +
+
+
SI CoverRead All ArticlesBuy Cover Reprint +
+
+
+ + + + +
+
+
+ + + + + + + + +
+ + + + + + + + + + +
+ +
+
+
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_article_only.py b/tests/test_article_only.py new file mode 100644 index 0000000..41bfd85 --- /dev/null +++ b/tests/test_article_only.py @@ -0,0 +1,39 @@ +import os +import unittest + +from readability import Document + + +SAMPLES = os.path.join(os.path.dirname(__file__), 'samples') + + +def load_sample(filename): + """Helper to get the content out of the sample files""" + return open(os.path.join(SAMPLES, filename)).read() + + +class TestArticleOnly(unittest.TestCase): + """The option to not get back a full html doc should work + + Given a full html document, the call can request just divs of processed + content. In this way the developer can then wrap the article however they + want in their own view or application. + + """ + + def setUp(self): + """""" + pass + + def tearDown(self): + """""" + pass + + def test_si_sample(self): + """Using the si sample, make sure we can get the article alone.""" + sample = load_sample('si-game.sample.html') + doc = Document(sample) + res = doc.summary(document_only=True) + + self.assertEqual('
') + # create a new html document with a html->body->div + if document_only: + output = fragment_fromstring('
') + else: + output = document_fromstring('
') best_elem = best_candidate['elem'] for sibling in best_elem.getparent().getchildren(): #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text @@ -163,7 +166,12 @@ class Document: append = True if append: - output.append(sibling) + # We don't want to append directly to output, but the div + # in html->body->div + if document_only: + output.append(sibling) + else: + output.getchildren()[0].getchildren()[0].append(sibling) #if output is not None: # output.append(best_elem) return output @@ -454,13 +462,7 @@ class Document: if not (self.options['attributes']): #el.attrib = {} #FIXME:Checkout the effects of disabling this pass - # There can be two nodes here. We really want to tounicode only one of - # them. - # To start with let's hack it to get the longest tree as our document. - if len(node.getchildren()) > 1: - children = node.getchildren() - sorted_list = sorted(children, key=len, reverse=True) - node = sorted_list[0] + return clean_attributes(tounicode(node)) diff --git a/tests/test_article_only.py b/tests/test_article_only.py index 41bfd85..28240bd 100644 --- a/tests/test_article_only.py +++ b/tests/test_article_only.py @@ -21,19 +21,19 @@ class TestArticleOnly(unittest.TestCase): """ - def setUp(self): - """""" - pass - - def tearDown(self): - """""" - pass - def test_si_sample(self): + """Using the si sample, load article with only opening body element""" + sample = load_sample('si-game.sample.html') + doc = Document( + sample, + url='http://sportsillustrated.cnn.com/baseball/mlb/gameflash/2012/04/16/40630_preview.html') + res = doc.summary() + self.assertEqual('
Date: Mon, 16 Apr 2012 21:23:19 -0400 Subject: [PATCH 3/4] Try to pep8 all the things but give up when I got close. --- readability/readability.py | 189 +++++++++++++++++++++++++------------ 1 file changed, 129 insertions(+), 60 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index ae760c5..0a40198 100755 --- a/readability/readability.py +++ b/readability/readability.py @@ -1,21 +1,32 @@ #!/usr/bin/env python -from cleaners import html_cleaner, clean_attributes -from collections import defaultdict -from htmls import build_doc, get_body, get_title, shorten_title -from lxml.etree import tostring, tounicode -from lxml.html import fragment_fromstring, document_fromstring import logging import re import sys +from collections import defaultdict +from lxml.etree import tostring +from lxml.etree import tounicode +from lxml.html import document_fromstring +from lxml.html import fragment_fromstring + +from cleaners import clean_attributes +from cleaners import html_cleaner +from htmls import build_doc +from htmls import get_body +from htmls import get_title +from htmls import shorten_title + + logging.basicConfig(level=logging.INFO) +log = logging.getLogger() + REGEXES = { - 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), - 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), - 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), - 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), - 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), + 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter', re.I), + 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow', re.I), + 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story', re.I), + 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget', re.I), + 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)', re.I), #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), #'trimRe': re.compile('^\s+|\s+$/'), @@ -25,21 +36,29 @@ REGEXES = { #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } + +class Unparseable(ValueError): + pass + + def describe(node, depth=1): if not hasattr(node, 'tag'): return "[%s]" % type(node) name = node.tag - if node.get('id', ''): name += '#'+node.get('id') + if node.get('id', ''): + name += '#' + node.get('id') if node.get('class', ''): - name += '.' + node.get('class').replace(' ','.') + name += '.' + node.get('class').replace(' ', '.') if name[:4] in ['div#', 'div.']: name = name[3:] if depth and node.getparent() is not None: - return name+' - '+describe(node.getparent(), depth-1) + return name + ' - ' + describe(node.getparent(), depth - 1) return name + def to_int(x): - if not x: return None + if not x: + return None x = x.strip() if x.endswith('px'): return int(x[:-2]) @@ -47,26 +66,37 @@ def to_int(x): return int(x[:-2]) * 12 return int(x) + def clean(text): text = re.sub('\s*\n\s*', '\n', text) text = re.sub('[ \t]{2,}', ' ', text) return text.strip() + def text_length(i): return len(clean(i.text_content() or "")) -class Unparseable(ValueError): - pass class Document: + """Class to build a etree document out of html.""" TEXT_LENGTH_THRESHOLD = 25 RETRY_LENGTH = 250 def __init__(self, input, **options): + """Generate the document + + :param input: string of the html content. + + kwargs: + - attributes: + - debug: output debug messages + - min_text_length: + - retry_length: + - url: will allow adjusting links to be absolute + + """ self.input = input - self.options = defaultdict(lambda: None) - for k, v in options.items(): - self.options[k] = v + self.options = options self.html = None def _html(self, force=False): @@ -77,7 +107,7 @@ class Document: def _parse(self, input): doc = build_doc(input) doc = html_cleaner.clean_html(doc) - base_href = self.options['url'] + base_href = self.options.get('url', None) if base_href: doc.make_links_absolute(base_href, resolve_base_href=True) else: @@ -94,6 +124,12 @@ class Document: return shorten_title(self._html(True)) def summary(self, document_only=False): + """Generate the summary of the html docuemnt + + :param document_only: return only the div of the document, don't wrap + in html and body tags. + + """ try: ruthless = True while True: @@ -114,32 +150,43 @@ class Document: document_only=document_only) else: if ruthless: - logging.debug("ruthless removal did not work. ") + log.debug("ruthless removal did not work. ") ruthless = False - self.debug("ended up stripping too much - going for a safer _parse") + self.debug( + ("ended up stripping too much - " + "going for a safer _parse")) # try again continue else: - logging.debug("Ruthless and lenient parsing did not work. Returning raw html") + log.debug( + ("Ruthless and lenient parsing did not work. " + "Returning raw html")) article = self.html.find('body') if article is None: article = self.html cleaned_article = self.sanitize(article, candidates) - of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) + article_length = len(cleaned_article or '') + retry_length = self.options.get( + 'retry_length', + self.RETRY_LENGTH) + of_acceptable_length = article_length >= retry_length if ruthless and not of_acceptable_length: ruthless = False - continue # try again + # Loop through and try again. + continue else: return cleaned_article except StandardError, e: - #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) - logging.exception('error getting summary: ' ) + log.exception('error getting summary: ') raise Unparseable(str(e)), None, sys.exc_info()[2] def get_article(self, candidates, best_candidate, document_only=False): - # Now that we have the top candidate, look through its siblings for content that might also be related. + # Now that we have the top candidate, look through its siblings for + # content that might also be related. # Things like preambles, content split by ads that we removed, etc. - sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) + sibling_score_threshold = max([ + 10, + best_candidate['content_score'] * 0.2]) # create a new html document with a html->body->div if document_only: output = fragment_fromstring('
') @@ -147,12 +194,14 @@ class Document: output = document_fromstring('
') best_elem = best_candidate['elem'] for sibling in best_elem.getparent().getchildren(): - #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text + # in lxml there no concept of simple text + # if isinstance(sibling, NavigableString): continue append = False if sibling is best_elem: append = True - sibling_key = sibling #HashableElement(sibling) - if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: + sibling_key = sibling # HashableElement(sibling) + if sibling_key in candidates and \ + candidates[sibling_key]['content_score'] >= sibling_score_threshold: append = True if sibling.tag == "p": @@ -162,7 +211,9 @@ class Document: if node_length > 80 and link_density < 0.25: append = True - elif node_length <= 80 and link_density == 0 and re.search('\.( |$)', node_content): + elif node_length <= 80 \ + and link_density == 0 \ + and re.search('\.( |$)', node_content): append = True if append: @@ -180,7 +231,9 @@ class Document: sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) for candidate in sorted_candidates[:5]: elem = candidate['elem'] - self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) + self.debug("Top 5 : %6.3f %s" % ( + candidate['content_score'], + describe(elem))) if len(sorted_candidates) == 0: return None @@ -188,7 +241,6 @@ class Document: best_candidate = sorted_candidates[0] return best_candidate - def get_link_density(self, elem): link_length = 0 for i in elem.findall(".//a"): @@ -199,10 +251,10 @@ class Document: return float(link_length) / max(total_length, 1) def score_paragraphs(self, ): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + MIN_LEN = self.options.get( + 'min_text_length', + self.TEXT_LENGTH_THRESHOLD) candidates = {} - #self.debug(str([describe(node) for node in self.tags(self.html, "div")])) - ordered = [] for elem in self.tags(self._html(), "p", "pre", "td"): parent_node = elem.getparent() @@ -213,7 +265,8 @@ class Document: inner_text = clean(elem.text_content() or "") inner_text_len = len(inner_text) - # If this paragraph is less than 25 characters, don't even count it. + # If this paragraph is less than 25 characters + # don't even count it. if inner_text_len < MIN_LEN: continue @@ -222,7 +275,8 @@ class Document: ordered.append(parent_node) if grand_parent_node is not None and grand_parent_node not in candidates: - candidates[grand_parent_node] = self.score_node(grand_parent_node) + candidates[grand_parent_node] = self.score_node( + grand_parent_node) ordered.append(grand_parent_node) content_score = 1 @@ -236,13 +290,18 @@ class Document: if grand_parent_node is not None: candidates[grand_parent_node]['content_score'] += content_score / 2.0 - # Scale the final candidates score based on link density. Good content should have a - # relatively small link density (5% or less) and be mostly unaffected by this operation. + # Scale the final candidates score based on link density. Good content + # should have a relatively small link density (5% or less) and be + # mostly unaffected by this operation. for elem in ordered: candidate = candidates[elem] ld = self.get_link_density(elem) score = candidate['content_score'] - self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) + self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % ( + score, + describe(elem), + ld, + score * (1 - ld))) candidate['content_score'] *= (1 - ld) return candidates @@ -282,8 +341,8 @@ class Document: } def debug(self, *a): - #if self.options['debug']: - logging.debug(*a) + if self.options.get('debug', False): + log.debug(*a) def remove_unlikely_candidates(self): for elem in self.html.iter(): @@ -297,10 +356,14 @@ class Document: def transform_misused_divs_into_paragraphs(self): for elem in self.tags(self.html, 'div'): - # transform
s that do not contain other block elements into

s - #FIXME: The current implementation ignores all descendants that are not direct children of elem - # This results in incorrect results in case there is an buried within an for example - if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): + # transform

s that do not contain other block elements into + #

s + #FIXME: The current implementation ignores all descendants that + # are not direct children of elem + # This results in incorrect results in case there is an + # buried within an for example + if not REGEXES['divToPElementsRe'].search( + unicode(''.join(map(tostring, list(elem))))): #self.debug("Altering %s to p" % (describe(elem))) elem.tag = "p" #print "Fixed element "+describe(elem) @@ -335,7 +398,8 @@ class Document: yield e def sanitize(self, node, candidates): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + MIN_LEN = self.options.get('min_text_length', + self.TEXT_LENGTH_THRESHOLD) for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: header.drop_tree() @@ -362,10 +426,11 @@ class Document: elif el.text_content().count(",") < 10: counts = {} for kind in ['p', 'img', 'li', 'a', 'embed', 'input']: - counts[kind] = len(el.findall('.//%s' %kind)) + counts[kind] = len(el.findall('.//%s' % kind)) counts["li"] -= 100 - content_length = text_length(el) # Count the text length excluding any surrounding whitespace + # Count the text length excluding any surrounding whitespace + content_length = text_length(el) link_density = self.get_link_density(el) parent_node = el.getparent() if parent_node is not None: @@ -397,10 +462,12 @@ class Document: reason = "too short content length %s without a single image" % content_length to_remove = True elif weight < 25 and link_density > 0.2: - reason = "too many links %.3f for its weight %s" % (link_density, weight) + reason = "too many links %.3f for its weight %s" % ( + link_density, weight) to_remove = True elif weight >= 25 and link_density > 0.5: - reason = "too many links %.3f for its weight %s" % (link_density, weight) + reason = "too many links %.3f for its weight %s" % ( + link_density, weight) to_remove = True elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: reason = "s with too short content length, or too many s" @@ -426,7 +493,7 @@ class Document: #find x non empty preceding and succeeding siblings i, j = 0, 0 - x = 1 + x = 1 siblings = [] for sib in el.itersiblings(): #self.debug(sib.text_content()) @@ -445,7 +512,7 @@ class Document: if j == x: break #self.debug(str(siblings)) - if siblings and sum(siblings) > 1000 : + if siblings and sum(siblings) > 1000: to_remove = False self.debug("Allowing %s" % describe(el)) for desnode in self.tags(el, "table", "ul", "div"): @@ -459,7 +526,7 @@ class Document: el.drop_tree() for el in ([node] + [n for n in node.iter()]): - if not (self.options['attributes']): + if not self.options.get('attributes', None): #el.attrib = {} #FIXME:Checkout the effects of disabling this pass @@ -492,17 +559,17 @@ class HashableElement(): def __getattr__(self, tag): return getattr(self.node, tag) + def main(): from optparse import OptionParser parser = OptionParser(usage="%prog: [options] [file]") parser.add_option('-v', '--verbose', action='store_true') - parser.add_option('-u', '--url', help="use URL instead of a local file") + parser.add_option('-u', '--url', default=None, help="use URL instead of a local file") (options, args) = parser.parse_args() if not (len(args) == 1 or options.url): parser.print_help() sys.exit(1) - logging.basicConfig(level=logging.INFO) file = None if options.url: @@ -512,7 +579,9 @@ def main(): file = open(args[0], 'rt') enc = sys.__stdout__.encoding or 'utf-8' try: - print Document(file.read(), debug=options.verbose).summary().encode(enc, 'replace') + print Document(file.read(), + debug=options.verbose, + url=options.url).summary().encode(enc, 'replace') finally: file.close() From 8d3e39f04ed0c6e9401fcfa72f89b40273e66df2 Mon Sep 17 00:00:00 2001 From: Richard Harding Date: Mon, 16 Apr 2012 21:24:33 -0400 Subject: [PATCH 4/4] Update readme --- README | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README b/README index acc96fa..c7087b0 100644 --- a/README +++ b/README @@ -36,5 +36,9 @@ Command-line usage:: Document() kwarg options: - url=xxx will run make_links_absolute() + - attributes: + - debug: output debug messages + - min_text_length: + - retry_length: + - url: will allow adjusting links to be absolute