From 8877754d7e217620b129ff0a70db86ccfa4df781 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Wed, 29 Jun 2011 13:33:53 -0700 Subject: [PATCH 01/11] Add version number to track changes more easily --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 3df9c6d..84aa8b7 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from distutils.core import setup setup( name="lxml-readability", + version="0.1dev", author="Yuri Baburov", author_email="burchik+github@gmail.com", description="python port of arc90's readability bookmarklet", From 7ceb8e6d7b39cb25538143b5bb40af925071eea2 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Wed, 29 Jun 2011 13:50:51 -0700 Subject: [PATCH 02/11] Change to automatically find packages --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 84aa8b7..5f03288 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from distutils.core import setup - +from setuptools import setup, find_packages setup( name="lxml-readability", @@ -11,9 +10,7 @@ setup( long_description=open("README").read(), license="Apache License 2.0", url="http://github.com/buriy/python-readability", - packages=[ - "lxml_readability", - ], + packages=find_packages(), classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", From 33f935e39a143a021b0517777646bd3d4d70435a Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Wed, 29 Jun 2011 13:57:14 -0700 Subject: [PATCH 03/11] Expose Document in readability package --- readability/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readability/__init__.py b/readability/__init__.py index e69de29..8822a51 100644 --- a/readability/__init__.py +++ b/readability/__init__.py @@ -0,0 +1 @@ +from .readability import Document From 01247903b879170b9ad57e53e5e9188ef7cb8e27 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Wed, 29 Jun 2011 14:00:30 -0700 Subject: [PATCH 04/11] Add chardet to installation requirements --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f03288..d4def63 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,13 @@ setup( license="Apache License 2.0", url="http://github.com/buriy/python-readability", packages=find_packages(), + install_requires=[ + "chardet" + ], classifiers=[ "Environment :: Web Environment", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python", - ], + ], ) From ac517834e6cffbc2b66532d3fc31539b96f8dea9 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Thu, 30 Jun 2011 11:04:31 -0700 Subject: [PATCH 05/11] Convert tabs to spaces; put article in body --- readability/readability.py | 947 +++++++++++++++++++------------------ 1 file changed, 475 insertions(+), 472 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index 50e191d..f835dfc 100644 --- a/readability/readability.py +++ b/readability/readability.py @@ -4,6 +4,7 @@ from collections import defaultdict from htmls import build_doc, get_body, get_title, shorten_title from lxml.etree import tostring, tounicode from lxml.html import fragment_fromstring, document_fromstring +from lxml.html import builder as B import logging import re import sys @@ -11,495 +12,497 @@ import sys logging.basicConfig(level=logging.INFO) REGEXES = { - 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), - 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), - 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), - 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), - 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), - #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), - #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), - #'trimRe': re.compile('^\s+|\s+$/'), - #'normalizeRe': re.compile('\s{2,}/'), - #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), - #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), - #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, + 'unlikelyCandidatesRe': re.compile('combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter',re.I), + 'okMaybeItsACandidateRe': re.compile('and|article|body|column|main|shadow',re.I), + 'positiveRe': re.compile('article|body|content|entry|hentry|main|page|pagination|post|text|blog|story',re.I), + 'negativeRe': re.compile('combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget',re.I), + 'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I), + #'replaceBrsRe': re.compile('(]*>[ \n\r\t]*){2,}',re.I), + #'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I), + #'trimRe': re.compile('^\s+|\s+$/'), + #'normalizeRe': re.compile('\s{2,}/'), + #'killBreaksRe': re.compile('((\s| ?)*){1,}/'), + #'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I), + #skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i, } def describe(node, depth=1): - if not hasattr(node, 'tag'): - return "[%s]" % type(node) - name = node.tag - if node.get('id', ''): name += '#'+node.get('id') - if node.get('class', ''): - name += '.' + node.get('class').replace(' ','.') - if name[:4] in ['div#', 'div.']: - name = name[3:] - if depth and node.getparent() is not None: - return name+' - '+describe(node.getparent(), depth-1) - return name + if not hasattr(node, 'tag'): + return "[%s]" % type(node) + name = node.tag + if node.get('id', ''): name += '#'+node.get('id') + if node.get('class', ''): + name += '.' + node.get('class').replace(' ','.') + if name[:4] in ['div#', 'div.']: + name = name[3:] + if depth and node.getparent() is not None: + return name+' - '+describe(node.getparent(), depth-1) + return name def to_int(x): - if not x: return None - x = x.strip() - if x.endswith('px'): - return int(x[:-2]) - if x.endswith('em'): - return int(x[:-2]) * 12 - return int(x) + if not x: return None + x = x.strip() + if x.endswith('px'): + return int(x[:-2]) + if x.endswith('em'): + return int(x[:-2]) * 12 + return int(x) def clean(text): - text = re.sub('\s*\n\s*', '\n', text) - text = re.sub('[ \t]{2,}', ' ', text) - return text.strip() + text = re.sub('\s*\n\s*', '\n', text) + text = re.sub('[ \t]{2,}', ' ', text) + return text.strip() def text_length(i): - return len(clean(i.text_content() or "")) + return len(clean(i.text_content() or "")) class Unparseable(ValueError): - pass + pass class Document: - TEXT_LENGTH_THRESHOLD = 25 - RETRY_LENGTH = 250 - - def __init__(self, input, **options): - self.input = input - self.options = defaultdict(lambda: None) - for k, v in options.items(): - self.options[k] = v - self.html = None - - def _html(self, force=False): - if force or self.html is None: - self.html = self._parse(self.input) - return self.html - - def _parse(self, input): - doc = build_doc(input) - doc = html_cleaner.clean_html(doc) - base_href = self.options['url'] - if base_href: - doc.make_links_absolute(base_href, resolve_base_href=True) - else: - doc.resolve_base_href() - return doc - - def content(self): - return get_body(self._html(True)) - - def title(self): - return get_title(self._html(True)) - - def short_title(self): - return shorten_title(self._html(True)) - - def summary(self): - try: - ruthless = True - while True: - self._html(True) - - for i in self.tags(self.html, 'script', 'style'): - i.drop_tree() - for i in self.tags(self.html, 'body'): - i.set('id', 'readabilityBody') - if ruthless: - self.remove_unlikely_candidates() - self.transform_misused_divs_into_paragraphs() - candidates = self.score_paragraphs() - - best_candidate = self.select_best_candidate(candidates) - if best_candidate: - article = self.get_article(candidates, best_candidate) - else: - if ruthless: - logging.debug("ruthless removal did not work. ") - ruthless = False - self.debug("ended up stripping too much - going for a safer _parse") - # try again - continue - else: - logging.debug("Ruthless and lenient parsing did not work. Returning raw html") - article = self.html.find('body') or self.html - - cleaned_article = self.sanitize(article, candidates) - of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) - if ruthless and not of_acceptable_length: - ruthless = False - continue # try again - else: - return cleaned_article - except StandardError, e: - #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) - logging.exception('error getting summary: ' ) - raise Unparseable(str(e)), None, sys.exc_info()[2] - - def get_article(self, candidates, best_candidate): - # Now that we have the top candidate, look through its siblings for content that might also be related. - # Things like preambles, content split by ads that we removed, etc. - - sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) - output = document_fromstring('
') - best_elem = best_candidate['elem'] - for sibling in best_elem.getparent().getchildren(): - #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text - append = False - if sibling is best_elem: - append = True - sibling_key = sibling #HashableElement(sibling) - if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: - append = True - - if sibling.tag == "p": - link_density = self.get_link_density(sibling) - node_content = sibling.text or "" - node_length = len(node_content) - - if node_length > 80 and link_density < 0.25: - append = True - elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content): - append = True - - if append: - output.append(sibling) - #if output is not None: - # output.append(best_elem) - return output - - def select_best_candidate(self, candidates): - sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) - for candidate in sorted_candidates[:5]: - elem = candidate['elem'] - self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) - - if len(sorted_candidates) == 0: - return None - - best_candidate = sorted_candidates[0] - return best_candidate - - - def get_link_density(self, elem): - link_length = 0 - for i in elem.findall(".//a"): - link_length += text_length(i) - #if len(elem.findall(".//div") or elem.findall(".//p")): - # link_length = link_length - total_length = text_length(elem) - return float(link_length) / max(total_length, 1) - - def score_paragraphs(self, ): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) - candidates = {} - #self.debug(str([describe(node) for node in self.tags(self.html, "div")])) - - ordered = [] - for elem in self.tags(self.html, "p", "pre", "td"): - parent_node = elem.getparent() - if parent_node is None: - continue - grand_parent_node = parent_node.getparent() - - inner_text = clean(elem.text_content() or "") - inner_text_len = len(inner_text) - - # If this paragraph is less than 25 characters, don't even count it. - if inner_text_len < MIN_LEN: - continue - - if parent_node not in candidates: - candidates[parent_node] = self.score_node(parent_node) - ordered.append(parent_node) - - if grand_parent_node is not None and grand_parent_node not in candidates: - candidates[grand_parent_node] = self.score_node(grand_parent_node) - ordered.append(grand_parent_node) - - content_score = 1 - content_score += len(inner_text.split(',')) - content_score += min((inner_text_len / 100), 3) - #if elem not in candidates: - # candidates[elem] = self.score_node(elem) - - #WTF? candidates[elem]['content_score'] += content_score - candidates[parent_node]['content_score'] += content_score - if grand_parent_node is not None: - candidates[grand_parent_node]['content_score'] += content_score / 2.0 - - # Scale the final candidates score based on link density. Good content should have a - # relatively small link density (5% or less) and be mostly unaffected by this operation. - for elem in ordered: - candidate = candidates[elem] - ld = self.get_link_density(elem) - score = candidate['content_score'] - self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) - candidate['content_score'] *= (1 - ld) - - return candidates - - def class_weight(self, e): - weight = 0 - if e.get('class', None): - if REGEXES['negativeRe'].search(e.get('class')): - weight -= 25 - - if REGEXES['positiveRe'].search(e.get('class')): - weight += 25 - - if e.get('id', None): - if REGEXES['negativeRe'].search(e.get('id')): - weight -= 25 - - if REGEXES['positiveRe'].search(e.get('id')): - weight += 25 - - return weight - - def score_node(self, elem): - content_score = self.class_weight(elem) - name = elem.tag.lower() - if name == "div": - content_score += 5 - elif name in ["pre", "td", "blockquote"]: - content_score += 3 - elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: - content_score -= 3 - elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: - content_score -= 5 - return { - 'content_score': content_score, - 'elem': elem - } - - def debug(self, *a): - #if self.options['debug']: - logging.debug(*a) - - def remove_unlikely_candidates(self): - for elem in self.html.iter(): - s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) - #self.debug(s) - if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': - self.debug("Removing unlikely candidate - %s" % describe(elem)) - elem.drop_tree() - - def transform_misused_divs_into_paragraphs(self): - for elem in self.tags(self.html, 'div'): - # transform
s that do not contain other block elements into

s - if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): - #self.debug("Altering %s to p" % (describe(elem))) - elem.tag = "p" - #print "Fixed element "+describe(elem) - - for elem in self.tags(self.html, 'div'): - if elem.text and elem.text.strip(): - p = fragment_fromstring('

') - p.text = elem.text - elem.text = None - elem.insert(0, p) - #print "Appended "+tounicode(p)+" to "+describe(elem) - - for pos, child in reversed(list(enumerate(elem))): - if child.tail and child.tail.strip(): - p = fragment_fromstring('

') - p.text = child.tail - child.tail = None - elem.insert(pos + 1, p) - #print "Inserted "+tounicode(p)+" to "+describe(elem) - if child.tag == 'br': - #print 'Dropped
at '+describe(elem) - child.drop_tree() - - def tags(self, node, *tag_names): - for tag_name in tag_names: - for e in node.findall('.//%s' % tag_name): - yield e - - def reverse_tags(self, node, *tag_names): - for tag_name in tag_names: - for e in reversed(node.findall('.//%s' % tag_name)): - yield e - - def sanitize(self, node, candidates): - MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) - for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): - if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: - header.drop_tree() - - for elem in self.tags(node, "form", "iframe", "textarea"): - elem.drop_tree() - allowed = {} - # Conditionally clean s,
    s, and
    s - for el in self.reverse_tags(node, "table", "ul", "div"): - if el in allowed: - continue - weight = self.class_weight(el) - if el in candidates: - content_score = candidates[el]['content_score'] - #print '!',el, '-> %6.3f' % content_score - else: - content_score = 0 - tag = el.tag - - if weight + content_score < 0: - self.debug("Cleaned %s with score %6.3f and weight %-3s" % - (describe(el), content_score, weight, )) - el.drop_tree() - elif el.text_content().count(",") < 10: - counts = {} - for kind in ['p', 'img', 'li', 'a', 'embed', 'input']: - counts[kind] = len(el.findall('.//%s' %kind)) - counts["li"] -= 100 - - content_length = text_length(el) # Count the text length excluding any surrounding whitespace - link_density = self.get_link_density(el) - parent_node = el.getparent() - if parent_node is not None: - if parent_node in candidates: - content_score = candidates[parent_node]['content_score'] - else: - content_score = 0 - #if parent_node is not None: - #pweight = self.class_weight(parent_node) + content_score - #pname = describe(parent_node) - #else: - #pweight = 0 - #pname = "no parent" - to_remove = False - reason = "" - - #if el.tag == 'div' and counts["img"] >= 1: - # continue - if counts["p"] and counts["img"] > counts["p"]: - reason = "too many images (%s)" % counts["img"] - to_remove = True - elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol": - reason = "more
  • s than

    s" - to_remove = True - elif counts["input"] > (counts["p"] / 3): - reason = "less than 3x

    s than s" - to_remove = True - elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2): - reason = "too short content length %s without a single image" % content_length - to_remove = True - elif weight < 25 and link_density > 0.2: - reason = "too many links %.3f for its weight %s" % (link_density, weight) - to_remove = True - elif weight >= 25 and link_density > 0.5: - reason = "too many links %.3f for its weight %s" % (link_density, weight) - to_remove = True - elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: - reason = "s with too short content length, or too many s" - to_remove = True -# if el.tag == 'div' and counts['img'] >= 1 and to_remove: -# imgs = el.findall('.//img') -# valid_img = False -# self.debug(tounicode(el)) -# for img in imgs: + TEXT_LENGTH_THRESHOLD = 25 + RETRY_LENGTH = 250 + + def __init__(self, input, **options): + self.input = input + self.options = defaultdict(lambda: None) + for k, v in options.items(): + self.options[k] = v + self.html = None + + def _html(self, force=False): + if force or self.html is None: + self.html = self._parse(self.input) + return self.html + + def _parse(self, input): + doc = build_doc(input) + doc = html_cleaner.clean_html(doc) + base_href = self.options['url'] + if base_href: + doc.make_links_absolute(base_href, resolve_base_href=True) + else: + doc.resolve_base_href() + return doc + + def content(self): + return get_body(self._html(True)) + + def title(self): + return get_title(self._html(True)) + + def short_title(self): + return shorten_title(self._html(True)) + + def summary(self): + try: + ruthless = True + while True: + self._html(True) + + for i in self.tags(self.html, 'script', 'style'): + i.drop_tree() + for i in self.tags(self.html, 'body'): + i.set('id', 'readabilityBody') + if ruthless: + self.remove_unlikely_candidates() + self.transform_misused_divs_into_paragraphs() + candidates = self.score_paragraphs() + + best_candidate = self.select_best_candidate(candidates) + if best_candidate: + article = self.get_article(candidates, best_candidate) + else: + if ruthless: + logging.debug("ruthless removal did not work. ") + ruthless = False + self.debug("ended up stripping too much - going for a safer _parse") + # try again + continue + else: + logging.debug("Ruthless and lenient parsing did not work. Returning raw html") + article = self.html.find('body') or self.html + + cleaned_article = self.sanitize(article, candidates) + of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) + if ruthless and not of_acceptable_length: + ruthless = False + continue # try again + else: + return cleaned_article + except StandardError, e: + #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) + logging.exception('error getting summary: ' ) + raise Unparseable(str(e)), None, sys.exc_info()[2] + + def get_article(self, candidates, best_candidate): + # Now that we have the top candidate, look through its siblings for content that might also be related. + # Things like preambles, content split by ads that we removed, etc. + + sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) + body = B.BODY() + html = B.HTML(body) + best_elem = best_candidate['elem'] + for sibling in best_elem.getparent().getchildren(): + #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text + append = False + if sibling is best_elem: + append = True + sibling_key = sibling #HashableElement(sibling) + if sibling_key in candidates and candidates[sibling_key]['content_score'] >= sibling_score_threshold: + append = True + + if sibling.tag == "p": + link_density = self.get_link_density(sibling) + node_content = sibling.text or "" + node_length = len(node_content) + + if node_length > 80 and link_density < 0.25: + append = True + elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content): + append = True + + if append: + body.append(sibling) + + #if body is not None: + # body.append(best_elem) + return html + + def select_best_candidate(self, candidates): + sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) + for candidate in sorted_candidates[:5]: + elem = candidate['elem'] + self.debug("Top 5 : %6.3f %s" % (candidate['content_score'], describe(elem))) + + if len(sorted_candidates) == 0: + return None + + best_candidate = sorted_candidates[0] + return best_candidate + + + def get_link_density(self, elem): + link_length = 0 + for i in elem.findall(".//a"): + link_length += text_length(i) + #if len(elem.findall(".//div") or elem.findall(".//p")): + # link_length = link_length + total_length = text_length(elem) + return float(link_length) / max(total_length, 1) + + def score_paragraphs(self, ): + MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + candidates = {} + #self.debug(str([describe(node) for node in self.tags(self.html, "div")])) + + ordered = [] + for elem in self.tags(self.html, "p", "pre", "td"): + parent_node = elem.getparent() + if parent_node is None: + continue + grand_parent_node = parent_node.getparent() + + inner_text = clean(elem.text_content() or "") + inner_text_len = len(inner_text) + + # If this paragraph is less than 25 characters, don't even count it. + if inner_text_len < MIN_LEN: + continue + + if parent_node not in candidates: + candidates[parent_node] = self.score_node(parent_node) + ordered.append(parent_node) + + if grand_parent_node is not None and grand_parent_node not in candidates: + candidates[grand_parent_node] = self.score_node(grand_parent_node) + ordered.append(grand_parent_node) + + content_score = 1 + content_score += len(inner_text.split(',')) + content_score += min((inner_text_len / 100), 3) + #if elem not in candidates: + # candidates[elem] = self.score_node(elem) + + #WTF? candidates[elem]['content_score'] += content_score + candidates[parent_node]['content_score'] += content_score + if grand_parent_node is not None: + candidates[grand_parent_node]['content_score'] += content_score / 2.0 + + # Scale the final candidates score based on link density. Good content should have a + # relatively small link density (5% or less) and be mostly unaffected by this operation. + for elem in ordered: + candidate = candidates[elem] + ld = self.get_link_density(elem) + score = candidate['content_score'] + self.debug("Candid: %6.3f %s link density %.3f -> %6.3f" % (score, describe(elem), ld, score*(1-ld))) + candidate['content_score'] *= (1 - ld) + + return candidates + + def class_weight(self, e): + weight = 0 + if e.get('class', None): + if REGEXES['negativeRe'].search(e.get('class')): + weight -= 25 + + if REGEXES['positiveRe'].search(e.get('class')): + weight += 25 + + if e.get('id', None): + if REGEXES['negativeRe'].search(e.get('id')): + weight -= 25 + + if REGEXES['positiveRe'].search(e.get('id')): + weight += 25 + + return weight + + def score_node(self, elem): + content_score = self.class_weight(elem) + name = elem.tag.lower() + if name == "div": + content_score += 5 + elif name in ["pre", "td", "blockquote"]: + content_score += 3 + elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form"]: + content_score -= 3 + elif name in ["h1", "h2", "h3", "h4", "h5", "h6", "th"]: + content_score -= 5 + return { + 'content_score': content_score, + 'elem': elem + } + + def debug(self, *a): + #if self.options['debug']: + logging.debug(*a) + + def remove_unlikely_candidates(self): + for elem in self.html.iter(): + s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) + #self.debug(s) + if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': + self.debug("Removing unlikely candidate - %s" % describe(elem)) + elem.drop_tree() + + def transform_misused_divs_into_paragraphs(self): + for elem in self.tags(self.html, 'div'): + # transform

    s that do not contain other block elements into

    s + if not REGEXES['divToPElementsRe'].search(unicode(''.join(map(tostring, list(elem))))): + #self.debug("Altering %s to p" % (describe(elem))) + elem.tag = "p" + #print "Fixed element "+describe(elem) + + for elem in self.tags(self.html, 'div'): + if elem.text and elem.text.strip(): + p = fragment_fromstring('

    ') + p.text = elem.text + elem.text = None + elem.insert(0, p) + #print "Appended "+tounicode(p)+" to "+describe(elem) + + for pos, child in reversed(list(enumerate(elem))): + if child.tail and child.tail.strip(): + p = fragment_fromstring('

    ') + p.text = child.tail + child.tail = None + elem.insert(pos + 1, p) + #print "Inserted "+tounicode(p)+" to "+describe(elem) + if child.tag == 'br': + #print 'Dropped
    at '+describe(elem) + child.drop_tree() + + def tags(self, node, *tag_names): + for tag_name in tag_names: + for e in node.findall('.//%s' % tag_name): + yield e + + def reverse_tags(self, node, *tag_names): + for tag_name in tag_names: + for e in reversed(node.findall('.//%s' % tag_name)): + yield e + + def sanitize(self, node, candidates): + MIN_LEN = self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD) + for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"): + if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33: + header.drop_tree() + + for elem in self.tags(node, "form", "iframe", "textarea"): + elem.drop_tree() + allowed = {} + # Conditionally clean

s,
    s, and
    s + for el in self.reverse_tags(node, "table", "ul", "div"): + if el in allowed: + continue + weight = self.class_weight(el) + if el in candidates: + content_score = candidates[el]['content_score'] + #print '!',el, '-> %6.3f' % content_score + else: + content_score = 0 + tag = el.tag + + if weight + content_score < 0: + self.debug("Cleaned %s with score %6.3f and weight %-3s" % + (describe(el), content_score, weight, )) + el.drop_tree() + elif el.text_content().count(",") < 10: + counts = {} + for kind in ['p', 'img', 'li', 'a', 'embed', 'input']: + counts[kind] = len(el.findall('.//%s' %kind)) + counts["li"] -= 100 + + content_length = text_length(el) # Count the text length excluding any surrounding whitespace + link_density = self.get_link_density(el) + parent_node = el.getparent() + if parent_node is not None: + if parent_node in candidates: + content_score = candidates[parent_node]['content_score'] + else: + content_score = 0 + #if parent_node is not None: + #pweight = self.class_weight(parent_node) + content_score + #pname = describe(parent_node) + #else: + #pweight = 0 + #pname = "no parent" + to_remove = False + reason = "" + + #if el.tag == 'div' and counts["img"] >= 1: + # continue + if counts["p"] and counts["img"] > counts["p"]: + reason = "too many images (%s)" % counts["img"] + to_remove = True + elif counts["li"] > counts["p"] and tag != "ul" and tag != "ol": + reason = "more
  • s than

    s" + to_remove = True + elif counts["input"] > (counts["p"] / 3): + reason = "less than 3x

    s than s" + to_remove = True + elif content_length < (MIN_LEN) and (counts["img"] == 0 or counts["img"] > 2): + reason = "too short content length %s without a single image" % content_length + to_remove = True + elif weight < 25 and link_density > 0.2: + reason = "too many links %.3f for its weight %s" % (link_density, weight) + to_remove = True + elif weight >= 25 and link_density > 0.5: + reason = "too many links %.3f for its weight %s" % (link_density, weight) + to_remove = True + elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1: + reason = "s with too short content length, or too many s" + to_remove = True +# if el.tag == 'div' and counts['img'] >= 1 and to_remove: +# imgs = el.findall('.//img') +# valid_img = False +# self.debug(tounicode(el)) +# for img in imgs: # -# height = img.get('height') -# text_length = img.get('text_length') -# self.debug ("height %s text_length %s" %(repr(height), repr(text_length))) -# if to_int(height) >= 100 or to_int(text_length) >= 100: -# valid_img = True -# self.debug("valid image" + tounicode(img)) -# break -# if valid_img: -# to_remove = False -# self.debug("Allowing %s" %el.text_content()) -# for desnode in self.tags(el, "table", "ul", "div"): -# allowed[desnode] = True - - #find x non empty preceding and succeeding siblings - i, j = 0, 0 - x = 1 - siblings = [] - for sib in el.itersiblings(): - #self.debug(sib.text_content()) - sib_content_length = text_length(sib) - if sib_content_length: - i =+ 1 - siblings.append(sib_content_length) - if i == x: - break - for sib in el.itersiblings(preceding=True): - #self.debug(sib.text_content()) - sib_content_length = text_length(sib) - if sib_content_length: - j =+ 1 - siblings.append(sib_content_length) - if j == x: - break - #self.debug(str(siblings)) - if siblings and sum(siblings) > 1000 : - to_remove = False - self.debug("Allowing %s" % describe(el)) - for desnode in self.tags(el, "table", "ul", "div"): - allowed[desnode] = True - - if to_remove: - self.debug("Cleaned %6.3f %s with weight %s cause it has %s." % - (content_score, describe(el), weight, reason)) - #print tounicode(el) - #self.debug("pname %s pweight %.3f" %(pname, pweight)) - el.drop_tree() - - for el in ([node] + [n for n in node.iter()]): - if not (self.options['attributes']): - #el.attrib = {} #FIXME:Checkout the effects of disabling this - pass - - return clean_attributes(tounicode(node)) - +# height = img.get('height') +# text_length = img.get('text_length') +# self.debug ("height %s text_length %s" %(repr(height), repr(text_length))) +# if to_int(height) >= 100 or to_int(text_length) >= 100: +# valid_img = True +# self.debug("valid image" + tounicode(img)) +# break +# if valid_img: +# to_remove = False +# self.debug("Allowing %s" %el.text_content()) +# for desnode in self.tags(el, "table", "ul", "div"): +# allowed[desnode] = True + + #find x non empty preceding and succeeding siblings + i, j = 0, 0 + x = 1 + siblings = [] + for sib in el.itersiblings(): + #self.debug(sib.text_content()) + sib_content_length = text_length(sib) + if sib_content_length: + i =+ 1 + siblings.append(sib_content_length) + if i == x: + break + for sib in el.itersiblings(preceding=True): + #self.debug(sib.text_content()) + sib_content_length = text_length(sib) + if sib_content_length: + j =+ 1 + siblings.append(sib_content_length) + if j == x: + break + #self.debug(str(siblings)) + if siblings and sum(siblings) > 1000 : + to_remove = False + self.debug("Allowing %s" % describe(el)) + for desnode in self.tags(el, "table", "ul", "div"): + allowed[desnode] = True + + if to_remove: + self.debug("Cleaned %6.3f %s with weight %s cause it has %s." % + (content_score, describe(el), weight, reason)) + #print tounicode(el) + #self.debug("pname %s pweight %.3f" %(pname, pweight)) + el.drop_tree() + + for el in ([node] + [n for n in node.iter()]): + if not (self.options['attributes']): + #el.attrib = {} #FIXME:Checkout the effects of disabling this + pass + + return clean_attributes(tounicode(node)) + class HashableElement(): - def __init__(self, node): - self.node = node - self._path = None - - def _get_path(self): - if self._path is None: - reverse_path = [] - node = self.node - while node is not None: - node_id = (node.tag, tuple(node.attrib.items()), node.text) - reverse_path.append(node_id) - node = node.getparent() - self._path = tuple(reverse_path) - return self._path - path = property(_get_path) - - def __hash__(self): - return hash(self.path) - - def __eq__(self, other): - return self.path == other.path - - def __getattr__(self, tag): - return getattr(self.node, tag) + def __init__(self, node): + self.node = node + self._path = None + + def _get_path(self): + if self._path is None: + reverse_path = [] + node = self.node + while node is not None: + node_id = (node.tag, tuple(node.attrib.items()), node.text) + reverse_path.append(node_id) + node = node.getparent() + self._path = tuple(reverse_path) + return self._path + path = property(_get_path) + + def __hash__(self): + return hash(self.path) + + def __eq__(self, other): + return self.path == other.path + + def __getattr__(self, tag): + return getattr(self.node, tag) def main(): - from optparse import OptionParser - parser = OptionParser(usage="%prog: [options] [file]") - parser.add_option('-v', '--verbose', action='store_true') - parser.add_option('-u', '--url', help="use URL instead of a local file") - (options, args) = parser.parse_args() - - if not (len(args) == 1 or options.url): - parser.print_help() - sys.exit(1) - logging.basicConfig(level=logging.INFO) - - file = None - if options.url: - import urllib - file = urllib.urlopen(options.url) - else: - file = open(args[0]) - try: - print Document(file.read(), debug=options.verbose).summary().encode('ascii','ignore') - finally: - file.close() + from optparse import OptionParser + parser = OptionParser(usage="%prog: [options] [file]") + parser.add_option('-v', '--verbose', action='store_true') + parser.add_option('-u', '--url', help="use URL instead of a local file") + (options, args) = parser.parse_args() + + if not (len(args) == 1 or options.url): + parser.print_help() + sys.exit(1) + logging.basicConfig(level=logging.INFO) + + file = None + if options.url: + import urllib + file = urllib.urlopen(options.url) + else: + file = open(args[0]) + try: + print Document(file.read(), debug=options.verbose).summary().encode('ascii','ignore') + finally: + file.close() if __name__ == '__main__': - main() + main() From 7aac0f085570d516eb87211920fbe38132053333 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Thu, 30 Jun 2011 14:25:19 -0700 Subject: [PATCH 06/11] Return a div fragment instead of a whole HTML page --- readability/readability.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/readability/readability.py b/readability/readability.py index f835dfc..c7ddcd7 100644 --- a/readability/readability.py +++ b/readability/readability.py @@ -140,8 +140,8 @@ class Document: # Things like preambles, content split by ads that we removed, etc. sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2]) - body = B.BODY() - html = B.HTML(body) + article = B.DIV() + article.attrib['id'] = 'article' best_elem = best_candidate['elem'] for sibling in best_elem.getparent().getchildren(): #if isinstance(sibling, NavigableString): continue#in lxml there no concept of simple text @@ -163,11 +163,11 @@ class Document: append = True if append: - body.append(sibling) + article.append(sibling) - #if body is not None: - # body.append(best_elem) - return html + #if article is not None: + # article.append(best_elem) + return article def select_best_candidate(self, candidates): sorted_candidates = sorted(candidates.values(), key=lambda x: x['content_score'], reverse=True) From cdd30f625eaedbaf47e11385666199245f31a309 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Tue, 5 Jul 2011 13:35:36 -0700 Subject: [PATCH 07/11] Return confidence level when retieving summary --- readability/readability.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index c7ddcd7..a6534ed 100644 --- a/readability/readability.py +++ b/readability/readability.py @@ -59,6 +59,17 @@ def text_length(i): class Unparseable(ValueError): pass +class Summary: + ''' + The type of object returned by Document.summary(). This includes the + confidence level we have in our summary. If this is low (<35), our summary + may not be valid, though we did our best. + ''' + + def __init__(self, confidence, html): + self.confidence = confidence + self.html = html + class Document: TEXT_LENGTH_THRESHOLD = 25 RETRY_LENGTH = 250 @@ -111,6 +122,7 @@ class Document: best_candidate = self.select_best_candidate(candidates) if best_candidate: + confidence = best_candidate['content_score'] article = self.get_article(candidates, best_candidate) else: if ruthless: @@ -121,6 +133,7 @@ class Document: continue else: logging.debug("Ruthless and lenient parsing did not work. Returning raw html") + confidence = 0; article = self.html.find('body') or self.html cleaned_article = self.sanitize(article, candidates) @@ -129,7 +142,7 @@ class Document: ruthless = False continue # try again else: - return cleaned_article + return Summary(confidence, cleaned_article) except StandardError, e: #logging.exception('error getting summary: ' + str(traceback.format_exception(*sys.exc_info()))) logging.exception('error getting summary: ' ) From 18fa6b51466a0305779fa20fee6de2e32e3ea2eb Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Tue, 5 Jul 2011 13:36:58 -0700 Subject: [PATCH 08/11] Bump version number for external use --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d4def63..bb2b510 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name="lxml-readability", - version="0.1dev", + version="0.2dev", author="Yuri Baburov", author_email="burchik+github@gmail.com", description="python port of arc90's readability bookmarklet", From cba19f209bad8a58bbc502b8c8ea7f5cdc9f6b6c Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Tue, 5 Jul 2011 17:17:38 -0700 Subject: [PATCH 09/11] Fix issue with trying to drop root node remove_unlikely_candidates would try to drop_tree the root node if it deemed it an unlikely candidate. This prevents that from happening. --- readability/readability.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/readability/readability.py b/readability/readability.py index a6534ed..75a1be0 100644 --- a/readability/readability.py +++ b/readability/readability.py @@ -295,7 +295,11 @@ class Document: for elem in self.html.iter(): s = "%s %s" % (elem.get('class', ''), elem.get('id', '')) #self.debug(s) - if REGEXES['unlikelyCandidatesRe'].search(s) and (not REGEXES['okMaybeItsACandidateRe'].search(s)) and elem.tag != 'body': + if (REGEXES['unlikelyCandidatesRe'].search(s) and + (not REGEXES['okMaybeItsACandidateRe'].search(s)) and + elem.tag != 'body' and + elem.getparent() is not None + ): self.debug("Removing unlikely candidate - %s" % describe(elem)) elem.drop_tree() From 82eabfc6b1dba0ea2708ae29ea19c676691ecb3f Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Tue, 5 Jul 2011 17:23:05 -0700 Subject: [PATCH 10/11] Bump version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bb2b510..7fe8cc0 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name="lxml-readability", - version="0.2dev", + version="0.3dev", author="Yuri Baburov", author_email="burchik+github@gmail.com", description="python port of arc90's readability bookmarklet", From cc0af7a105eee6f4e8ffbf6debf7fad7fd37e559 Mon Sep 17 00:00:00 2001 From: Jerry Charumilind Date: Fri, 8 Jul 2011 15:55:30 -0700 Subject: [PATCH 11/11] Add beginnings of regression tests --- .gitignore | 1 + readability/readability.py | 5 +- test.py | 240 +++++++++++++++++++++++++++++++++++++ test_output/.gitignore | 2 + 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 test.py create mode 100644 test_output/.gitignore diff --git a/.gitignore b/.gitignore index 0d20b64..e35672b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +env diff --git a/readability/readability.py b/readability/readability.py index 75a1be0..9826210 100644 --- a/readability/readability.py +++ b/readability/readability.py @@ -136,7 +136,10 @@ class Document: confidence = 0; article = self.html.find('body') or self.html - cleaned_article = self.sanitize(article, candidates) + unicode_cleaned_article = self.sanitize(article, candidates) + cleaned_doc = fragment_fromstring(unicode_cleaned_article) + cleaned_article = tostring(cleaned_doc) + of_acceptable_length = len(cleaned_article or '') >= (self.options['retry_length'] or self.RETRY_LENGTH) if ruthless and not of_acceptable_length: ruthless = False diff --git a/test.py b/test.py new file mode 100644 index 0000000..17aaf9e --- /dev/null +++ b/test.py @@ -0,0 +1,240 @@ +from lxml.html import builder as B +import lxml.html +import lxml.html.diff +import os +import os.path +import readability +import sys +import unittest +import yaml + +ORIGINAL_SUFFIX = '-orig.html' +READABLE_SUFFIX = '-rdbl.html' +RESULT_SUFFIX = '-result.html' + +TEST_DATA_PATH = 'test_data' +TEST_OUTPUT_PATH = 'test_output' + +class ReadabilityTest: + + def __init__(self, dir_path, name, orig_path, rdbl_path): + self.dir_path = dir_path + self.name = name + self.orig_path = orig_path + self.rdbl_path = rdbl_path + +class ReadabilityTestData: + + def __init__(self, test, orig_html, rdbl_html): + self.test = test + self.orig_html = orig_html + self.rdbl_html = rdbl_html + +class ReadabilityTestResult: + + def __init__(self, test_data, result_html, diff_html): + self.test_data = test_data + self.result_html = result_html + self.diff_html = diff_html + +def strip_with_suffix(suffix, files): + filtered = [x for x in files if x.endswith(suffix)] + stripped = [x.replace(suffix, '') for x in filtered] + return set(stripped) + +def check_missing(lhs, rhs, rhs_description): + only_lhs = lhs.difference(rhs) + if len(only_lhs) != 0: + is_are = 'is' if len(only_lhs) == 1 else 'are' + file_files = 'file' if len(only_lhs) == 1 else 'files' + s = ('(%s) %s missing %s %s' % + (', '.join(only_lhs), is_are, rhs_description, file_files) + ) + raise Exception(s) + +def resolve_test_names(files): + orig_names = strip_with_suffix(ORIGINAL_SUFFIX, files) + rdbl_names = strip_with_suffix(READABLE_SUFFIX, files) + check_missing(orig_names, rdbl_names, READABLE_SUFFIX) + check_missing(rdbl_names, orig_names, ORIGINAL_SUFFIX) + return orig_names + +def make_path(dir_path, name, suffix): + return os.path.join(dir_path, ''.join([name, suffix])) + +def make_readability_test(dir_path, name): + return ReadabilityTest( + dir_path, + name, + make_path(dir_path, name, ORIGINAL_SUFFIX), + make_path(dir_path, name, READABLE_SUFFIX) + ) + +def load_test_data(test): + orig = open(test.orig_path, 'r').read() + rdbl = open(test.rdbl_path, 'r').read() + return ReadabilityTestData(test, orig, rdbl) + +def load_readability_tests(dir_path, files): + names = resolve_test_names(files) + return [make_readability_test(dir_path, name) for name in names] + +def execute_test(test_data): + doc = readability.Document(test_data.orig_html) + summary = doc.summary() + benchmark_doc = (test_data.rdbl_html, 'benchmark') + result_doc = (summary.html, 'result') + # diff = lxml.html.diff.html_annotate([benchmark_doc, result_doc]) + diff = lxml.html.diff.htmldiff(test_data.rdbl_html, summary.html) + # diff = test_data.orig_html + return ReadabilityTestResult(test_data, summary.html, diff) + +DIFF_CSS = ''' +#article { + margin: 0 auto; + max-width: 705px; + min-width: 225px; + font-family: Georgia, 'Times New Roman', serif; + font-size: 19px; + line-height: 29px; +} + +#article p { + font-size: 19px; + line-height: 29px; + margin: 19px 0px 19px 0px; +} + +ins { + background-color: #C6F7C3; + text-decoration: none; +} + +ins img { + border-width: 3px; + border-style: dotted; + border-color: #51B548; +} + +del { + background-color: #F7C3C3; + text-decoration: none; +} + +del img { + border-width: 3px; + border-style: dotted; + border-color: #D12626; +} +''' + +def add_diff_css(doc): + style = B.STYLE(DIFF_CSS, type = 'text/css') + head = B.HEAD(style) + doc.insert(0, head) + +def write_result(output_dir_path, result): + doc = lxml.html.document_fromstring(result.diff_html) + add_diff_css(doc) + output_file = ''.join([result.test_data.test.name, RESULT_SUFFIX]) + output_path = os.path.join(output_dir_path, output_file) + with open(output_path, 'w') as f: + f.write(lxml.html.tostring(doc)) + +def run_readability_tests(): + files = os.listdir(TEST_DATA_PATH) + tests = load_readability_tests(TEST_DATA_PATH, files) + for test in tests: + test_data = load_test_data(test) + result = execute_test(test_data) + write_result(TEST_OUTPUT_PATH, result) + +class TestStripWithSuffix(unittest.TestCase): + + def test_no_files(self): + expected = set() + actual = strip_with_suffix('.test', []) + self.assertEqual(expected, actual) + + def test_files(self): + expected = {'foo', 'bar'} + actual = strip_with_suffix('.test', ['foo.test', 'bar.test']) + self.assertEqual(expected, actual) + + def test_extra_files(self): + expected = {'foo', 'bar'} + actual = strip_with_suffix('.test', ['foo.test', 'bar.test', 'extra']) + self.assertEqual(expected, actual) + +class TestResolveTestNames(unittest.TestCase): + + def test_no_files(self): + expected = set() + actual = resolve_test_names([]) + self.assertEqual(expected, actual) + + def test_files(self): + expected = {'foo', 'bar'} + files = [ + 'foo-orig.html', + 'foo-rdbl.html', + 'bar-orig.html', + 'bar-rdbl.html' + ] + actual = resolve_test_names(files) + self.assertEqual(expected, actual) + + def test_missing_rdbl(self): + files = [ + 'foo-orig.html', + 'foo-rdbl.html', + 'bar-orig.html' + ] + with self.assertRaisesRegexp( + Exception, + r'\(bar\) is missing -rdbl.html file' + ): + resolve_test_names(files) + + def test_missing_multiple_rdbl(self): + files = [ + 'foo-orig.html', + 'bar-orig.html' + ] + with self.assertRaisesRegexp( + Exception, + r'\(foo, bar\) are missing -rdbl.html files' + ): + resolve_test_names(files) + + def test_missing_orig(self): + files = [ + 'foo-orig.html', + 'foo-rdbl.html', + 'bar-rdbl.html' + ] + with self.assertRaisesRegexp( + Exception, + r'\(bar\) is missing -orig.html file' + ): + resolve_test_names(files) + + def test_missing_multiple_orig(self): + files = [ + 'foo-rdbl.html', + 'bar-rdbl.html' + ] + with self.assertRaisesRegexp( + Exception, + r'\(foo, bar\) are missing -orig.html files' + ): + resolve_test_names(files) + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == 'unittest': + del sys.argv[1] + return unittest.main() + run_readability_tests() + +if __name__ == '__main__': + main() diff --git a/test_output/.gitignore b/test_output/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/test_output/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore