From 5b3f050fd6f30f6b7bd945f6f4dc2f6811830c60 Mon Sep 17 00:00:00 2001
From: "Chakib (Spike) Benziane" <spykspygel@gmail.com>
Date: Sat, 14 May 2011 14:57:11 +0200
Subject: [PATCH] initial commit

---
 .scrapy/scrapy.db                        | Bin 0 -> 4096 bytes
 PyGoogleSearch/__init__.py               |   0
 PyGoogleSearch/__init__.pyc              | Bin 0 -> 164 bytes
 PyGoogleSearch/items.py                  |  13 +++++++
 PyGoogleSearch/items.pyc                 | Bin 0 -> 522 bytes
 PyGoogleSearch/pipelines.py              |   8 +++++
 PyGoogleSearch/settings.py               |  16 +++++++++
 PyGoogleSearch/settings.pyc              | Bin 0 -> 438 bytes
 PyGoogleSearch/spiders/__init__.py       |   8 +++++
 PyGoogleSearch/spiders/__init__.pyc      | Bin 0 -> 172 bytes
 PyGoogleSearch/spiders/google_spider.py  |  40 +++++++++++++++++++++
 PyGoogleSearch/spiders/google_spider.pyc | Bin 0 -> 1713 bytes
 README                                   |   8 +++++
 pygs                                     |  42 +++++++++++++++++++++++
 scrapy.cfg                               |  11 ++++++
 15 files changed, 146 insertions(+)
 create mode 100644 .scrapy/scrapy.db
 create mode 100755 PyGoogleSearch/__init__.py
 create mode 100644 PyGoogleSearch/__init__.pyc
 create mode 100644 PyGoogleSearch/items.py
 create mode 100644 PyGoogleSearch/items.pyc
 create mode 100644 PyGoogleSearch/pipelines.py
 create mode 100644 PyGoogleSearch/settings.py
 create mode 100644 PyGoogleSearch/settings.pyc
 create mode 100755 PyGoogleSearch/spiders/__init__.py
 create mode 100644 PyGoogleSearch/spiders/__init__.pyc
 create mode 100644 PyGoogleSearch/spiders/google_spider.py
 create mode 100644 PyGoogleSearch/spiders/google_spider.pyc
 create mode 100644 README
 create mode 100755 pygs
 create mode 100644 scrapy.cfg

diff --git a/.scrapy/scrapy.db b/.scrapy/scrapy.db
new file mode 100644
index 0000000000000000000000000000000000000000..423a5b96f570a538131dd8b4c89711087cd0af89
GIT binary patch
literal 4096
zcmeH|%}T^D5XY0ItRU>Ur@~woR!~It1zeZj_M-M)QadiecDtLV(jGkce7=RRp$F5|
zK<fjP9%dlS2Vwr<Hw<~a-ztOfT<Ss^STUb*&Tau1W1c;}J<htEyX0ToD-S}CU44dg
z@y&$y#Y8PWtRo`=Qy|cYvtY5{?<*tI9IJ!MP$$P4>p|>q9-?&|LA<`(M$o?nOO=7z
z8{DCWgI0yqExcfR*(Ie`rUi{M|1A|*RdR=r=4E;j^H~sv{Ou(CRN5Aeshsx)g02cn
z+ZnoxL#bDp*MD8Fy;-mcIaB)#o9dX`y-s9p%2S@WM#+tfgdKuE?<UdT@+nM)-arJ1
fz$gUHI~CghQCO1*5tteQ`v0dk5qb|17>~dYq=97&

literal 0
HcmV?d00001

diff --git a/PyGoogleSearch/__init__.py b/PyGoogleSearch/__init__.py
new file mode 100755
index 0000000..e69de29
diff --git a/PyGoogleSearch/__init__.pyc b/PyGoogleSearch/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b3b895c3753db73dc8e2279c1cce8ffe8ece780f
GIT binary patch
literal 164
zcmcckiI<D9J<T_n0SXv_v;z<qvjB+{28Lh_kcgiKkYGR~ihY4%`WgATsrtnQnc1oO
z1x5K;smUe9`o)QPDM|Si`o+mbi3OGV0hR9g`RO^S!KsNw$r<R}`1s7c%#!$cy@JXT
V4xouPx%nxjIjMFady9dX0RZ}UC>8(!

literal 0
HcmV?d00001

diff --git a/PyGoogleSearch/items.py b/PyGoogleSearch/items.py
new file mode 100644
index 0000000..791a003
--- /dev/null
+++ b/PyGoogleSearch/items.py
@@ -0,0 +1,13 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/topics/items.html
+
+from scrapy.item import Item, Field
+
+class PyGSItem(Item):
+    # define the fields for your item here like:
+    # name = Field()
+    title = Field()
+    link = Field()
+    desc = Field()
diff --git a/PyGoogleSearch/items.pyc b/PyGoogleSearch/items.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2a6f006a7b6d49ca172372d5fc80dfeee394d7e5
GIT binary patch
literal 522
zcmcIgJ5Izf5FI<&6}t%0aRHiCYYu=Aq5y#eLP<p$E5rk9;(QbviPUf~uE15e0L*w7
z3VJMi{N`!==6OCo_m;5#eqF<MrF5Ut@kbO4Z2=8{6Hp3p5=sfeNI(G(0!#vL0IvXd
zWdbIJBAd}8`$a30gaV6Q25bZK=N6p_JA`hhi-PIjrh>>bI{uIX0{246!lb(e=?|De
zUSw1T*_=Yu)4Ho#)H*DQ(Yo!;v!gU;+1NVR;4r#zwre6V=<AIP6+uM)ijZ3OEh<0Q
z2Gub3Pgn=9eASuz{zdtEtcFRE|Gs~8SYS2Q>%Xa`pZV1=<varZu4kk3WFZ#fCgYKW
fU1p=!imP)<A-auk{?}HV*zsmYC+y6#&ja}lpObVs

literal 0
HcmV?d00001

diff --git a/PyGoogleSearch/pipelines.py b/PyGoogleSearch/pipelines.py
new file mode 100644
index 0000000..93dddd9
--- /dev/null
+++ b/PyGoogleSearch/pipelines.py
@@ -0,0 +1,8 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/topics/item-pipeline.html
+
+class PygooglesearchPipeline(object):
+    def process_item(self, item, spider):
+        return item
diff --git a/PyGoogleSearch/settings.py b/PyGoogleSearch/settings.py
new file mode 100644
index 0000000..6f8467d
--- /dev/null
+++ b/PyGoogleSearch/settings.py
@@ -0,0 +1,16 @@
+# Scrapy settings for PyGoogleSearch project
+#
+# For simplicity, this file contains only the most important settings by
+# default. All the other settings are documented here:
+#
+#     http://doc.scrapy.org/topics/settings.html
+#
+
+BOT_NAME = 'PyGoogleSearch'
+BOT_VERSION = '1.0'
+
+SPIDER_MODULES = ['PyGoogleSearch.spiders']
+NEWSPIDER_MODULE = 'PyGoogleSearch.spiders'
+DEFAULT_ITEM_CLASS = 'PyGoogleSearch.items.PygooglesearchItem'
+USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
+
diff --git a/PyGoogleSearch/settings.pyc b/PyGoogleSearch/settings.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5c151f92e6a482713574f12a09eebd56a1a5d621
GIT binary patch
literal 438
zcmZWl%}&BV5T3R`1cNaicmpD*t{%J@W0i$8X$xUnO?qjtyA~8^(^(VycD{lKU&JSH
ziX;ZvWWSm3o1dN8&+l&MtNY=nusasnTe^5fAOT`R17ZWD2^l$N?Zg&H3*rXI1}Fe!
zz_SUxxCyceV$h;WeL^s;#(AEtlu+|BU0~h%(!RjnLCD5pDOHJhei&M6wMKhdW!qM`
zZROTllXCmW`WXDactree5+;G;bA7xc4}2!vNw6J9Ox+=$CH`a>d0gmIN(uPmo~qAC
z9P*J9d12y)+)wU3M+n^^ITBRtjCl|ah`}GAPn0e4wPLi#t71i&zo=9r!}&%&<?jsB
pa$Zzy|3CgyR9Y`L8QMjq>s_wbxqMrxn>P7a5707NX4@b%egJvDYzqJY

literal 0
HcmV?d00001

diff --git a/PyGoogleSearch/spiders/__init__.py b/PyGoogleSearch/spiders/__init__.py
new file mode 100755
index 0000000..c930799
--- /dev/null
+++ b/PyGoogleSearch/spiders/__init__.py
@@ -0,0 +1,8 @@
+# This package will contain the spiders of your Scrapy project
+#
+# To create the first spider for your project use this command:
+#
+#   scrapy genspider myspider myspider-domain.com
+#
+# For more info see:
+# http://doc.scrapy.org/topics/spiders.html
diff --git a/PyGoogleSearch/spiders/__init__.pyc b/PyGoogleSearch/spiders/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..01efa2da1aee368ba8a935a29ac34e753d205d12
GIT binary patch
literal 172
zcmcckiI<D9J<T_n0SXv_v;z<qvjB+{28Lh_kcgiKkYGR~ibH{7`WgATsrtnQnc1oO
z1x5K;smUe9`o)QPDM|Si`o+mbi3OGV0hR9g`RO^S!KsNw$r<Qepx%_!qGJ8{_{_Y_
clK6PNg31yOpvgA5`6;D2sdgayi-DK{0CZm~M*si-

literal 0
HcmV?d00001

diff --git a/PyGoogleSearch/spiders/google_spider.py b/PyGoogleSearch/spiders/google_spider.py
new file mode 100644
index 0000000..371ff59
--- /dev/null
+++ b/PyGoogleSearch/spiders/google_spider.py
@@ -0,0 +1,40 @@
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.selector import HtmlXPathSelector
+import time
+
+from PyGoogleSearch.items import PyGSItem
+import sys
+
+class PyGSSpider(CrawlSpider):
+    '''
+    docstring for PyGSSpider
+    '''
+    name = 'google.com'
+    allowed_domains = ['google.com']
+    #start_urls = [
+    #    'http://www.google.fr/search?sclient=psy&hl=fr&source=hp&q=sexe&btnG=Rechercher'
+    #    ]
+    rules = (
+        Rule(SgmlLinkExtractor(restrict_xpaths='//a[@id="pnnext"]',
+        ), 
+        callback='parse_item',
+        follow=True),
+        )
+
+    def parse_item(self, response):
+        time.sleep(3)
+        hxs = HtmlXPathSelector(response)
+        sites = hxs.select('//div[@id="ires"]/ol/li')
+        items = []
+        for site in sites:
+            item = PyGSItem()
+            
+            item['title'] = site.select('h3[@class="r"]/a/text() | \
+            h3[@class="r"]/a/em/text()').extract()
+            item['desc'] = site.select('div/text()').extract()
+            item['link'] = site.select('h3[@class="r"]/a/@href').extract()
+            items.append(item)
+        return items
+
+        
diff --git a/PyGoogleSearch/spiders/google_spider.pyc b/PyGoogleSearch/spiders/google_spider.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..829568ac28ae241a738820a9f600298434ae226e
GIT binary patch
literal 1713
zcmcgs&5qkP5FW~symoe*<_9T~)50h9VjY6slEO$@<OgjK$OXDpun@EoZL6k4MP#yG
zz&@pq(ih1i<W>3t?KhOOo2ETSGRGumH01ZqM}GM0Xnb<={^x@B&j|lBG_!#z@e{Nr
zQi4)J%A?dH!LuH%2c&#T{eC>4bwDblH0;Mdt%sBjU50?x2c$-njxK#;O2<@&Bn2%5
zl|%Z2=pE5-=`|{Snnwp%y!snY<e_J;!94Erb2V?PQg+tIIC-OFB-nccX1=V|PgS%2
zabvr@u)4c)J+-wud70bQTq?<~7rl?yczN+;{>;kSj`xFpflD?*Og}<1&tW7n2|y1L
zKubDcyF|w)8ITN_m2v<a0vO7`;TzIB;Q?ZLuty0Jk{+L#Am{iPPpJ!IyQ*0pFLZaz
zd+N90oQAiy)OxApw9vJ^j$tP`zOdP*1ukQlfjG`z&8qU@q-`3xv6EBF0E=9yv%FY4
zfNkCxnN^IJfi1LB`n+I<hiFXdV>I(8j0OL|DzJSL)}yWK?a=Y40&tHVy8$hUc0O%1
zZ3pzj>5_H<p*kq#E~IVXLcxfjD&Wa2IP&Si-hru49pR;eN&DoB12}xDkT7J2c>Nvy
ziivr|T77}tV9(#c#Bo`@?K!Bx(Bw4MDpr-5!MD2qYF4P+n1_=Njy$&DDEjtz%luzH
z|A~_IKF|h8Vk@hps|ObP=wRAPX$o`a|CX7pI=OIqRZwTdJiC#tYUy$sC1sm123Hf~
zG@5Wy%fZ+s>)HA8wv|nJl>@O0S7d-~bz`LEG^>s26Qj<3j|*<xss>%x;i(A0<cvwt
zi&wgqv1zNdjN4BCj-z5?lQ-p=-o&Qp^7bNzG@TIVGVhAj=R<sp!)fAPtXV&twigKt
z@h+OV2SegW9E$5A6hjKd$P2|SaTo6m5wX%8pe$?hT4tG3ZI;!##9_yf8Obsn7xbAT
zj&cVAD%a<-gsaZ0#w2_%B<||quwS}2J=q=77`L4(@Lz-hRu8Myd<7E@LZBZ7H-Y_d
z5FKL5vUqxuamX7S?z8Eh&IBHEfGl4ap9Qh1_8QThnHpS!J#M}x_OZRDbmvr-9Km64
p!~DCLQ>T(%c<w}9n1pxzWr=dtf6&+adoH43cJ7F;@g92I{sAjFkWl~t

literal 0
HcmV?d00001

diff --git a/README b/README
new file mode 100644
index 0000000..b2b71a0
--- /dev/null
+++ b/README
@@ -0,0 +1,8 @@
+This project is released to the public domain.
+It is based on Scrapy open source screen scrapping.
+contact: spike@boxls.com
+
+PyGoogleSeach is a simple Google Data crawler. I created a simple command line
+interface to make queries, it exports the results in json format to the file
+data.json.
+
diff --git a/pygs b/pygs
new file mode 100755
index 0000000..9601bc7
--- /dev/null
+++ b/pygs
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+import subprocess
+from optparse import OptionParser
+from urllib import quote
+import sys
+
+__VERSION__ = 0.1
+SEARCH_URL ='''\
+http://www.google.com/search?sclient=psy&hl=en&site=&source=hp&q=foo&btnG=Google+Search\
+'''
+SCRAPY_PARAMS = ['--set', 'FEED_URI=data.json', '--set', 'FEED_FORMAT=json']
+
+
+if __name__ == '__main__':
+    usage = 'usage: %prog query [OPTION...]'
+    op = OptionParser(usage, version='%%prog %s' % __VERSION__)
+    opts, args = op.parse_args()
+    #try:
+    if len(args) != 1:
+        print 'Bad arguments: ' 
+        op.print_usage()
+    else:
+        scrapy = 'scrapy'
+        scrapy_cmd = 'crawl'
+        query = args[0]
+        url_query = quote(query)
+        search = SEARCH_URL.replace('foo', url_query)
+        print url_query
+        print 'Launching scrapy with paramters <%s> <%s>' % (scrapy,
+        search)
+        p = subprocess.Popen([
+                            scrapy, scrapy_cmd, 
+                            search, '--set',
+                            'FEED_URI=data.json',
+                            '--set',
+                            'FEED_FORMAT=JSON'
+                            ],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        output = p.communicate()[0]
+        print output
diff --git a/scrapy.cfg b/scrapy.cfg
new file mode 100644
index 0000000..de5a1a7
--- /dev/null
+++ b/scrapy.cfg
@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# http://doc.scrapy.org/topics/scrapyd.html
+
+[settings]
+default = PyGoogleSearch.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = PyGoogleSearch