From 54f6a6323056baa0d6b6c77c9506b6ac517d2bc9 Mon Sep 17 00:00:00 2001 From: bendtherules Date: Sun, 11 May 2014 19:55:41 +0530 Subject: [PATCH] max_pages and SAVED_JSON options added --- Readme.md | 6 ++-- output.json | 50 +++++++++++++++++++++++++++++ try_scrapy/pipelines.py | 4 ++- try_scrapy/settings.py | 2 ++ try_scrapy/spiders/test_spider.py | 8 +++-- try_scrapy/spiders/test_spider.pyc | Bin 2261 -> 2508 bytes 6 files changed, 62 insertions(+), 8 deletions(-) diff --git a/Readme.md b/Readme.md index b0f4010..e66fa14 100644 --- a/Readme.md +++ b/Readme.md @@ -1,7 +1,5 @@ Usage: - scrapy crawl get_lynda + scrapy crawl get_lynda [-a max_pages=10] -For saving to json: - - scrapy crawl get_lynda -o output.json -t json +Saves to "output.json" by default ( Configurable as SAVED_JSON in settings.py) diff --git a/output.json b/output.json index c5941fb..5ae48f3 100644 --- a/output.json +++ b/output.json @@ -199,4 +199,54 @@ {"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Direct Mail Strategy"}, {"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Building Web Services with Java EE"}, {"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Web Design Fundamentals"}, +{"course_link": "/Illustrator-tutorials/Type-Project-Victorian-Ad/155652-2.html", "course_title": "Type Project: Victorian Ad"}, +{"course_link": "/Illustrator-tutorials/Type-Project-Victorian-Ad/155652-2.html", "course_title": "Mograph Techniques: Dynamic Falling Objects in CINEMA 4D"}, +{"course_link": "/CINEMA-4D-tutorials/Mograph-Techniques-Dynamic-Falling-Objects-CINEMA-4D/137890-2.html", "course_title": "XML Essential Training"}, +{"course_link": "/CINEMA-4D-tutorials/Mograph-Techniques-Dynamic-Falling-Objects-CINEMA-4D/137890-2.html", "course_title": "SEO: Keyword Strategy in Depth"}, +{"course_link": "/XML-tutorials/XML-Essential-Training/145930-2.html", "course_title": "Making Beats in Logic Pro X"}, +{"course_link": "/XML-tutorials/XML-Essential-Training/145930-2.html", "course_title": "Dreamweaver CC and WordPress 3.8: Core Concepts"}, +{"course_link": "/Business-Online-Marketing-tutorials/SEO-Keyword-Strategy-Depth/147030-2.html", "course_title": "Creating a PDF Presentation"}, +{"course_link": "/Business-Online-Marketing-tutorials/SEO-Keyword-Strategy-Depth/147030-2.html", "course_title": "Up and Running with Evernote for Mac"}, +{"course_link": "/Logic-Pro-tutorials/Making-Beats-Logic-Pro-X/151550-2.html", "course_title": "Embracing Change"}, +{"course_link": "/Logic-Pro-tutorials/Making-Beats-Logic-Pro-X/151550-2.html", "course_title": "After Effects Guru: Working with Vectors"}, +{"course_link": "/Dreamweaver-tutorials/Dreamweaver-CC-WordPress-38-Core-Concepts/142034-2.html", "course_title": "Uploading Files Securely with PHP"}, +{"course_link": "/Dreamweaver-tutorials/Dreamweaver-CC-WordPress-38-Core-Concepts/142034-2.html", "course_title": "Colorizing Black-and-White Photographs with Photoshop"}, +{"course_link": "/InDesign-tutorials/Creating-PDF-Presentation/144077-2.html", "course_title": "Up and Running with 3D Printing"}, +{"course_link": "/InDesign-tutorials/Creating-PDF-Presentation/144077-2.html", "course_title": "GarageBand Essential Training"}, +{"course_link": "/Evernote-tutorials/Up-Running-Evernote-Mac/157839-2.html", "course_title": "The Creative Spark: Alberto Scirocco and the Motion Graphics of leftchannel"}, +{"course_link": "/Evernote-tutorials/Up-Running-Evernote-Mac/157839-2.html", "course_title": "Travel Photography: Geologging and Journaling on the Road"}, +{"course_link": "/Business-Business-Skills-tutorials/Embracing-Change/150003-2.html", "course_title": "WordPress Ecommerce: Integrating Shopify"}, +{"course_link": "/Business-Business-Skills-tutorials/Embracing-Change/150003-2.html", "course_title": "UX Design Techniques: Creating Scenarios and Storyboards"}, +{"course_link": "/After-Effects-tutorials/After-Effects-Guru-Working-Vectors/160069-2.html", "course_title": "Delivering Employee Feedback"}, +{"course_link": "/After-Effects-tutorials/After-Effects-Guru-Working-Vectors/160069-2.html", "course_title": "Edge Animate Essential Training"}, +{"course_link": "/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Introducing Photoshop: Photography"}, +{"course_link": "/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Up and Running with Chrome OS"}, +{"course_link": "/Photoshop-tutorials/Colorizing-Black-White-Photographs-Photoshop/133329-2.html", "course_title": "Getting Started with After Effects Expressions"}, +{"course_link": "/Photoshop-tutorials/Colorizing-Black-White-Photographs-Photoshop/133329-2.html", "course_title": "Foundations of Programming: Web Security"}, +{"course_link": "/3D-Animation-Prototyping-tutorials/Up-Running-3D-Printing/151814-2.html", "course_title": "Google Sites Essential Training"}, +{"course_link": "/3D-Animation-Prototyping-tutorials/Up-Running-3D-Printing/151814-2.html", "course_title": "Entertainment Creation Suite: Getting Started with Toxik"}, +{"course_link": "/GarageBand-tutorials/GarageBand-Essential-Training/156620-2.html", "course_title": "Artist at Work: From Sketch to Finished Vector Illustration"}, +{"course_link": "/GarageBand-tutorials/GarageBand-Essential-Training/156620-2.html", "course_title": "VFX Techniques: Space Scene 02: Compositing in After Effects"}, +{"course_link": "/Video-Motion-Graphics-tutorials/Creative-Spark-Alberto-Scirocco-Motion-Graphics-leftchannel/153774-2.html", "course_title": "Building an Online Community"}, +{"course_link": "/Video-Motion-Graphics-tutorials/Creative-Spark-Alberto-Scirocco-Motion-Graphics-leftchannel/153774-2.html", "course_title": "Cinematography in Maya"}, +{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Geologging-Journaling-Road/149290-2.html", "course_title": "The Art of Photoshop Compositing"}, +{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Geologging-Journaling-Road/149290-2.html", "course_title": "Up and Running with Lectora Inspire"}, +{"course_link": "/WordPress-tutorials/WordPress-Ecommerce-Integrating-Shopify/154413-2.html", "course_title": "UX Design Techniques: Ideation"}, +{"course_link": "/WordPress-tutorials/WordPress-Ecommerce-Integrating-Shopify/154413-2.html", "course_title": "Up and Running with Promethean ActivInspire"}, +{"course_link": "/Web-User-Experience-tutorials/UX-Design-Techniques-Creating-Scenarios-Storyboards/148581-2.html", "course_title": "Artist at Work: Tertiary Colors"}, +{"course_link": "/Web-User-Experience-tutorials/UX-Design-Techniques-Creating-Scenarios-Storyboards/148581-2.html", "course_title": "Excel 2013: Macros in Depth"}, +{"course_link": "/Business-Business-Skills-tutorials/Delivering-Employee-Feedback/149836-2.html", "course_title": "Primitive and Spline Modeling in CINEMA 4D"}, +{"course_link": "/Business-Business-Skills-tutorials/Delivering-Employee-Feedback/149836-2.html", "course_title": "InDesign Insider Training: Data Merge and Database Publishing"}, +{"course_link": "/Edge-Animate-tutorials/Edge-Animate-Essential-Training/108880-2.html", "course_title": "Maya Tips and Tricks"}, +{"course_link": "/Edge-Animate-tutorials/Edge-Animate-Essential-Training/108880-2.html", "course_title": "Artist at Work: Creating Depth of Field"}, +{"course_link": "/Photoshop-tutorials/Introducing-Photoshop-Photography/111242-2.html", "course_title": "Entertainment Creation Suite: Getting Started with Backburner"}, +{"course_link": "/Photoshop-tutorials/Introducing-Photoshop-Photography/111242-2.html", "course_title": "Pro Tools Mixing and Mastering"}, +{"course_link": "/Chrome-OS-tutorials/Up-Running-Chrome-OS/160114-2.html", "course_title": "Photoshop Color Correction: Advanced"}, +{"course_link": "/Chrome-OS-tutorials/Up-Running-Chrome-OS/160114-2.html", "course_title": "Designs dekeConstructed: Gradient Dot Patterns with Illustrator"}, +{"course_link": "/After-Effects-tutorials/Getting-Started-After-Effects-Expressions/126108-2.html", "course_title": "Preventing Scope Creep"}, +{"course_link": "/After-Effects-tutorials/Getting-Started-After-Effects-Expressions/126108-2.html", "course_title": "Distributing Android Apps"}, +{"course_link": "/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Portrait Project: Changing a Studio Background"}, +{"course_link": "/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Entertainment Creation Suite: Getting Started with MatchMover"}, +{"course_link": "/Google-Sites-tutorials/Google-Sites-Essential-Training/153986-2.html", "course_title": "Up and Running with AutoCAD for Mac"}, +{"course_link": "/Google-Sites-tutorials/Google-Sites-Essential-Training/153986-2.html", "course_title": "Sony Vegas Production Workflow"}, ] \ No newline at end of file diff --git a/try_scrapy/pipelines.py b/try_scrapy/pipelines.py index 35c965d..786057f 100644 --- a/try_scrapy/pipelines.py +++ b/try_scrapy/pipelines.py @@ -3,6 +3,8 @@ # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html +from try_scrapy import settings + class TryScrapyPipeline(object): def process_item(self, item, spider): return item @@ -12,7 +14,7 @@ import json class JsonWriterPipeline(object): def __init__(self): - self.file = open('output.json', 'wb') + self.file = open(settings.SAVED_JSON, 'wb') def open_spider(self,spider): self.file.write("[\n") diff --git a/try_scrapy/settings.py b/try_scrapy/settings.py index ee1fc17..c767ffe 100644 --- a/try_scrapy/settings.py +++ b/try_scrapy/settings.py @@ -18,3 +18,5 @@ ITEM_PIPELINES = { 'try_scrapy.pipelines.JsonWriterPipeline': 800, } + +SAVED_JSON = 'output.json' diff --git a/try_scrapy/spiders/test_spider.py b/try_scrapy/spiders/test_spider.py index 8b0c59b..9d5031f 100644 --- a/try_scrapy/spiders/test_spider.py +++ b/try_scrapy/spiders/test_spider.py @@ -15,12 +15,14 @@ items=[] def do_nothing(*args,**kwargs): pass -class lynda_spider(CrawlSpider): +class lynda_spider(Spider): name="get_lynda" # allowed_domains = ["dmoz.org"] start_urls=[urlparse.urljoin(base_url,"allcourses")] - + def __init__(self, max_pages=None, *args, **kwargs): + super(lynda_spider, self).__init__(*args, **kwargs) + self.max_pages=max_pages or 5 def parse(self,response): sel = Selector(response) @@ -57,7 +59,7 @@ class lynda_spider(CrawlSpider): # print titles[i].strip() # print links[i] print len(items) - if int(parsed_qs["page"])<=4: + if int(parsed_qs["page"])<=self.max_pages: return Request(url=final_url) else: return items diff --git a/try_scrapy/spiders/test_spider.pyc b/try_scrapy/spiders/test_spider.pyc index 23388db4fe2295c147dfd70de1704d34b93bcc25..09b0d4dc36dbc369ea92145a5624cfb1b153e7e6 100644 GIT binary patch delta 446 zcmX9)u}T9$6r8u0TrM$DV>~sc5R}LX3RV*AMC%Bf&gKYS42dzu+y)d*EDQ)1if-XA z*s6u4ot0uCeuE!iVe8uyc80h6X6McBSN^SFTR*wX?Zu7#LiF%@N>tHXqCAs8ATT8C zI0!5Zft@sD%}eTmD4!(&Sd48(xXh$@5hCDRpe+#bvTy}ZMCi>Uhz?aY756wmpe`LjRhSi(mwigL6<#^;zxL8f(e!5xm6%k9DOB~o^4Tm@ sCQkLR+lUj(HGSk%-1q|~jLy8uCR?~7t1naUjd{vN*`7P&WFQsrS{XklQ8VrK{n)?l98$fU~0$H2hA z0>sWhTs&oSKXWCcLJBuSHX}n(4`Yc4LkbT=4F^Lr17mQBAVY8p14FO|?__P(P5hZ4 zT_wywqO>SSLv-?U*2_XnK%NH3#1fEXacWN5taOBE&(8xj>3b5{pXWfkqVv8BfmOw3wX3!Nti2lwf0IV-lU*$Z=KxtV=