bendtherules/try_scrapy
1
0
Fork 0
mirror of https://github.com/bendtherules/try_scrapy.git synced 2026-08-18 22:02:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

max_pages and SAVED_JSON options added

Browse Source
This commit is contained in:
bendtherules
2014-05-11 19:55:41 +05:30
parent 5c334c2a08
commit 54f6a63230
6 changed files with 62 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Readme.md
+2 -4
View File
@@ -1,7 +1,5 @@
Usage:
scrapy crawl get_lynda
scrapy crawl get_lynda [-a max_pages=10]
For saving to json:
scrapy crawl get_lynda -o output.json -t json
Saves to "output.json" by default ( Configurable as SAVED_JSON in settings.py)
output.json
+50
View File
@@ -199,4 +199,54 @@
{"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Direct Mail Strategy"},
{"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Building Web Services with Java EE"},
{"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Web Design Fundamentals"},
{"course_link": "/Illustrator-tutorials/Type-Project-Victorian-Ad/155652-2.html", "course_title": "Type Project: Victorian Ad"},
{"course_link": "/Illustrator-tutorials/Type-Project-Victorian-Ad/155652-2.html", "course_title": "Mograph Techniques: Dynamic Falling Objects in CINEMA 4D"},
{"course_link": "/CINEMA-4D-tutorials/Mograph-Techniques-Dynamic-Falling-Objects-CINEMA-4D/137890-2.html", "course_title": "XML Essential Training"},
{"course_link": "/CINEMA-4D-tutorials/Mograph-Techniques-Dynamic-Falling-Objects-CINEMA-4D/137890-2.html", "course_title": "SEO: Keyword Strategy in Depth"},
{"course_link": "/XML-tutorials/XML-Essential-Training/145930-2.html", "course_title": "Making Beats in Logic Pro X"},
{"course_link": "/XML-tutorials/XML-Essential-Training/145930-2.html", "course_title": "Dreamweaver CC and WordPress 3.8: Core Concepts"},
{"course_link": "/Business-Online-Marketing-tutorials/SEO-Keyword-Strategy-Depth/147030-2.html", "course_title": "Creating a PDF Presentation"},
{"course_link": "/Business-Online-Marketing-tutorials/SEO-Keyword-Strategy-Depth/147030-2.html", "course_title": "Up and Running with Evernote for Mac"},
{"course_link": "/Logic-Pro-tutorials/Making-Beats-Logic-Pro-X/151550-2.html", "course_title": "Embracing Change"},
{"course_link": "/Logic-Pro-tutorials/Making-Beats-Logic-Pro-X/151550-2.html", "course_title": "After Effects Guru: Working with Vectors"},
{"course_link": "/Dreamweaver-tutorials/Dreamweaver-CC-WordPress-38-Core-Concepts/142034-2.html", "course_title": "Uploading Files Securely with PHP"},
{"course_link": "/Dreamweaver-tutorials/Dreamweaver-CC-WordPress-38-Core-Concepts/142034-2.html", "course_title": "Colorizing Black-and-White Photographs with Photoshop"},
{"course_link": "/InDesign-tutorials/Creating-PDF-Presentation/144077-2.html", "course_title": "Up and Running with 3D Printing"},
{"course_link": "/InDesign-tutorials/Creating-PDF-Presentation/144077-2.html", "course_title": "GarageBand Essential Training"},
{"course_link": "/Evernote-tutorials/Up-Running-Evernote-Mac/157839-2.html", "course_title": "The Creative Spark: Alberto Scirocco and the Motion Graphics of leftchannel"},
{"course_link": "/Evernote-tutorials/Up-Running-Evernote-Mac/157839-2.html", "course_title": "Travel Photography: Geologging and Journaling on the Road"},
{"course_link": "/Business-Business-Skills-tutorials/Embracing-Change/150003-2.html", "course_title": "WordPress Ecommerce: Integrating Shopify"},
{"course_link": "/Business-Business-Skills-tutorials/Embracing-Change/150003-2.html", "course_title": "UX Design Techniques: Creating Scenarios and Storyboards"},
{"course_link": "/After-Effects-tutorials/After-Effects-Guru-Working-Vectors/160069-2.html", "course_title": "Delivering Employee Feedback"},
{"course_link": "/After-Effects-tutorials/After-Effects-Guru-Working-Vectors/160069-2.html", "course_title": "Edge Animate Essential Training"},
{"course_link": "/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Introducing Photoshop: Photography"},
{"course_link": "/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Up and Running with Chrome OS"},
{"course_link": "/Photoshop-tutorials/Colorizing-Black-White-Photographs-Photoshop/133329-2.html", "course_title": "Getting Started with After Effects Expressions"},
{"course_link": "/Photoshop-tutorials/Colorizing-Black-White-Photographs-Photoshop/133329-2.html", "course_title": "Foundations of Programming: Web Security"},
{"course_link": "/3D-Animation-Prototyping-tutorials/Up-Running-3D-Printing/151814-2.html", "course_title": "Google Sites Essential Training"},
{"course_link": "/3D-Animation-Prototyping-tutorials/Up-Running-3D-Printing/151814-2.html", "course_title": "Entertainment Creation Suite: Getting Started with Toxik"},
{"course_link": "/GarageBand-tutorials/GarageBand-Essential-Training/156620-2.html", "course_title": "Artist at Work: From Sketch to Finished Vector Illustration"},
{"course_link": "/GarageBand-tutorials/GarageBand-Essential-Training/156620-2.html", "course_title": "VFX Techniques: Space Scene 02: Compositing in After Effects"},
{"course_link": "/Video-Motion-Graphics-tutorials/Creative-Spark-Alberto-Scirocco-Motion-Graphics-leftchannel/153774-2.html", "course_title": "Building an Online Community"},
{"course_link": "/Video-Motion-Graphics-tutorials/Creative-Spark-Alberto-Scirocco-Motion-Graphics-leftchannel/153774-2.html", "course_title": "Cinematography in Maya"},
{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Geologging-Journaling-Road/149290-2.html", "course_title": "The Art of Photoshop Compositing"},
{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Geologging-Journaling-Road/149290-2.html", "course_title": "Up and Running with Lectora Inspire"},
{"course_link": "/WordPress-tutorials/WordPress-Ecommerce-Integrating-Shopify/154413-2.html", "course_title": "UX Design Techniques: Ideation"},
{"course_link": "/WordPress-tutorials/WordPress-Ecommerce-Integrating-Shopify/154413-2.html", "course_title": "Up and Running with Promethean ActivInspire"},
{"course_link": "/Web-User-Experience-tutorials/UX-Design-Techniques-Creating-Scenarios-Storyboards/148581-2.html", "course_title": "Artist at Work: Tertiary Colors"},
{"course_link": "/Web-User-Experience-tutorials/UX-Design-Techniques-Creating-Scenarios-Storyboards/148581-2.html", "course_title": "Excel 2013: Macros in Depth"},
{"course_link": "/Business-Business-Skills-tutorials/Delivering-Employee-Feedback/149836-2.html", "course_title": "Primitive and Spline Modeling in CINEMA 4D"},
{"course_link": "/Business-Business-Skills-tutorials/Delivering-Employee-Feedback/149836-2.html", "course_title": "InDesign Insider Training: Data Merge and Database Publishing"},
{"course_link": "/Edge-Animate-tutorials/Edge-Animate-Essential-Training/108880-2.html", "course_title": "Maya Tips and Tricks"},
{"course_link": "/Edge-Animate-tutorials/Edge-Animate-Essential-Training/108880-2.html", "course_title": "Artist at Work: Creating Depth of Field"},
{"course_link": "/Photoshop-tutorials/Introducing-Photoshop-Photography/111242-2.html", "course_title": "Entertainment Creation Suite: Getting Started with Backburner"},
{"course_link": "/Photoshop-tutorials/Introducing-Photoshop-Photography/111242-2.html", "course_title": "Pro Tools Mixing and Mastering"},
{"course_link": "/Chrome-OS-tutorials/Up-Running-Chrome-OS/160114-2.html", "course_title": "Photoshop Color Correction: Advanced"},
{"course_link": "/Chrome-OS-tutorials/Up-Running-Chrome-OS/160114-2.html", "course_title": "Designs dekeConstructed: Gradient Dot Patterns with Illustrator"},
{"course_link": "/After-Effects-tutorials/Getting-Started-After-Effects-Expressions/126108-2.html", "course_title": "Preventing Scope Creep"},
{"course_link": "/After-Effects-tutorials/Getting-Started-After-Effects-Expressions/126108-2.html", "course_title": "Distributing Android Apps"},
{"course_link": "/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Portrait Project: Changing a Studio Background"},
{"course_link": "/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Entertainment Creation Suite: Getting Started with MatchMover"},
{"course_link": "/Google-Sites-tutorials/Google-Sites-Essential-Training/153986-2.html", "course_title": "Up and Running with AutoCAD for Mac"},
{"course_link": "/Google-Sites-tutorials/Google-Sites-Essential-Training/153986-2.html", "course_title": "Sony Vegas Production Workflow"},
]
try_scrapy/pipelines.py
+3 -1
View File
@@ -3,6 +3,8 @@
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from try_scrapy import settings
class TryScrapyPipeline(object):
def process_item(self, item, spider):
return item
@@ -12,7 +14,7 @@ import json
class JsonWriterPipeline(object):
def __init__(self):
self.file = open('output.json', 'wb')
self.file = open(settings.SAVED_JSON, 'wb')
def open_spider(self,spider):
self.file.write("[\n")
try_scrapy/settings.py
+2
View File
@@ -18,3 +18,5 @@ ITEM_PIPELINES = {
'try_scrapy.pipelines.JsonWriterPipeline': 800,
}
SAVED_JSON = 'output.json'
try_scrapy/spiders/test_spider.py
+5 -3
View File
@@ -15,12 +15,14 @@ items=[]
def do_nothing(*args,**kwargs):
pass
class lynda_spider(CrawlSpider):
class lynda_spider(Spider):
name="get_lynda"
# allowed_domains = ["dmoz.org"]
start_urls=[urlparse.urljoin(base_url,"allcourses")]
def __init__(self, max_pages=None, *args, **kwargs):
super(lynda_spider, self).__init__(*args, **kwargs)
self.max_pages=max_pages or 5
def parse(self,response):
sel = Selector(response)
@@ -57,7 +59,7 @@ class lynda_spider(CrawlSpider):
# print titles[i].strip()
# print links[i]
print len(items)
if int(parsed_qs["page"])<=4:
if int(parsed_qs["page"])<=self.max_pages:
return Request(url=final_url)
else:
return items
try_scrapy/spiders/test_spider.pyc
BIN
View File
Binary file not shown.