bendtherules/try_scrapy
1
0
Fork 0
mirror of https://github.com/bendtherules/try_scrapy.git synced 2026-08-18 22:02:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

url parsing done using urlparse

Browse Source
This commit is contained in:
bendtherules
2014-05-11 18:52:23 +05:30
parent d9560bcda2
commit 5445c795a9
3 changed files with 58 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
output.json
+50 -50
View File
@@ -1,50 +1,50 @@
[{"course_link": "/3D-Animation-Design-Techniques-tutorials/Power-PreViz-One-Co/98575-2.html", "course_title": "The Power of PreViz at One & Co"}, [{"course_link": "/After-Effects-tutorials/Getting-Started-Stop-Motion-Animation/163241-2.html", "course_title": "Getting Started with Stop Motion Animation"},
{"course_link": "/3D-Animation-Design-Techniques-tutorials/Power-PreViz-One-Co/98575-2.html", "course_title": "Foundations of Typography"}, {"course_link": "/After-Effects-tutorials/Getting-Started-Stop-Motion-Animation/163241-2.html", "course_title": "Revit: Family Curves and Formulas"},
{"course_link": "/Design-Typography-tutorials/Foundations-Typography/106698-2.html", "course_title": "Up and Running with Excel 2010"}, {"course_link": "/Revit-Architecture-tutorials/Revit-Family-Curves-Formulas/148423-2.html", "course_title": "Persuasive UX: The Power of Self-Image"},
{"course_link": "/Design-Typography-tutorials/Foundations-Typography/106698-2.html", "course_title": "Writing Fundamentals: The Craft of Story"}, {"course_link": "/Revit-Architecture-tutorials/Revit-Family-Curves-Formulas/148423-2.html", "course_title": "Motion Graphics for Video Editors: Creating Animated Logos"},
{"course_link": "/Excel-tutorials/Up-Running-Excel-2010/123569-2.html", "course_title": "Mixing a Short Film with Pro Tools"}, {"course_link": "/Web-User-Experience-tutorials/Persuasive-UX-Power-Self-Image/162702-2.html", "course_title": "SketchUp Pro: Tools and Techniques"},
{"course_link": "/Excel-tutorials/Up-Running-Excel-2010/123569-2.html", "course_title": "Office 2013 New Features"}, {"course_link": "/Web-User-Experience-tutorials/Persuasive-UX-Power-Self-Image/162702-2.html", "course_title": "Up and Running with OS X Server App"},
{"course_link": "/Business-Business-Skills-tutorials/Writing-Fundamentals-Craft-Story/89962-2.html", "course_title": "Word 2013 Essential Training"}, {"course_link": "/After-Effects-tutorials/Motion-Graphics-Video-Editors-Creating-Animated-Logos/149129-2.html", "course_title": "iOS Game Development with Sprite Kit"},
{"course_link": "/Business-Business-Skills-tutorials/Writing-Fundamentals-Craft-Story/89962-2.html", "course_title": "Excel 2013 Essential Training"}, {"course_link": "/After-Effects-tutorials/Motion-Graphics-Video-Editors-Creating-Animated-Logos/149129-2.html", "course_title": "Design the Web: Adobe Generator for Graphics"},
{"course_link": "/Pro-Tools-tutorials/Mixing-Short-Film-Pro-Tools/111780-2.html", "course_title": "Creating a Mail Merge in Word 2010"}, {"course_link": "/SketchUp-tutorials/SketchUp-Pro-Tools-Techniques/151486-2.html", "course_title": "Lighting a Video Interview"},
{"course_link": "/Pro-Tools-tutorials/Mixing-Short-Film-Pro-Tools/111780-2.html", "course_title": "VFX Techniques: Tracking Objects onto a Face"}, {"course_link": "/SketchUp-tutorials/SketchUp-Pro-Tools-Techniques/151486-2.html", "course_title": "WordPress Plugins: Contact Forms"},
{"course_link": "/Office-tutorials/Office-2013-New-Features/84528-2.html", "course_title": "Design the Web: Clipping Masks"}, {"course_link": "/Mac-OS-X-tutorials/Up-Running-OS-X-Server-App/149127-2.html", "course_title": "Calculating Earned Value"},
{"course_link": "/Office-tutorials/Office-2013-New-Features/84528-2.html", "course_title": "Android SDK: Local Data Storage"}, {"course_link": "/Mac-OS-X-tutorials/Up-Running-OS-X-Server-App/149127-2.html", "course_title": "Photoshop Color Correction: Advanced Projects"},
{"course_link": "/Office-tutorials/Word-2013-Essential-Training/115862-2.html", "course_title": "Foundations of Programming: Databases"}, {"course_link": "/iOS-tutorials/iOS-Game-Development-Sprite-Kit/146015-2.html", "course_title": "UX Design Tools: InDesign"},
{"course_link": "/Office-tutorials/Word-2013-Essential-Training/115862-2.html", "course_title": "Photoshop CS6 Creative Cloud New Features Workshop"}, {"course_link": "/iOS-tutorials/iOS-Game-Development-Sprite-Kit/146015-2.html", "course_title": "Keynote: Builds, Transitions, Animations, and Effects"},
{"course_link": "/Excel-tutorials/Excel-2013-Essential-Training/116478-2.html", "course_title": "Rendering Using Cycles in Blender"}, {"course_link": "/Photoshop-tutorials/Design-Web-Adobe-Generator-Graphics/162713-2.html", "course_title": "Managing Project Teams"},
{"course_link": "/Excel-tutorials/Excel-2013-Essential-Training/116478-2.html", "course_title": "Photoshop Elements 11 Essentials: 03 Sharing and Printing Photos"}, {"course_link": "/Photoshop-tutorials/Design-Web-Adobe-Generator-Graphics/162713-2.html", "course_title": "Photoshop CC for Web Design"},
{"course_link": "/Office-tutorials/Creating-Mail-Merge-Word-2010/123568-2.html", "course_title": "Audio Mastering Techniques"}, {"course_link": "/Video-tutorials/Lighting-Video-Interview/103708-2.html", "course_title": "Package Design with Illustrator"},
{"course_link": "/Office-tutorials/Creating-Mail-Merge-Word-2010/123568-2.html", "course_title": "CSS: Frameworks and Grids"}, {"course_link": "/Video-tutorials/Lighting-Video-Interview/103708-2.html", "course_title": "Creating Icon Fonts for the Web"},
{"course_link": "/After-Effects-tutorials/VFX-Techniques-Tracking-Objects-Onto-Face/114013-2.html", "course_title": "The Creative Spark: Brian Taylor, Handmade Photography"}, {"course_link": "/WordPress-tutorials/WordPress-Plugins-Contact-Forms/162451-2.html", "course_title": "Travel Photography: The Family Cabin"},
{"course_link": "/After-Effects-tutorials/VFX-Techniques-Tracking-Objects-Onto-Face/114013-2.html", "course_title": "JavaScript and AJAX"}, {"course_link": "/WordPress-tutorials/WordPress-Plugins-Contact-Forms/162451-2.html", "course_title": "SQL Essential Training"},
{"course_link": "/Photoshop-tutorials/Web-Design-Lab-Using-Clipping-Masks-Nondestructive-Cropping/114906-2.html", "course_title": "Up and Running with PowerPoint 2010"}, {"course_link": "/Business-Business-Skills-tutorials/Calculating-Earned-Value/162457-2.html", "course_title": "Type Project: Dada Poster"},
{"course_link": "/Photoshop-tutorials/Web-Design-Lab-Using-Clipping-Masks-Nondestructive-Cropping/114906-2.html", "course_title": "Managing Small Projects"}, {"course_link": "/Business-Business-Skills-tutorials/Calculating-Earned-Value/162457-2.html", "course_title": "Premiere Pro Guru: Customizing Workspaces"},
{"course_link": "/Android-tutorials/Android-SDK-Local-Data-Storage/112584-2.html", "course_title": "After Effects Apprentice 16: Creating a Medical Opening Title"}, {"course_link": "/Photoshop-tutorials/Photoshop-Color-Correction-Advanced-Projects/160066-2.html", "course_title": "Shooting with Blackmagic Cameras"},
{"course_link": "/Android-tutorials/Android-SDK-Local-Data-Storage/112584-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, {"course_link": "/Photoshop-tutorials/Photoshop-Color-Correction-Advanced-Projects/160066-2.html", "course_title": "Adobe Digital Publishing Suite Essential Training"},
{"course_link": "/Programming-tutorials/Foundations-Programming-Databases/112585-2.html", "course_title": "iOS SDK and SQLite: Building Data-Driven Apps"}, {"course_link": "/InDesign-tutorials/UX-Design-Tools-InDesign/153985-2.html", "course_title": "Visual Studio 2013 for Windows Store Developers"},
{"course_link": "/Programming-tutorials/Foundations-Programming-Databases/112585-2.html", "course_title": "Photoshop CS6 New Features"}, {"course_link": "/InDesign-tutorials/UX-Design-Tools-InDesign/153985-2.html", "course_title": "Visual Studio 2013 for Web Developers"},
{"course_link": "/Photoshop-tutorials/Photoshop-CS6-Creative-Cloud-New-Features-Workshop/123564-2.html", "course_title": "VFX Techniques: Crowd Replication with After Effects"}, {"course_link": "/Keynote-tutorials/Keynote-Builds-Transitions-Animations-Effects/161975-2.html", "course_title": "Up and Running with ArcGIS"},
{"course_link": "/Photoshop-tutorials/Photoshop-CS6-Creative-Cloud-New-Features-Workshop/123564-2.html", "course_title": "Aperture 3.3/3.4 New Features Overview"}, {"course_link": "/Keynote-tutorials/Keynote-Builds-Transitions-Animations-Effects/161975-2.html", "course_title": "Revit Construction Modeling Tools"},
{"course_link": "/Blender-tutorials/Photorealistic-Rendering-using-Cycles-Blender/117544-2.html", "course_title": "Responsive Design with Drupal"}, {"course_link": "/Business-Business-Skills-tutorials/Managing-Project-Teams/139981-2.html", "course_title": "UX Design Tools: Illustrator"},
{"course_link": "/Blender-tutorials/Photorealistic-Rendering-using-Cycles-Blender/117544-2.html", "course_title": "Up and Running with Windows 8"}, {"course_link": "/Business-Business-Skills-tutorials/Managing-Project-Teams/139981-2.html", "course_title": "iPad for Business"},
{"course_link": "/Photoshop-Elements-tutorials/Photoshop-Elements-11-Essentials-03-Sharing-Printing-Photos/117104-2.html", "course_title": "Multilingual Publishing Strategies with InDesign"}, {"course_link": "/Photoshop-tutorials/Photoshop-CC-Web-Design/145211-2.html", "course_title": "InDesign CC: Interactive Document Fundamentals"},
{"course_link": "/Photoshop-Elements-tutorials/Photoshop-Elements-11-Essentials-03-Sharing-Printing-Photos/117104-2.html", "course_title": "Templating with Joomla! 3 and Bootstrap"}, {"course_link": "/Photoshop-tutorials/Photoshop-CC-Web-Design/145211-2.html", "course_title": "Up and Running with Constant Contact"},
{"course_link": "/Mastering-tutorials/Audio-Mastering-Techniques/109358-2.html", "course_title": "ASP.NET MVC 4 Essential Training"}, {"course_link": "/Illustrator-tutorials/Package-Design-Illustrator/148421-2.html", "course_title": "Unity 4.3 Essential Training"},
{"course_link": "/Mastering-tutorials/Audio-Mastering-Techniques/109358-2.html", "course_title": "Getting Started with Muse"}, {"course_link": "/Illustrator-tutorials/Package-Design-Illustrator/148421-2.html", "course_title": "Enhancing Exterior Architectural Photos"},
{"course_link": "/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "Photoshop Artist in Action: Tim Grey's Abandoned Farmhouse"}, {"course_link": "/Glyphs-App-tutorials/Creating-Icon-Fonts-Web/157228-2.html", "course_title": "Synth Programming Basics"},
{"course_link": "/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "Photoshop Elements 11 Essentials: 01 Importing and Organizing Photos"}, {"course_link": "/Glyphs-App-tutorials/Creating-Icon-Fonts-Web/157228-2.html", "course_title": "The Creative Spark: Between Two Worlds, The Hybrid Animation of Tiny Inventions"},
{"course_link": "/Photography-Documentaries-tutorials/Creative-Spark-Brian-Taylor-Handmade-Photography/101505-2.html", "course_title": "Digital Painting: Architecture"}, {"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Family-Cabin/149289-2.html", "course_title": "Foundations of Programming: Programming for Kids"},
{"course_link": "/Photography-Documentaries-tutorials/Creative-Spark-Brian-Taylor-Handmade-Photography/101505-2.html", "course_title": "Leading Productive Meetings"}, {"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Family-Cabin/149289-2.html", "course_title": "Persuasive UX: Ethics of Persuasive Design"},
{"course_link": "/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "Discovering Your Strengths"}, {"course_link": "/SQL-tutorials/SQL-Essential-Training/139988-2.html", "course_title": "Type Project: Art Nouveau Poster"},
{"course_link": "/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "Becoming a Thought Leader"}, {"course_link": "/SQL-tutorials/SQL-Essential-Training/139988-2.html", "course_title": "Customer Service Fundamentals"},
{"course_link": "/Office-tutorials/Up-Running-PowerPoint-2010/123567-2.html", "course_title": "Enhancing Your Productivity"}, {"course_link": "/Illustrator-tutorials/Type-Project-Dada-Poster/155654-2.html", "course_title": "Creating Accessible PDFs"},
{"course_link": "/Office-tutorials/Up-Running-PowerPoint-2010/123567-2.html", "course_title": "Building Your Professional Network"}, {"course_link": "/Illustrator-tutorials/Type-Project-Dada-Poster/155654-2.html", "course_title": "UX Design Tools: Photoshop"},
{"course_link": "/Business-Skills-tutorials/Managing-Small-Projects/105326-2.html", "course_title": "Thinking Like a Leader"}, {"course_link": "/Premiere-Pro-tutorials/Premiere-Pro-Guru-Customizing-Workspaces/161521-2.html", "course_title": "Enhancing Interior Architectural Photos"},
{"course_link": "/Business-Skills-tutorials/Managing-Small-Projects/105326-2.html", "course_title": "How to Use the lynda.com Partners Site"}, {"course_link": "/Premiere-Pro-tutorials/Premiere-Pro-Guru-Customizing-Workspaces/161521-2.html", "course_title": "Solving Business Problems"},
{"course_link": "/After-Effects-tutorials/After-Effects-Apprentice-16-Creating-Medical-Opening-Title/79644-2.html", "course_title": "Developing Brand Identity Collateral"}, {"course_link": "/Video-Shooting-Video-tutorials/Shooting-Blackmagic-Cameras/157510-2.html", "course_title": "Up and Running with Photomatix Pro"},
{"course_link": "/After-Effects-tutorials/After-Effects-Apprentice-16-Creating-Medical-Opening-Title/79644-2.html", "course_title": "Photoshop Artist in Action: Tim Grey's Prayer Sticks"}, {"course_link": "/Video-Shooting-Video-tutorials/Shooting-Blackmagic-Cameras/157510-2.html", "course_title": "Up and Running with Office Web Apps"},
{"course_link": "/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Drawing Vector Graphics"}, {"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Final Cut Pro X 10.1.1 Essential Training"},
{"course_link": "/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "iPhone and iPod touch iOS 6 Essential Training"}, {"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Direct Mail Strategy"},
{"course_link": "/Developer-tutorials/iOS-SDK-SQLite-Building-Data-Driven-Apps/107075-2.html", "course_title": "User Experience Fundamentals for Web Design"}, {"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Building Web Services with Java EE"},
{"course_link": "/Developer-tutorials/iOS-SDK-SQLite-Building-Data-Driven-Apps/107075-2.html", "course_title": "Photorealistic Lighting with Maya and Nuke"}] {"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Web Design Fundamentals"}]
try_scrapy/spiders/test_spider.py
+8 -5
View File
@@ -6,17 +6,19 @@ from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.shell import inspect_response from scrapy.shell import inspect_response
import urlparse,urllib import urlparse,urllib
from scrapy import log
base_url=r"http://www.lynda.com" base_url=r"http://www.lynda.com"
items=[]
log.start()
class lynda_spider(CrawlSpider): class lynda_spider(CrawlSpider):
name="get_lynda" name="get_lynda"
# allowed_domains = ["dmoz.org"] # allowed_domains = ["dmoz.org"]
start_urls=[base_url+"/allcourses"] start_urls=[urlparse.urljoin(base_url,"allcourses")]
def parse(self,response): def parse(self,response):
sel = Selector(response) sel = Selector(response)
items=[]
next_page =sel.xpath("//a[contains(@class,'see-more-results')]/@data-url").extract() next_page =sel.xpath("//a[contains(@class,'see-more-results')]/@data-url").extract()
if next_page: if next_page:
@@ -34,12 +36,12 @@ class lynda_spider(CrawlSpider):
final_url=urlparse.urlunparse(parsed_url) final_url=urlparse.urlunparse(parsed_url)
print final_url print final_url
if int(parsed_qs["page"])<=10: if int(parsed_qs["page"])<=4:
return Request(final_url, self.parse) return Request(final_url, self.parse)
sel = Selector(response) sel = Selector(response)
course_a=sel.xpath(r'//*[@class="course-list"]//li//a') course_a=sel.xpath(r'//*[@class="course-list"]//li//a')
print course_a.extract() # print course_a.extract()
titles=course_a.xpath("text()").extract() titles=course_a.xpath("text()").extract()
links=course_a.xpath("@href").extract() links=course_a.xpath("@href").extract()
for i in range(len(titles)): for i in range(len(titles)):
@@ -47,8 +49,9 @@ class lynda_spider(CrawlSpider):
tmp_item["course_title"]=titles[i].strip() tmp_item["course_title"]=titles[i].strip()
tmp_item["course_link"]=links[i] tmp_item["course_link"]=links[i]
items.append(tmp_item) items.append(tmp_item)
print titles[i].strip() # print titles[i].strip()
# print links[i] # print links[i]
print len(items)
return items return items
# class NextPageLinkExtractor(SgmlLinkExtractor): # class NextPageLinkExtractor(SgmlLinkExtractor):
try_scrapy/spiders/test_spider.pyc
BIN
View File
Binary file not shown.