diff --git a/output.json b/output.json index 010e0c5..21642c1 100644 --- a/output.json +++ b/output.json @@ -1,50 +1,50 @@ -[{"course_link": "/3D-Animation-Design-Techniques-tutorials/Power-PreViz-One-Co/98575-2.html", "course_title": "The Power of PreViz at One & Co"}, -{"course_link": "/3D-Animation-Design-Techniques-tutorials/Power-PreViz-One-Co/98575-2.html", "course_title": "Foundations of Typography"}, -{"course_link": "/Design-Typography-tutorials/Foundations-Typography/106698-2.html", "course_title": "Up and Running with Excel 2010"}, -{"course_link": "/Design-Typography-tutorials/Foundations-Typography/106698-2.html", "course_title": "Writing Fundamentals: The Craft of Story"}, -{"course_link": "/Excel-tutorials/Up-Running-Excel-2010/123569-2.html", "course_title": "Mixing a Short Film with Pro Tools"}, -{"course_link": "/Excel-tutorials/Up-Running-Excel-2010/123569-2.html", "course_title": "Office 2013 New Features"}, -{"course_link": "/Business-Business-Skills-tutorials/Writing-Fundamentals-Craft-Story/89962-2.html", "course_title": "Word 2013 Essential Training"}, -{"course_link": "/Business-Business-Skills-tutorials/Writing-Fundamentals-Craft-Story/89962-2.html", "course_title": "Excel 2013 Essential Training"}, -{"course_link": "/Pro-Tools-tutorials/Mixing-Short-Film-Pro-Tools/111780-2.html", "course_title": "Creating a Mail Merge in Word 2010"}, -{"course_link": "/Pro-Tools-tutorials/Mixing-Short-Film-Pro-Tools/111780-2.html", "course_title": "VFX Techniques: Tracking Objects onto a Face"}, -{"course_link": "/Office-tutorials/Office-2013-New-Features/84528-2.html", "course_title": "Design the Web: Clipping Masks"}, -{"course_link": "/Office-tutorials/Office-2013-New-Features/84528-2.html", "course_title": "Android SDK: Local Data Storage"}, -{"course_link": "/Office-tutorials/Word-2013-Essential-Training/115862-2.html", "course_title": "Foundations of Programming: Databases"}, -{"course_link": "/Office-tutorials/Word-2013-Essential-Training/115862-2.html", "course_title": "Photoshop CS6 Creative Cloud New Features Workshop"}, -{"course_link": "/Excel-tutorials/Excel-2013-Essential-Training/116478-2.html", "course_title": "Rendering Using Cycles in Blender"}, -{"course_link": "/Excel-tutorials/Excel-2013-Essential-Training/116478-2.html", "course_title": "Photoshop Elements 11 Essentials: 03 Sharing and Printing Photos"}, -{"course_link": "/Office-tutorials/Creating-Mail-Merge-Word-2010/123568-2.html", "course_title": "Audio Mastering Techniques"}, -{"course_link": "/Office-tutorials/Creating-Mail-Merge-Word-2010/123568-2.html", "course_title": "CSS: Frameworks and Grids"}, -{"course_link": "/After-Effects-tutorials/VFX-Techniques-Tracking-Objects-Onto-Face/114013-2.html", "course_title": "The Creative Spark: Brian Taylor, Handmade Photography"}, -{"course_link": "/After-Effects-tutorials/VFX-Techniques-Tracking-Objects-Onto-Face/114013-2.html", "course_title": "JavaScript and AJAX"}, -{"course_link": "/Photoshop-tutorials/Web-Design-Lab-Using-Clipping-Masks-Nondestructive-Cropping/114906-2.html", "course_title": "Up and Running with PowerPoint 2010"}, -{"course_link": "/Photoshop-tutorials/Web-Design-Lab-Using-Clipping-Masks-Nondestructive-Cropping/114906-2.html", "course_title": "Managing Small Projects"}, -{"course_link": "/Android-tutorials/Android-SDK-Local-Data-Storage/112584-2.html", "course_title": "After Effects Apprentice 16: Creating a Medical Opening Title"}, -{"course_link": "/Android-tutorials/Android-SDK-Local-Data-Storage/112584-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, -{"course_link": "/Programming-tutorials/Foundations-Programming-Databases/112585-2.html", "course_title": "iOS SDK and SQLite: Building Data-Driven Apps"}, -{"course_link": "/Programming-tutorials/Foundations-Programming-Databases/112585-2.html", "course_title": "Photoshop CS6 New Features"}, -{"course_link": "/Photoshop-tutorials/Photoshop-CS6-Creative-Cloud-New-Features-Workshop/123564-2.html", "course_title": "VFX Techniques: Crowd Replication with After Effects"}, -{"course_link": "/Photoshop-tutorials/Photoshop-CS6-Creative-Cloud-New-Features-Workshop/123564-2.html", "course_title": "Aperture 3.3/3.4 New Features Overview"}, -{"course_link": "/Blender-tutorials/Photorealistic-Rendering-using-Cycles-Blender/117544-2.html", "course_title": "Responsive Design with Drupal"}, -{"course_link": "/Blender-tutorials/Photorealistic-Rendering-using-Cycles-Blender/117544-2.html", "course_title": "Up and Running with Windows 8"}, -{"course_link": "/Photoshop-Elements-tutorials/Photoshop-Elements-11-Essentials-03-Sharing-Printing-Photos/117104-2.html", "course_title": "Multilingual Publishing Strategies with InDesign"}, -{"course_link": "/Photoshop-Elements-tutorials/Photoshop-Elements-11-Essentials-03-Sharing-Printing-Photos/117104-2.html", "course_title": "Templating with Joomla! 3 and Bootstrap"}, -{"course_link": "/Mastering-tutorials/Audio-Mastering-Techniques/109358-2.html", "course_title": "ASP.NET MVC 4 Essential Training"}, -{"course_link": "/Mastering-tutorials/Audio-Mastering-Techniques/109358-2.html", "course_title": "Getting Started with Muse"}, -{"course_link": "/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "Photoshop Artist in Action: Tim Grey's Abandoned Farmhouse"}, -{"course_link": "/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "Photoshop Elements 11 Essentials: 01 Importing and Organizing Photos"}, -{"course_link": "/Photography-Documentaries-tutorials/Creative-Spark-Brian-Taylor-Handmade-Photography/101505-2.html", "course_title": "Digital Painting: Architecture"}, -{"course_link": "/Photography-Documentaries-tutorials/Creative-Spark-Brian-Taylor-Handmade-Photography/101505-2.html", "course_title": "Leading Productive Meetings"}, -{"course_link": "/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "Discovering Your Strengths"}, -{"course_link": "/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "Becoming a Thought Leader"}, -{"course_link": "/Office-tutorials/Up-Running-PowerPoint-2010/123567-2.html", "course_title": "Enhancing Your Productivity"}, -{"course_link": "/Office-tutorials/Up-Running-PowerPoint-2010/123567-2.html", "course_title": "Building Your Professional Network"}, -{"course_link": "/Business-Skills-tutorials/Managing-Small-Projects/105326-2.html", "course_title": "Thinking Like a Leader"}, -{"course_link": "/Business-Skills-tutorials/Managing-Small-Projects/105326-2.html", "course_title": "How to Use the lynda.com Partners Site"}, -{"course_link": "/After-Effects-tutorials/After-Effects-Apprentice-16-Creating-Medical-Opening-Title/79644-2.html", "course_title": "Developing Brand Identity Collateral"}, -{"course_link": "/After-Effects-tutorials/After-Effects-Apprentice-16-Creating-Medical-Opening-Title/79644-2.html", "course_title": "Photoshop Artist in Action: Tim Grey's Prayer Sticks"}, -{"course_link": "/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Drawing Vector Graphics"}, -{"course_link": "/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "iPhone and iPod touch iOS 6 Essential Training"}, -{"course_link": "/Developer-tutorials/iOS-SDK-SQLite-Building-Data-Driven-Apps/107075-2.html", "course_title": "User Experience Fundamentals for Web Design"}, -{"course_link": "/Developer-tutorials/iOS-SDK-SQLite-Building-Data-Driven-Apps/107075-2.html", "course_title": "Photorealistic Lighting with Maya and Nuke"}] \ No newline at end of file +[{"course_link": "/After-Effects-tutorials/Getting-Started-Stop-Motion-Animation/163241-2.html", "course_title": "Getting Started with Stop Motion Animation"}, +{"course_link": "/After-Effects-tutorials/Getting-Started-Stop-Motion-Animation/163241-2.html", "course_title": "Revit: Family Curves and Formulas"}, +{"course_link": "/Revit-Architecture-tutorials/Revit-Family-Curves-Formulas/148423-2.html", "course_title": "Persuasive UX: The Power of Self-Image"}, +{"course_link": "/Revit-Architecture-tutorials/Revit-Family-Curves-Formulas/148423-2.html", "course_title": "Motion Graphics for Video Editors: Creating Animated Logos"}, +{"course_link": "/Web-User-Experience-tutorials/Persuasive-UX-Power-Self-Image/162702-2.html", "course_title": "SketchUp Pro: Tools and Techniques"}, +{"course_link": "/Web-User-Experience-tutorials/Persuasive-UX-Power-Self-Image/162702-2.html", "course_title": "Up and Running with OS X Server App"}, +{"course_link": "/After-Effects-tutorials/Motion-Graphics-Video-Editors-Creating-Animated-Logos/149129-2.html", "course_title": "iOS Game Development with Sprite Kit"}, +{"course_link": "/After-Effects-tutorials/Motion-Graphics-Video-Editors-Creating-Animated-Logos/149129-2.html", "course_title": "Design the Web: Adobe Generator for Graphics"}, +{"course_link": "/SketchUp-tutorials/SketchUp-Pro-Tools-Techniques/151486-2.html", "course_title": "Lighting a Video Interview"}, +{"course_link": "/SketchUp-tutorials/SketchUp-Pro-Tools-Techniques/151486-2.html", "course_title": "WordPress Plugins: Contact Forms"}, +{"course_link": "/Mac-OS-X-tutorials/Up-Running-OS-X-Server-App/149127-2.html", "course_title": "Calculating Earned Value"}, +{"course_link": "/Mac-OS-X-tutorials/Up-Running-OS-X-Server-App/149127-2.html", "course_title": "Photoshop Color Correction: Advanced Projects"}, +{"course_link": "/iOS-tutorials/iOS-Game-Development-Sprite-Kit/146015-2.html", "course_title": "UX Design Tools: InDesign"}, +{"course_link": "/iOS-tutorials/iOS-Game-Development-Sprite-Kit/146015-2.html", "course_title": "Keynote: Builds, Transitions, Animations, and Effects"}, +{"course_link": "/Photoshop-tutorials/Design-Web-Adobe-Generator-Graphics/162713-2.html", "course_title": "Managing Project Teams"}, +{"course_link": "/Photoshop-tutorials/Design-Web-Adobe-Generator-Graphics/162713-2.html", "course_title": "Photoshop CC for Web Design"}, +{"course_link": "/Video-tutorials/Lighting-Video-Interview/103708-2.html", "course_title": "Package Design with Illustrator"}, +{"course_link": "/Video-tutorials/Lighting-Video-Interview/103708-2.html", "course_title": "Creating Icon Fonts for the Web"}, +{"course_link": "/WordPress-tutorials/WordPress-Plugins-Contact-Forms/162451-2.html", "course_title": "Travel Photography: The Family Cabin"}, +{"course_link": "/WordPress-tutorials/WordPress-Plugins-Contact-Forms/162451-2.html", "course_title": "SQL Essential Training"}, +{"course_link": "/Business-Business-Skills-tutorials/Calculating-Earned-Value/162457-2.html", "course_title": "Type Project: Dada Poster"}, +{"course_link": "/Business-Business-Skills-tutorials/Calculating-Earned-Value/162457-2.html", "course_title": "Premiere Pro Guru: Customizing Workspaces"}, +{"course_link": "/Photoshop-tutorials/Photoshop-Color-Correction-Advanced-Projects/160066-2.html", "course_title": "Shooting with Blackmagic Cameras"}, +{"course_link": "/Photoshop-tutorials/Photoshop-Color-Correction-Advanced-Projects/160066-2.html", "course_title": "Adobe Digital Publishing Suite Essential Training"}, +{"course_link": "/InDesign-tutorials/UX-Design-Tools-InDesign/153985-2.html", "course_title": "Visual Studio 2013 for Windows Store Developers"}, +{"course_link": "/InDesign-tutorials/UX-Design-Tools-InDesign/153985-2.html", "course_title": "Visual Studio 2013 for Web Developers"}, +{"course_link": "/Keynote-tutorials/Keynote-Builds-Transitions-Animations-Effects/161975-2.html", "course_title": "Up and Running with ArcGIS"}, +{"course_link": "/Keynote-tutorials/Keynote-Builds-Transitions-Animations-Effects/161975-2.html", "course_title": "Revit Construction Modeling Tools"}, +{"course_link": "/Business-Business-Skills-tutorials/Managing-Project-Teams/139981-2.html", "course_title": "UX Design Tools: Illustrator"}, +{"course_link": "/Business-Business-Skills-tutorials/Managing-Project-Teams/139981-2.html", "course_title": "iPad for Business"}, +{"course_link": "/Photoshop-tutorials/Photoshop-CC-Web-Design/145211-2.html", "course_title": "InDesign CC: Interactive Document Fundamentals"}, +{"course_link": "/Photoshop-tutorials/Photoshop-CC-Web-Design/145211-2.html", "course_title": "Up and Running with Constant Contact"}, +{"course_link": "/Illustrator-tutorials/Package-Design-Illustrator/148421-2.html", "course_title": "Unity 4.3 Essential Training"}, +{"course_link": "/Illustrator-tutorials/Package-Design-Illustrator/148421-2.html", "course_title": "Enhancing Exterior Architectural Photos"}, +{"course_link": "/Glyphs-App-tutorials/Creating-Icon-Fonts-Web/157228-2.html", "course_title": "Synth Programming Basics"}, +{"course_link": "/Glyphs-App-tutorials/Creating-Icon-Fonts-Web/157228-2.html", "course_title": "The Creative Spark: Between Two Worlds, The Hybrid Animation of Tiny Inventions"}, +{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Family-Cabin/149289-2.html", "course_title": "Foundations of Programming: Programming for Kids"}, +{"course_link": "/Photography-Cameras-Gear-tutorials/Travel-Photography-Family-Cabin/149289-2.html", "course_title": "Persuasive UX: Ethics of Persuasive Design"}, +{"course_link": "/SQL-tutorials/SQL-Essential-Training/139988-2.html", "course_title": "Type Project: Art Nouveau Poster"}, +{"course_link": "/SQL-tutorials/SQL-Essential-Training/139988-2.html", "course_title": "Customer Service Fundamentals"}, +{"course_link": "/Illustrator-tutorials/Type-Project-Dada-Poster/155654-2.html", "course_title": "Creating Accessible PDFs"}, +{"course_link": "/Illustrator-tutorials/Type-Project-Dada-Poster/155654-2.html", "course_title": "UX Design Tools: Photoshop"}, +{"course_link": "/Premiere-Pro-tutorials/Premiere-Pro-Guru-Customizing-Workspaces/161521-2.html", "course_title": "Enhancing Interior Architectural Photos"}, +{"course_link": "/Premiere-Pro-tutorials/Premiere-Pro-Guru-Customizing-Workspaces/161521-2.html", "course_title": "Solving Business Problems"}, +{"course_link": "/Video-Shooting-Video-tutorials/Shooting-Blackmagic-Cameras/157510-2.html", "course_title": "Up and Running with Photomatix Pro"}, +{"course_link": "/Video-Shooting-Video-tutorials/Shooting-Blackmagic-Cameras/157510-2.html", "course_title": "Up and Running with Office Web Apps"}, +{"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Final Cut Pro X 10.1.1 Essential Training"}, +{"course_link": "/Digital-Publishing-Suite-tutorials/Adobe-Digital-Publishing-Suite-Essential-Training/149122-2.html", "course_title": "Direct Mail Strategy"}, +{"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Building Web Services with Java EE"}, +{"course_link": "/Visual-Studio-tutorials/Visual-Studio-2013-Windows-Store-Developers/157130-2.html", "course_title": "Web Design Fundamentals"}] \ No newline at end of file diff --git a/try_scrapy/spiders/test_spider.py b/try_scrapy/spiders/test_spider.py index b995f69..fff4ce0 100644 --- a/try_scrapy/spiders/test_spider.py +++ b/try_scrapy/spiders/test_spider.py @@ -6,17 +6,19 @@ from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.shell import inspect_response import urlparse,urllib +from scrapy import log base_url=r"http://www.lynda.com" +items=[] +log.start() class lynda_spider(CrawlSpider): name="get_lynda" # allowed_domains = ["dmoz.org"] - start_urls=[base_url+"/allcourses"] + start_urls=[urlparse.urljoin(base_url,"allcourses")] def parse(self,response): sel = Selector(response) - items=[] next_page =sel.xpath("//a[contains(@class,'see-more-results')]/@data-url").extract() if next_page: @@ -34,12 +36,12 @@ class lynda_spider(CrawlSpider): final_url=urlparse.urlunparse(parsed_url) print final_url - if int(parsed_qs["page"])<=10: + if int(parsed_qs["page"])<=4: return Request(final_url, self.parse) sel = Selector(response) course_a=sel.xpath(r'//*[@class="course-list"]//li//a') - print course_a.extract() + # print course_a.extract() titles=course_a.xpath("text()").extract() links=course_a.xpath("@href").extract() for i in range(len(titles)): @@ -47,8 +49,9 @@ class lynda_spider(CrawlSpider): tmp_item["course_title"]=titles[i].strip() tmp_item["course_link"]=links[i] items.append(tmp_item) - print titles[i].strip() + # print titles[i].strip() # print links[i] + print len(items) return items # class NextPageLinkExtractor(SgmlLinkExtractor): diff --git a/try_scrapy/spiders/test_spider.pyc b/try_scrapy/spiders/test_spider.pyc index a54e9a6..3188500 100644 Binary files a/try_scrapy/spiders/test_spider.pyc and b/try_scrapy/spiders/test_spider.pyc differ