From d2c7c4139a8078aa08127b5dd513924061d66158 Mon Sep 17 00:00:00 2001 From: bendtherules Date: Sun, 11 May 2014 10:59:51 +0530 Subject: [PATCH] Lynda single page scraping done --- output.json | 197 +++++++++++++++++++++++++++++ scrapy.cfg | 11 ++ try_scrapy/__init__.py | 0 try_scrapy/__init__.pyc | Bin 0 -> 141 bytes try_scrapy/items.py | 13 ++ try_scrapy/items.pyc | Bin 0 -> 407 bytes try_scrapy/pipelines.py | 8 ++ try_scrapy/settings.py | 15 +++ try_scrapy/settings.pyc | Bin 0 -> 257 bytes try_scrapy/spiders/__init__.py | 4 + try_scrapy/spiders/__init__.pyc | Bin 0 -> 149 bytes try_scrapy/spiders/test_spider.py | 34 +++++ try_scrapy/spiders/test_spider.pyc | Bin 0 -> 1520 bytes 13 files changed, 282 insertions(+) create mode 100644 output.json create mode 100644 scrapy.cfg create mode 100644 try_scrapy/__init__.py create mode 100644 try_scrapy/__init__.pyc create mode 100644 try_scrapy/items.py create mode 100644 try_scrapy/items.pyc create mode 100644 try_scrapy/pipelines.py create mode 100644 try_scrapy/settings.py create mode 100644 try_scrapy/settings.pyc create mode 100644 try_scrapy/spiders/__init__.py create mode 100644 try_scrapy/spiders/__init__.pyc create mode 100644 try_scrapy/spiders/test_spider.py create mode 100644 try_scrapy/spiders/test_spider.pyc diff --git a/output.json b/output.json new file mode 100644 index 0000000..887c7ef --- /dev/null +++ b/output.json @@ -0,0 +1,197 @@ +[{"course_link": "http://www.lynda.com/ASPNET-tutorials/Up-Running-ASPNET/158377-2.html", "course_title": "Up and Running with ASP.NET"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Exporting-Data-Files-PHP/158375-2.html", "course_title": "Exporting Data to Files with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Design-tutorials/Up-Running-CoffeeScript/154415-2.html", "course_title": "Up and Running with CoffeeScript"}, +{"course_link": "http://www.lynda.com/Visual-Studio-tutorials/Visual-Studio-2013-Web-Developers/157134-2.html", "course_title": "Visual Studio 2013 for Web Developers"}, +{"course_link": "http://www.lynda.com/Developer-Programming-Languages-tutorials/Building-Web-Services-Java-EE/149837-2.html", "course_title": "Building Web Services with Java EE"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Uploading Files Securely with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Foundations of Programming: Web Security"}, +{"course_link": "http://www.lynda.com/Brackets-tutorials/Adobe-Edge-Code-Brackets-First-Look/149704-2.html", "course_title": "Adobe Edge Code and Brackets: First Look"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-4-Essential-Training/139989-2.html", "course_title": "Ruby on Rails 4 Essential Training"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/PHP-Web-Designers/146234-2.html", "course_title": "PHP for Web Designers"}, +{"course_link": "http://www.lynda.com/CakePHP-tutorials/Up-Running-CakePHP/126123-2.html", "course_title": "Up and Running with CakePHP"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Choosing-PHP-Editor/139995-2.html", "course_title": "Choosing a PHP Editor"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-Events/140780-2.html", "course_title": "JavaScript: Events"}, +{"course_link": "http://www.lynda.com/Joomla-tutorials/Joomla-3-Essential-Training/110275-2.html", "course_title": "Joomla! 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Shopify-tutorials/Up-Running-Shopify/133352-2.html", "course_title": "Up and Running with Shopify"}, +{"course_link": "http://www.lynda.com/OData-tutorials/Foundations-Programming-Web-Services/126131-2.html", "course_title": "Foundations of Programming: Web Services"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/Working-Data-Web/133326-2.html", "course_title": "Working with Data on the Web"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure/128285-2.html", "course_title": "Up and Running with Azure"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure-Mobile-Services/135357-2.html", "course_title": "Up and Running with Azure Mobile Services"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-3-Advanced-Web-Development/124079-2.html", "course_title": "Bootstrap 3: Advanced Web Development"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-3-Essential-Training/55960-2.html", "course_title": "Ruby on Rails 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Emmet-tutorials/Coding-Faster-Emmet/133353-2.html", "course_title": "Coding Faster with Emmet"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Interactive-Animated-Timeline-jQuery/124092-2.html", "course_title": "Create an Interactive Animated Timeline with jQuery"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/Start-Theme-Food-Blogs-WordPress/133348-2.html", "course_title": "Start with a Theme: Food Blogs in WordPress"}, +{"course_link": "http://www.lynda.com/CodeIgniter-tutorials/Up-Running-PHP-CodeIgniter/126122-2.html", "course_title": "Up and Running with PHP CodeIgniter"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Creating-Responsive-Presentation/129007-2.html", "course_title": "HTML5 Projects: Creating a Responsive Presentation"}, +{"course_link": "http://www.lynda.com/Dreamweaver-tutorials/Dreamweaver-CC-Essential-Training/124084-2.html", "course_title": "Dreamweaver CC Essential Training"}, +{"course_link": "http://www.lynda.com/HTML-tutorials/JavaScript-Enhancing-DOM/122462-2.html", "course_title": "JavaScript: Enhancing the DOM"}, +{"course_link": "http://www.lynda.com/MySQL-tutorials/PHP-MySQL-Essential-Training/119003-2.html", "course_title": "PHP with MySQL Essential Training"}, +{"course_link": "http://www.lynda.com/Composer-tutorials/Up-Running-PHP-PEAR-PECL-Composer/119001-2.html", "course_title": "Up and Running with PHP: PEAR, PECL, and Composer"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/jQuery-Essential-Training/48370-2.html", "course_title": "jQuery Essential Training"}, +{"course_link": "http://www.lynda.com/Foundation-tutorials/Up-Running-Foundation/122442-2.html", "course_title": "Up and Running with Foundation"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-Adding-Interactivity-Your-Site/122456-2.html", "course_title": "Bootstrap: Adding Interactivity to Your Site"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Easy-Digital-Downloads/122465-2.html", "course_title": "WordPress Ecommerce: Easy Digital Downloads"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Animated-Infographic-Edge-Animate/124091-2.html", "course_title": "Creating an Animated Infographic with Edge Animate"}, +{"course_link": "http://www.lynda.com/Developer-Databases-tutorials/Validating-Processing-Forms-JavaScript-PHP/120466-2.html", "course_title": "Validating and Processing Forms with JavaScript and PHP"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Core-Concepts/122463-2.html", "course_title": "WordPress Ecommerce: Core Concepts"}, +{"course_link": "http://www.lynda.com/Backbonejs-tutorials/Up-Running-Backbonejs/120900-2.html", "course_title": "Up and Running with Backbone.js"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Interactive-Projects-Edge-Animate/124307-2.html", "course_title": "Creating Interactive Projects with Edge Animate"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Sortable-Photo-Gallery-jQuery/114897-2.html", "course_title": "Create a Sortable Photo Gallery with jQuery"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-JSON/114901-2.html", "course_title": "JavaScript and JSON"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Create-Homepage-Marquee-Edge-Animate/114898-2.html", "course_title": "Create a Homepage Marquee with Edge Animate"}, +{"course_link": "http://www.lynda.com/Visual-Studio-Express-tutorials/Building-Windows-Store-Game-Using-HTML-JavaScript/114892-2.html", "course_title": "Building a Windows Store Game Using HTML and JavaScript"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Geolocation/118953-2.html", "course_title": "HTML5 Projects: Integrating Google Store Locator"}, +{"course_link": "http://www.lynda.com/ASP-NET-3-5-tutorials/asp-net-essential-training/784-2.html", "course_title": "ASP.NET Essential Training"}, +{"course_link": "http://www.lynda.com/Sublime-Text-tutorials/Up-Running-Sublime-Text-2/114325-2.html", "course_title": "Up and Running with Sublime Text 2"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "CSS: Frameworks and Grids"}, +{"course_link": "http://www.lynda.com/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "JavaScript and AJAX"}, +{"course_link": "http://www.lynda.com/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, +{"course_link": "http://www.lynda.com/Drupal-tutorials/Responsive-Design-Drupal/107419-2.html", "course_title": "Responsive Design with Drupal"}][{"course_link": "http://www.lynda.com/ASPNET-tutorials/Up-Running-ASPNET/158377-2.html", "course_title": "Up and Running with ASP.NET"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Exporting-Data-Files-PHP/158375-2.html", "course_title": "Exporting Data to Files with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Design-tutorials/Up-Running-CoffeeScript/154415-2.html", "course_title": "Up and Running with CoffeeScript"}, +{"course_link": "http://www.lynda.com/Visual-Studio-tutorials/Visual-Studio-2013-Web-Developers/157134-2.html", "course_title": "Visual Studio 2013 for Web Developers"}, +{"course_link": "http://www.lynda.com/Developer-Programming-Languages-tutorials/Building-Web-Services-Java-EE/149837-2.html", "course_title": "Building Web Services with Java EE"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Uploading Files Securely with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Foundations of Programming: Web Security"}, +{"course_link": "http://www.lynda.com/Brackets-tutorials/Adobe-Edge-Code-Brackets-First-Look/149704-2.html", "course_title": "Adobe Edge Code and Brackets: First Look"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-4-Essential-Training/139989-2.html", "course_title": "Ruby on Rails 4 Essential Training"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/PHP-Web-Designers/146234-2.html", "course_title": "PHP for Web Designers"}, +{"course_link": "http://www.lynda.com/CakePHP-tutorials/Up-Running-CakePHP/126123-2.html", "course_title": "Up and Running with CakePHP"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Choosing-PHP-Editor/139995-2.html", "course_title": "Choosing a PHP Editor"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-Events/140780-2.html", "course_title": "JavaScript: Events"}, +{"course_link": "http://www.lynda.com/Joomla-tutorials/Joomla-3-Essential-Training/110275-2.html", "course_title": "Joomla! 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Shopify-tutorials/Up-Running-Shopify/133352-2.html", "course_title": "Up and Running with Shopify"}, +{"course_link": "http://www.lynda.com/OData-tutorials/Foundations-Programming-Web-Services/126131-2.html", "course_title": "Foundations of Programming: Web Services"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/Working-Data-Web/133326-2.html", "course_title": "Working with Data on the Web"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure/128285-2.html", "course_title": "Up and Running with Azure"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure-Mobile-Services/135357-2.html", "course_title": "Up and Running with Azure Mobile Services"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-3-Advanced-Web-Development/124079-2.html", "course_title": "Bootstrap 3: Advanced Web Development"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-3-Essential-Training/55960-2.html", "course_title": "Ruby on Rails 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Emmet-tutorials/Coding-Faster-Emmet/133353-2.html", "course_title": "Coding Faster with Emmet"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Interactive-Animated-Timeline-jQuery/124092-2.html", "course_title": "Create an Interactive Animated Timeline with jQuery"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/Start-Theme-Food-Blogs-WordPress/133348-2.html", "course_title": "Start with a Theme: Food Blogs in WordPress"}, +{"course_link": "http://www.lynda.com/CodeIgniter-tutorials/Up-Running-PHP-CodeIgniter/126122-2.html", "course_title": "Up and Running with PHP CodeIgniter"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Creating-Responsive-Presentation/129007-2.html", "course_title": "HTML5 Projects: Creating a Responsive Presentation"}, +{"course_link": "http://www.lynda.com/Dreamweaver-tutorials/Dreamweaver-CC-Essential-Training/124084-2.html", "course_title": "Dreamweaver CC Essential Training"}, +{"course_link": "http://www.lynda.com/HTML-tutorials/JavaScript-Enhancing-DOM/122462-2.html", "course_title": "JavaScript: Enhancing the DOM"}, +{"course_link": "http://www.lynda.com/MySQL-tutorials/PHP-MySQL-Essential-Training/119003-2.html", "course_title": "PHP with MySQL Essential Training"}, +{"course_link": "http://www.lynda.com/Composer-tutorials/Up-Running-PHP-PEAR-PECL-Composer/119001-2.html", "course_title": "Up and Running with PHP: PEAR, PECL, and Composer"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/jQuery-Essential-Training/48370-2.html", "course_title": "jQuery Essential Training"}, +{"course_link": "http://www.lynda.com/Foundation-tutorials/Up-Running-Foundation/122442-2.html", "course_title": "Up and Running with Foundation"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-Adding-Interactivity-Your-Site/122456-2.html", "course_title": "Bootstrap: Adding Interactivity to Your Site"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Easy-Digital-Downloads/122465-2.html", "course_title": "WordPress Ecommerce: Easy Digital Downloads"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Animated-Infographic-Edge-Animate/124091-2.html", "course_title": "Creating an Animated Infographic with Edge Animate"}, +{"course_link": "http://www.lynda.com/Developer-Databases-tutorials/Validating-Processing-Forms-JavaScript-PHP/120466-2.html", "course_title": "Validating and Processing Forms with JavaScript and PHP"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Core-Concepts/122463-2.html", "course_title": "WordPress Ecommerce: Core Concepts"}, +{"course_link": "http://www.lynda.com/Backbonejs-tutorials/Up-Running-Backbonejs/120900-2.html", "course_title": "Up and Running with Backbone.js"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Interactive-Projects-Edge-Animate/124307-2.html", "course_title": "Creating Interactive Projects with Edge Animate"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Sortable-Photo-Gallery-jQuery/114897-2.html", "course_title": "Create a Sortable Photo Gallery with jQuery"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-JSON/114901-2.html", "course_title": "JavaScript and JSON"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Create-Homepage-Marquee-Edge-Animate/114898-2.html", "course_title": "Create a Homepage Marquee with Edge Animate"}, +{"course_link": "http://www.lynda.com/Visual-Studio-Express-tutorials/Building-Windows-Store-Game-Using-HTML-JavaScript/114892-2.html", "course_title": "Building a Windows Store Game Using HTML and JavaScript"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Geolocation/118953-2.html", "course_title": "HTML5 Projects: Integrating Google Store Locator"}, +{"course_link": "http://www.lynda.com/ASP-NET-3-5-tutorials/asp-net-essential-training/784-2.html", "course_title": "ASP.NET Essential Training"}, +{"course_link": "http://www.lynda.com/Sublime-Text-tutorials/Up-Running-Sublime-Text-2/114325-2.html", "course_title": "Up and Running with Sublime Text 2"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "CSS: Frameworks and Grids"}, +{"course_link": "http://www.lynda.com/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "JavaScript and AJAX"}, +{"course_link": "http://www.lynda.com/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, +{"course_link": "http://www.lynda.com/Drupal-tutorials/Responsive-Design-Drupal/107419-2.html", "course_title": "Responsive Design with Drupal"}][{"course_link": "http://www.lynda.com/ASPNET-tutorials/Up-Running-ASPNET/158377-2.html", "course_title": "Up and Running with ASP.NET"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Exporting-Data-Files-PHP/158375-2.html", "course_title": "Exporting Data to Files with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Design-tutorials/Up-Running-CoffeeScript/154415-2.html", "course_title": "Up and Running with CoffeeScript"}, +{"course_link": "http://www.lynda.com/Visual-Studio-tutorials/Visual-Studio-2013-Web-Developers/157134-2.html", "course_title": "Visual Studio 2013 for Web Developers"}, +{"course_link": "http://www.lynda.com/Developer-Programming-Languages-tutorials/Building-Web-Services-Java-EE/149837-2.html", "course_title": "Building Web Services with Java EE"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Uploading Files Securely with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Foundations of Programming: Web Security"}, +{"course_link": "http://www.lynda.com/Brackets-tutorials/Adobe-Edge-Code-Brackets-First-Look/149704-2.html", "course_title": "Adobe Edge Code and Brackets: First Look"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-4-Essential-Training/139989-2.html", "course_title": "Ruby on Rails 4 Essential Training"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/PHP-Web-Designers/146234-2.html", "course_title": "PHP for Web Designers"}, +{"course_link": "http://www.lynda.com/CakePHP-tutorials/Up-Running-CakePHP/126123-2.html", "course_title": "Up and Running with CakePHP"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Choosing-PHP-Editor/139995-2.html", "course_title": "Choosing a PHP Editor"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-Events/140780-2.html", "course_title": "JavaScript: Events"}, +{"course_link": "http://www.lynda.com/Joomla-tutorials/Joomla-3-Essential-Training/110275-2.html", "course_title": "Joomla! 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Shopify-tutorials/Up-Running-Shopify/133352-2.html", "course_title": "Up and Running with Shopify"}, +{"course_link": "http://www.lynda.com/OData-tutorials/Foundations-Programming-Web-Services/126131-2.html", "course_title": "Foundations of Programming: Web Services"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/Working-Data-Web/133326-2.html", "course_title": "Working with Data on the Web"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure/128285-2.html", "course_title": "Up and Running with Azure"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure-Mobile-Services/135357-2.html", "course_title": "Up and Running with Azure Mobile Services"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-3-Advanced-Web-Development/124079-2.html", "course_title": "Bootstrap 3: Advanced Web Development"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-3-Essential-Training/55960-2.html", "course_title": "Ruby on Rails 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Emmet-tutorials/Coding-Faster-Emmet/133353-2.html", "course_title": "Coding Faster with Emmet"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Interactive-Animated-Timeline-jQuery/124092-2.html", "course_title": "Create an Interactive Animated Timeline with jQuery"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/Start-Theme-Food-Blogs-WordPress/133348-2.html", "course_title": "Start with a Theme: Food Blogs in WordPress"}, +{"course_link": "http://www.lynda.com/CodeIgniter-tutorials/Up-Running-PHP-CodeIgniter/126122-2.html", "course_title": "Up and Running with PHP CodeIgniter"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Creating-Responsive-Presentation/129007-2.html", "course_title": "HTML5 Projects: Creating a Responsive Presentation"}, +{"course_link": "http://www.lynda.com/Dreamweaver-tutorials/Dreamweaver-CC-Essential-Training/124084-2.html", "course_title": "Dreamweaver CC Essential Training"}, +{"course_link": "http://www.lynda.com/HTML-tutorials/JavaScript-Enhancing-DOM/122462-2.html", "course_title": "JavaScript: Enhancing the DOM"}, +{"course_link": "http://www.lynda.com/MySQL-tutorials/PHP-MySQL-Essential-Training/119003-2.html", "course_title": "PHP with MySQL Essential Training"}, +{"course_link": "http://www.lynda.com/Composer-tutorials/Up-Running-PHP-PEAR-PECL-Composer/119001-2.html", "course_title": "Up and Running with PHP: PEAR, PECL, and Composer"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/jQuery-Essential-Training/48370-2.html", "course_title": "jQuery Essential Training"}, +{"course_link": "http://www.lynda.com/Foundation-tutorials/Up-Running-Foundation/122442-2.html", "course_title": "Up and Running with Foundation"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-Adding-Interactivity-Your-Site/122456-2.html", "course_title": "Bootstrap: Adding Interactivity to Your Site"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Easy-Digital-Downloads/122465-2.html", "course_title": "WordPress Ecommerce: Easy Digital Downloads"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Animated-Infographic-Edge-Animate/124091-2.html", "course_title": "Creating an Animated Infographic with Edge Animate"}, +{"course_link": "http://www.lynda.com/Developer-Databases-tutorials/Validating-Processing-Forms-JavaScript-PHP/120466-2.html", "course_title": "Validating and Processing Forms with JavaScript and PHP"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Core-Concepts/122463-2.html", "course_title": "WordPress Ecommerce: Core Concepts"}, +{"course_link": "http://www.lynda.com/Backbonejs-tutorials/Up-Running-Backbonejs/120900-2.html", "course_title": "Up and Running with Backbone.js"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Interactive-Projects-Edge-Animate/124307-2.html", "course_title": "Creating Interactive Projects with Edge Animate"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Sortable-Photo-Gallery-jQuery/114897-2.html", "course_title": "Create a Sortable Photo Gallery with jQuery"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-JSON/114901-2.html", "course_title": "JavaScript and JSON"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Create-Homepage-Marquee-Edge-Animate/114898-2.html", "course_title": "Create a Homepage Marquee with Edge Animate"}, +{"course_link": "http://www.lynda.com/Visual-Studio-Express-tutorials/Building-Windows-Store-Game-Using-HTML-JavaScript/114892-2.html", "course_title": "Building a Windows Store Game Using HTML and JavaScript"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Geolocation/118953-2.html", "course_title": "HTML5 Projects: Integrating Google Store Locator"}, +{"course_link": "http://www.lynda.com/ASP-NET-3-5-tutorials/asp-net-essential-training/784-2.html", "course_title": "ASP.NET Essential Training"}, +{"course_link": "http://www.lynda.com/Sublime-Text-tutorials/Up-Running-Sublime-Text-2/114325-2.html", "course_title": "Up and Running with Sublime Text 2"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "CSS: Frameworks and Grids"}, +{"course_link": "http://www.lynda.com/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "JavaScript and AJAX"}, +{"course_link": "http://www.lynda.com/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, +{"course_link": "http://www.lynda.com/Drupal-tutorials/Responsive-Design-Drupal/107419-2.html", "course_title": "Responsive Design with Drupal"}][{"course_link": "http://www.lynda.com/ASPNET-tutorials/Up-Running-ASPNET/158377-2.html", "course_title": "Up and Running with ASP.NET"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Exporting-Data-Files-PHP/158375-2.html", "course_title": "Exporting Data to Files with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Design-tutorials/Up-Running-CoffeeScript/154415-2.html", "course_title": "Up and Running with CoffeeScript"}, +{"course_link": "http://www.lynda.com/Visual-Studio-tutorials/Visual-Studio-2013-Web-Developers/157134-2.html", "course_title": "Visual Studio 2013 for Web Developers"}, +{"course_link": "http://www.lynda.com/Developer-Programming-Languages-tutorials/Building-Web-Services-Java-EE/149837-2.html", "course_title": "Building Web Services with Java EE"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Uploading-Files-Securely-PHP/158374-2.html", "course_title": "Uploading Files Securely with PHP"}, +{"course_link": "http://www.lynda.com/Developer-Web-Development-tutorials/Foundations-Programming-Web-Security/133330-2.html", "course_title": "Foundations of Programming: Web Security"}, +{"course_link": "http://www.lynda.com/Brackets-tutorials/Adobe-Edge-Code-Brackets-First-Look/149704-2.html", "course_title": "Adobe Edge Code and Brackets: First Look"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-4-Essential-Training/139989-2.html", "course_title": "Ruby on Rails 4 Essential Training"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/PHP-Web-Designers/146234-2.html", "course_title": "PHP for Web Designers"}, +{"course_link": "http://www.lynda.com/CakePHP-tutorials/Up-Running-CakePHP/126123-2.html", "course_title": "Up and Running with CakePHP"}, +{"course_link": "http://www.lynda.com/PHP-tutorials/Choosing-PHP-Editor/139995-2.html", "course_title": "Choosing a PHP Editor"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-Events/140780-2.html", "course_title": "JavaScript: Events"}, +{"course_link": "http://www.lynda.com/Joomla-tutorials/Joomla-3-Essential-Training/110275-2.html", "course_title": "Joomla! 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Shopify-tutorials/Up-Running-Shopify/133352-2.html", "course_title": "Up and Running with Shopify"}, +{"course_link": "http://www.lynda.com/OData-tutorials/Foundations-Programming-Web-Services/126131-2.html", "course_title": "Foundations of Programming: Web Services"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/Working-Data-Web/133326-2.html", "course_title": "Working with Data on the Web"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure/128285-2.html", "course_title": "Up and Running with Azure"}, +{"course_link": "http://www.lynda.com/Azure-tutorials/Up-Running-Azure-Mobile-Services/135357-2.html", "course_title": "Up and Running with Azure Mobile Services"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-3-Advanced-Web-Development/124079-2.html", "course_title": "Bootstrap 3: Advanced Web Development"}, +{"course_link": "http://www.lynda.com/Ruby-Rails-tutorials/Ruby-Rails-3-Essential-Training/55960-2.html", "course_title": "Ruby on Rails 3 Essential Training"}, +{"course_link": "http://www.lynda.com/Emmet-tutorials/Coding-Faster-Emmet/133353-2.html", "course_title": "Coding Faster with Emmet"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Interactive-Animated-Timeline-jQuery/124092-2.html", "course_title": "Create an Interactive Animated Timeline with jQuery"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/Start-Theme-Food-Blogs-WordPress/133348-2.html", "course_title": "Start with a Theme: Food Blogs in WordPress"}, +{"course_link": "http://www.lynda.com/CodeIgniter-tutorials/Up-Running-PHP-CodeIgniter/126122-2.html", "course_title": "Up and Running with PHP CodeIgniter"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Creating-Responsive-Presentation/129007-2.html", "course_title": "HTML5 Projects: Creating a Responsive Presentation"}, +{"course_link": "http://www.lynda.com/Dreamweaver-tutorials/Dreamweaver-CC-Essential-Training/124084-2.html", "course_title": "Dreamweaver CC Essential Training"}, +{"course_link": "http://www.lynda.com/HTML-tutorials/JavaScript-Enhancing-DOM/122462-2.html", "course_title": "JavaScript: Enhancing the DOM"}, +{"course_link": "http://www.lynda.com/MySQL-tutorials/PHP-MySQL-Essential-Training/119003-2.html", "course_title": "PHP with MySQL Essential Training"}, +{"course_link": "http://www.lynda.com/Composer-tutorials/Up-Running-PHP-PEAR-PECL-Composer/119001-2.html", "course_title": "Up and Running with PHP: PEAR, PECL, and Composer"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/jQuery-Essential-Training/48370-2.html", "course_title": "jQuery Essential Training"}, +{"course_link": "http://www.lynda.com/Foundation-tutorials/Up-Running-Foundation/122442-2.html", "course_title": "Up and Running with Foundation"}, +{"course_link": "http://www.lynda.com/Bootstrap-tutorials/Bootstrap-Adding-Interactivity-Your-Site/122456-2.html", "course_title": "Bootstrap: Adding Interactivity to Your Site"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Easy-Digital-Downloads/122465-2.html", "course_title": "WordPress Ecommerce: Easy Digital Downloads"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Animated-Infographic-Edge-Animate/124091-2.html", "course_title": "Creating an Animated Infographic with Edge Animate"}, +{"course_link": "http://www.lynda.com/Developer-Databases-tutorials/Validating-Processing-Forms-JavaScript-PHP/120466-2.html", "course_title": "Validating and Processing Forms with JavaScript and PHP"}, +{"course_link": "http://www.lynda.com/WordPress-tutorials/WordPress-Ecommerce-Core-Concepts/122463-2.html", "course_title": "WordPress Ecommerce: Core Concepts"}, +{"course_link": "http://www.lynda.com/Backbonejs-tutorials/Up-Running-Backbonejs/120900-2.html", "course_title": "Up and Running with Backbone.js"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Creating-Interactive-Projects-Edge-Animate/124307-2.html", "course_title": "Creating Interactive Projects with Edge Animate"}, +{"course_link": "http://www.lynda.com/jQuery-tutorials/Create-Sortable-Photo-Gallery-jQuery/114897-2.html", "course_title": "Create a Sortable Photo Gallery with jQuery"}, +{"course_link": "http://www.lynda.com/JavaScript-tutorials/JavaScript-JSON/114901-2.html", "course_title": "JavaScript and JSON"}, +{"course_link": "http://www.lynda.com/Edge-Animate-tutorials/Create-Homepage-Marquee-Edge-Animate/114898-2.html", "course_title": "Create a Homepage Marquee with Edge Animate"}, +{"course_link": "http://www.lynda.com/Visual-Studio-Express-tutorials/Building-Windows-Store-Game-Using-HTML-JavaScript/114892-2.html", "course_title": "Building a Windows Store Game Using HTML and JavaScript"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/HTML5-Projects-Geolocation/118953-2.html", "course_title": "HTML5 Projects: Integrating Google Store Locator"}, +{"course_link": "http://www.lynda.com/ASP-NET-3-5-tutorials/asp-net-essential-training/784-2.html", "course_title": "ASP.NET Essential Training"}, +{"course_link": "http://www.lynda.com/Sublime-Text-tutorials/Up-Running-Sublime-Text-2/114325-2.html", "course_title": "Up and Running with Sublime Text 2"}, +{"course_link": "http://www.lynda.com/CSS-tutorials/CSS-Frameworks-Grids/112675-2.html", "course_title": "CSS: Frameworks and Grids"}, +{"course_link": "http://www.lynda.com/Developer-tutorials/JavaScript-and-AJAX/114900-2.html", "course_title": "JavaScript and AJAX"}, +{"course_link": "http://www.lynda.com/Facebook-tutorials/Building-Facebook-Applications-PHP-MySQL/107060-2.html", "course_title": "Building Facebook Applications with PHP and MySQL"}, +{"course_link": "http://www.lynda.com/Drupal-tutorials/Responsive-Design-Drupal/107419-2.html", "course_title": "Responsive Design with Drupal"}] \ No newline at end of file diff --git a/scrapy.cfg b/scrapy.cfg new file mode 100644 index 0000000..b2970f1 --- /dev/null +++ b/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# http://doc.scrapy.org/en/latest/topics/scrapyd.html + +[settings] +default = try_scrapy.settings + +[deploy] +#url = http://localhost:6800/ +project = try_scrapy diff --git a/try_scrapy/__init__.py b/try_scrapy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/try_scrapy/__init__.pyc b/try_scrapy/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ba95c06cc62f061b817d9dc38794fad84bb7f97 GIT binary patch literal 141 zcmZSn%*$1jlNX%K00oRd+5w1*S%5?e14FO|NW@PANHCxg#b!V;SF0GO#H7-g^vsfs z(xmvb%$(HXn3AH(_~PWE#DYp>W_)~RUS>&ryk0?N2?tQCO>TZlX-=vg$bw=ZW&i*` C_#e;! literal 0 HcmV?d00001 diff --git a/try_scrapy/items.py b/try_scrapy/items.py new file mode 100644 index 0000000..2bc89b8 --- /dev/null +++ b/try_scrapy/items.py @@ -0,0 +1,13 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/en/latest/topics/items.html + +from scrapy.item import Item, Field + + +class LyndaItem(Item): + # define the fields for your item here like: + # name = Field() + course_title = Field() + course_link = Field() diff --git a/try_scrapy/items.pyc b/try_scrapy/items.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d439d1d563f5b4f8e27c6c3cab6fb58a9d6ea5d GIT binary patch literal 407 zcma)1K}y6h6n$yx;EcKwTzG=61`i-2E(Aebv@6R%C{2W5+Zmcai>~!H9?C;_0DVaZ z-5QdgmzV$F`(J;Ztv)`7s>NYx-nTsAiotR#bU?3AEA$!a49aG33NH#);~jVdo@))8 zqbiPA$&lp1rINMT|K)?fz8mT-ZH zqRRN<iDu3m9LW=d*Nu^&V&2T;<S&~tj6rYxvlUf{ua0D{5IJKlCGcUbZ cub{F7WW7ypeoARhsvRegUCaU`*qAsO0Zqd_s{jB1 literal 0 HcmV?d00001 diff --git a/try_scrapy/spiders/__init__.py b/try_scrapy/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/try_scrapy/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/try_scrapy/spiders/__init__.pyc b/try_scrapy/spiders/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7787c1e61acb551b8f1f5a966fd4b433dbe63b1f GIT binary patch literal 149 zcmZSn%*$1jlNX%K00oRd+5w1*S%5?e14FO|NW@PANHCxg#dbh3SF0GO#H7-g^vsfs z(xmvb%$(HXn3AH(_~PWE#DYp>W^qAgN@`JYOniK1US>&ryk0?N2?x*!o80`A(wtN~ KkVVBn%m4s<<0CEr literal 0 HcmV?d00001 diff --git a/try_scrapy/spiders/test_spider.py b/try_scrapy/spiders/test_spider.py new file mode 100644 index 0000000..b7e9231 --- /dev/null +++ b/try_scrapy/spiders/test_spider.py @@ -0,0 +1,34 @@ +from scrapy.spider import Spider +from scrapy.selector import Selector +from try_scrapy.items import LyndaItem +from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor + +class lynda_spider(CrawlSpider): + name="get_lynda" + # allowed_domains = ["dmoz.org"] + start_urls=[r"file:///C:/Users/b/Downloads/Learn%20Web%20Development%20with%20Video%20Courses%20and%20Tutorials%20from%20lynda.com.htm"] + rules = ( + Rule(SgmlLinkExtractor(attrs=["href","data-url"],allow=[r".*?page=\d+.*"])), + ) + + def parse(self,response): + sel = Selector(response) + course_a=sel.xpath(r'//*[@class="course-list"]//li//*[@class="details-row"]/h3/a') + # print course_a.extract() + titles=course_a.xpath("text()").extract() + links=course_a.xpath("@href").extract() + items=[] + for i in range(len(course_a)): + tmp_item=LyndaItem() + tmp_item["course_title"]=titles[i].strip() + tmp_item["course_link"]=links[i] + items.append(tmp_item) + # print titles[i].strip() + # print links[i] + return items + +# class NextPageLinkExtractor(SgmlLinkExtractor): +# # allow_domains=[r"http://www.lynda.com/"] +# attrs=["href","data-url"] +# allow=[r".*?page=\d+.*"] diff --git a/try_scrapy/spiders/test_spider.pyc b/try_scrapy/spiders/test_spider.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9db007e3de482b13ffd122cf629ae154297789a GIT binary patch literal 1520 zcmbVM&2HO95S}GPN_LXQ0h$0ofB-d6KrLXgXm3es7`1x{VB|n04az1Z*5XQ{yZp({ zxRwF=ls-z|qYu>wXl8cVBtTE8Brf;+c4v0xn_0zw-`f2v>gEd=J~6+K_`#lsihl|$ zK%<~gpae%?BWPkM;(%kYBWMyRl7L5GQ)tFejG>u8F$rt}b_beWD0UgAV8*aiP>taa zfVTiY7dN1qz*t=r#z+>q1ai*7P&#Z8I+xc71-yzm1*c*uhRQGrD$BGF~V zqQqcEU=op~ilG|8D&mCHTP67wDO8E1$Xev{sboVE@XAo>Ps0tPWo^x)EX$4`WiQY; z%+9it?y|L=t}r__+O=PP`^`&p#^Z_k-Po>gOiPR|Yg+T`C)TIq_i=aO&>+urTk-h( zf-|eN6=vzWhF|{B*`jM^YidYx(?kxvL~Cc3C<0fS^udL*3sFd5Q1m~O=OP1KkNzd&kH`d8yp+Nw2a16YXG1pO-^uGkxE=#XCdL!k z^4b{gpQnPi6Vdm{Hu0RGgi#9TV_9Nr_2yT2$wW~489wH4v+V0%juuuUe!su)0v*^I zY5#SW+4|$f%8;%t9=Pt37uVlq8hyRYhNfSkNOdF%`A!LrN~*~k-);k8xu>QgVeY$E zj5fWdHH~@3j-=VV>Q1-Z12MDKw7KtE*^K1s-Z#DOd(&3_I{4zCu}c!q&S2lQ7>t?q zZ`b^!4y$t@SvTSu@%93_@Z&@~7#dndRbd{`l z9!WbYx#=($J{L8DQoI@S2Mj-i`U&yoQbSSScXq1MXeW9YKby*Zr(44NY-b_coCzSG z*+38fHKCq+vie@3nb$NFPjnbOExMMgcs6*%T#D>-qW%AFk^&8V(K(zU|1bH}ec1mi WVK4cUAo8Kq8%6d#CEt5$FM0