{"_id":"js-crawler","_rev":"36-7c69882b7a1a56bacfa02b70c907897a","name":"js-crawler","description":"Web crawler for Node.js","dist-tags":{"latest":"0.3.21"},"versions":{"0.1.0":{"name":"js-crawler","version":"0.1.0","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.33.0","underscore":"~1.6.0"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","_id":"js-crawler@0.1.0","dist":{"shasum":"b4cb9e4b5a8e68d6a7769eae8bb879e1f3ef2cba","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.1.0.tgz","integrity":"sha512-2TGsVyz2oRr1tieJz+KDVDChhXH4nr2DxNPM8DdLaE5fRky7nIzpbqLIiWk3t9JDWtcQZbt3rFjTtmYFAPqRMQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCICiY8pplnUyPP8H78YoQCBAQF5rdFs1rIvOLRjD3x9EZAiEAjE8rFMQzXJ8vkBD1dJnvwjLKX6DNYKaenWtwCe/jQqg="}]},"_from":".","_npmVersion":"1.3.21","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.2.0":{"name":"js-crawler","version":"0.2.0","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.33.0","underscore":"~1.6.0"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"c60b87956d4d49ae03f2b976e8937e9d6d6f86ef","_id":"js-crawler@0.2.0","_shasum":"3d6c8596bfa861c2dca0180b00df2fb812ec9e43","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"3d6c8596bfa861c2dca0180b00df2fb812ec9e43","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.2.0.tgz","integrity":"sha512-CKCMovuA+xqlBidP0kBK6N0BUafSOvtYhz0xaJo3yatlFm3SUBOG3a2RI4lVEMgvOWn3JDLWwoC1CcE6ndEasQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGmyQFmyFJ6KRIrk/+sh8yo4DD6/7AzzMc5TamNRkcJwAiEA0Z8Gv/aN+HhjLWig7+Oml9d2h1rFit2CtqfWQQyzJ4Y="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.0":{"name":"js-crawler","version":"0.3.0","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"86b16874702bf539c3df6ff97fad0bddaa0aecca","_id":"js-crawler@0.3.0","_shasum":"121db93ddb6439be86f7c74596b726f808bc1682","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"121db93ddb6439be86f7c74596b726f808bc1682","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.0.tgz","integrity":"sha512-FAykI5Lz0tl2Co0WJpDTFB4eJqA9jhN4gTRlzzgzSJhqf8Iz321mgbEo/pnF2a4dWqDcXQOYAIiDkWP/SGlFuw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCxkHwTOf98IbRrxK2XZ//XA0vqw1AeC4hmTIIAdJQo2QIgZ/aDaGuRVlHrNQCwoVs+GSm6BrOwjuZAe6BuI4yKjOE="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.1":{"name":"js-crawler","version":"0.3.1","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"fbc7e4dbc68b0c2436b210acbb1c8d7b5d611014","_id":"js-crawler@0.3.1","_shasum":"10cfcc7a605125865906b696d1d154157eea55c6","_from":".","_npmVersion":"2.7.4","_nodeVersion":"0.12.2","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"10cfcc7a605125865906b696d1d154157eea55c6","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.1.tgz","integrity":"sha512-RrDylOmPo3HmtKSrUXOTxkx4R1i53DdRq1XfMndpi1p9FQBmx4n3SincA8fIFpibhDUEGBTRjZ6s/gOazjRvVA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDp1adGyrp6JcFGYU1Bc33+aKg90mMehQrovY/EBUh4hAiEAsfQIsN+ZkAaKKcYjIZXxWUWVzClryNkodcEmfjByWhY="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.2":{"name":"js-crawler","version":"0.3.2","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"0d25697c35b7c6cf09b9b1fbc171a694f73ed267","_id":"js-crawler@0.3.2","_shasum":"949d3f09b5027df475e9c18ca88e39c8aa1faf3f","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"949d3f09b5027df475e9c18ca88e39c8aa1faf3f","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.2.tgz","integrity":"sha512-I/jKXPCwozN+QdOUdLjYciA9J0QqyLoXHTjH7Qaz5AVgw9AVD4ArsFWfdczE5I6Zcr70srlr5KNb3Yn2QSDXKA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIF27VUuu1WnHpcY2e5NZfp32T9dIy5mHESVBJ+KU5N5RAiAzawpsbKAr26NhZSxzcsPUayWb4s4Gf1QJekf3XC5aAQ=="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.3":{"name":"js-crawler","version":"0.3.3","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"ae98f0c180722736e8cf015377ffb017dfddca99","_id":"js-crawler@0.3.3","_shasum":"cee52afd2959720ac5ad0008282c13323154c026","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"cee52afd2959720ac5ad0008282c13323154c026","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.3.tgz","integrity":"sha512-dk9JBdo2LGjPUhoqiPSvRyL6ZFHGRryYu7dijgAxHWECqxjFeJvxEjdeS0gDG/DJ+OXp3ExjWEn4cqy2G0gTCg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCJrBE1S5cjZpzPTXHjnmAE1erAiwctWW+EE+dhYxJDhAIhALEiUAKuL4OHbRO0ynE+TMvbVToUUccdTUgfIi9ZXVda"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.4":{"name":"js-crawler","version":"0.3.4","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"a7bccb756c381bf01ca8a50900cb8ef4184d5641","_id":"js-crawler@0.3.4","_shasum":"35725fb391880c5b1d050b18973afbe1ee0e2b4d","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"35725fb391880c5b1d050b18973afbe1ee0e2b4d","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.4.tgz","integrity":"sha512-RzvkmW2eXjKDAIcSK50DM7LYZT5O4pFDQ3NGL5/PsMcQdBHJEPAZHuWteF5YvhG4DW8lYxnsKzo/eBlXvVEi1g==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDC7+vH5rlmlspEiIQ8rRvpukYhtLSgyB/GxsEFVS8SIwIhAImIIkumuGSn2DfyjSWH7X+4b4wLtU65jtWx3ZpBWoSd"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.5":{"name":"js-crawler","version":"0.3.5","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"1fca4b8fc95258dd76865affbe4cd6f3234d8bc1","_id":"js-crawler@0.3.5","_shasum":"08131e120a5048ec2f693f6fb0a05afd76f95ef5","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"08131e120a5048ec2f693f6fb0a05afd76f95ef5","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.5.tgz","integrity":"sha512-8FjrAKOoKyOWlcdSohrSY7iIQSk7997NG7gXzNleand8CRYJawfyniQej4skW6laz7MaNsWtOZtyyORBQpi3cQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDTrYuzzd+nvfQNa3D9ixLyNeLfKIaCcw8vWEZeALPfzAIgb4E++RlwK9kHghywWCtbDOJSjHGEhusD4DH033y+coE="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.6":{"name":"js-crawler","version":"0.3.6","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"3d111629b311b7082889c919be0d0ed6cdeebca2","_id":"js-crawler@0.3.6","_shasum":"b16d25d30e4c3b4aa07370715722421dd327cadd","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"b16d25d30e4c3b4aa07370715722421dd327cadd","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.6.tgz","integrity":"sha512-RU9hsb/79/LvFgTDu682kF/kEGxDwusI6I5g30I32+ylKAwJOhfn/aAwvxdvvUOMK+iBIgsus9+6MGYalvLVrQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIA560/EiKIAJ1lXmqC/561VkR6vQUIvyjXM27ekdw4sRAiANg5Xtttd0tP9V3jVShWQ1rumBVanAYJaJLrzOL05DoA=="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.7":{"name":"js-crawler","version":"0.3.7","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"deebfad981887b543f64b7126c877663db69c245","_id":"js-crawler@0.3.7","_shasum":"3dc16edaa8404c7a47fd92845b936490554eed67","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"3dc16edaa8404c7a47fd92845b936490554eed67","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.7.tgz","integrity":"sha512-iMhyike/0mVhd+f6TSczB6bQ/b29RYdZ0DEmZfHuKEQJbZ5XJWNUrByGU1c5PuOA+0YcTKzC65vnUSBNBT2/nQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIBsGDilUnNORDzXgiGokZrZCq8a8MpAVQTHh92HN/KYdAiEAz8i/SaU7K3ylAZN0V7AxARjk8a6GkPI5afNq7XlG964="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.8":{"name":"js-crawler","version":"0.3.8","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"7ce2da212f362f4d786e0d62baed6e561819ed5d","_id":"js-crawler@0.3.8","_shasum":"727a8d403e40bc5c9683a204379f9f318f839d90","_from":".","_npmVersion":"2.1.14","_nodeVersion":"0.10.34","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"727a8d403e40bc5c9683a204379f9f318f839d90","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.8.tgz","integrity":"sha512-Juq8J8q90aVHM6kiOAMNOYxRhZtt5qbwHA8mjj6fS3LcN9n6/UNHQDI6GveZRfFlwQkCHHjEiP0mox7ICb8TDw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIG9HwoQbSMEBaLibUVlS+flY/YuEjcOokhYJEaSeg0aZAiEA0Qi2jHzS1TsRTeiPVonK1dt+hd2a+plZn3Pr1fXXAIU="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}]},"0.3.9":{"name":"js-crawler","version":"0.3.9","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"https://github.com/antivanov/js-crawler"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"d4612dc5694d2c9ada8c359b0476bc25d9d9139b","_id":"js-crawler@0.3.9","_shasum":"ffbc3e4a4065465e76634befe3bb6a515c8a4b66","_from":".","_npmVersion":"2.7.4","_nodeVersion":"0.12.2","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"ffbc3e4a4065465e76634befe3bb6a515c8a4b66","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.9.tgz","integrity":"sha512-x3OZx/rDPt+XKaFeC+emdq6N4lP3WUUVBT31D+Ru8juJlET/5ozqZnKLoD3Qnbe4xdQjnpnnjFW0YEXAqE53Xg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDtKRPwCpU9V8pBN4bbQxUrZzsup/8GlVNyjLd0ie6yMgIgWekumZLtZKaZsM1PRwi0QZ/RMDzj9ZOTMp+GU4k+iMI="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.9.tgz_1459634817968_0.24165740935131907"}},"0.3.10":{"name":"js-crawler","version":"0.3.10","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"855e671eae4344d33506ea207eb3dea66662e1be","_id":"js-crawler@0.3.10","_shasum":"11ffa822e4aa989f11e2e49f6d794e80b4176268","_from":".","_npmVersion":"3.9.5","_nodeVersion":"6.2.2","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"11ffa822e4aa989f11e2e49f6d794e80b4176268","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.10.tgz","integrity":"sha512-NLrS5m69jYSVwfF5fVYWZxSERWUmGKqRnunDneA0/cSl7tnuUfAxHzhbZWKAFwpog3xGALkKSTU9+1VR/fiTcg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGIFXmUy180T1CDL9jmOT3rfHON6Pm67bJ6MehE0gSp0AiEAn13/2KKWSlGndijQ4XAz9EUa37RNWGV/mLEREt36BkA="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.10.tgz_1467995974023_0.17756956676021218"}},"0.3.11":{"name":"js-crawler","version":"0.3.11","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"856d4feb1f90b5c5ea93361398e2e51475fb8d1e","_id":"js-crawler@0.3.11","_shasum":"a3166452c1d73273e47ec0c72f6629afb541993e","_from":".","_npmVersion":"3.9.5","_nodeVersion":"6.2.2","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"a3166452c1d73273e47ec0c72f6629afb541993e","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.11.tgz","integrity":"sha512-at/Bnbox6mRbHS6ej+6T2KKbUQSTYonGw1Kxuoqr0EcXINsMv/QCLX/p9B+EyH6bbOsZzeVAhBZAw6zC6xLB9g==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDjQyG0v3wc5p+v+V/7eNEEEe5Iq31olzrGQRxMIJV6fwIhAPoaNwj7UueMAj+9mdHS6cly9gjsat+EAr+E43J8T5Mi"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-16-east.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.11.tgz_1468875283499_0.1266312664374709"}},"0.3.13":{"name":"js-crawler","version":"0.3.13","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"56524353f5fc9b1acd408c26c0d9e8f72de61cc0","_id":"js-crawler@0.3.13","_shasum":"956e8a54b0ae7620e0d0d18cbe39448215054cad","_from":".","_npmVersion":"3.10.8","_nodeVersion":"6.9.1","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"956e8a54b0ae7620e0d0d18cbe39448215054cad","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.13.tgz","integrity":"sha512-CmmScUJl7cfhaUyMRH1joLLlbLmbnrPhFDLo3tQ5Uf6Rd1qPDdVAReu9RhKS/xD+1u/13CPM/hfPqt2LoJngxA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDjhWZsO9fbNOh6emHtr0mLhEpwXdXVzFSrMUM+xV9HxAiBng147b/UJDCnJ/86XWp+LhrAM+mIeActts8W3FvKYuw=="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.13.tgz_1479072353245_0.43167667044326663"}},"0.3.14":{"name":"js-crawler","version":"0.3.14","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"51ba730b3465bdad570ea3d1d395b716f403c078","_id":"js-crawler@0.3.14","_shasum":"19b394825869d1f712f0b2c04d3c66d9d266f5da","_from":".","_npmVersion":"3.10.8","_nodeVersion":"6.9.1","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"19b394825869d1f712f0b2c04d3c66d9d266f5da","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.14.tgz","integrity":"sha512-wwwYzuXg7v628l/3dnLc7whsPgA5F57VXgfbVcLNP5f6+IIOpWThfnUKlY6ottvbp0maDcytHUssoQGrZW6acA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDDvZr7q02Gkxl423KXZSZWfhPvSHLt0pCnJHjiUSiDNgIgXUjVAXnvOvxr9to80/3YZebAu47AsoryaH/65WoJTMs="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.14.tgz_1479418366047_0.007826814195141196"}},"0.3.15":{"name":"js-crawler","version":"0.3.15","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"5111dc912bcb8f2347e85a87933fd2e39e269289","_id":"js-crawler@0.3.15","_shasum":"e35e8ffe75325b6673537dc34be39393cbd406aa","_from":".","_npmVersion":"3.10.8","_nodeVersion":"6.9.1","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"e35e8ffe75325b6673537dc34be39393cbd406aa","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.15.tgz","integrity":"sha512-rvzM0KLM7+vE49dwPH4kqsxH9EFE5k5aFQzunnq3kJswoZjg+q42OpLMgYObCea9M2+cW8WzlEJX9GVWcG45Cw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDCKQXK59qu/xu3aRl7gKVeZauvLgZEVg007tY6ascYLAIhAMhBcQEVFLYDf2T8ZzqAJkcuNAusBAECc/zvaLDYeT9F"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.15.tgz_1480023726075_0.7563091923948377"}},"0.3.16":{"name":"js-crawler","version":"0.3.16","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"1e92d69e66f4cd68e67426da6eccea165e5d2bd5","_id":"js-crawler@0.3.16","_shasum":"f47910202e69405b8937053ee427724be5effec9","_from":".","_npmVersion":"4.1.2","_nodeVersion":"7.5.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"f47910202e69405b8937053ee427724be5effec9","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.16.tgz","integrity":"sha512-Qk6Et6Vhr8dr0Uaxl7+eritCh4kbtOu3FZ7QqofAGhykSnQfgiCI0RsglWCwANZjpMttiRPmgIA3F9lBmv7Ddw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDz1mAnrEJUxz7N809o+wdaze3OeMMHiaPEZHMaEe6aNwIgFgBfTL7DWX2Qe2fpiYx+q6CRdr61hAZaonBdtrqGio0="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.16.tgz_1487196343903_0.46064612665213645"}},"0.3.17":{"name":"js-crawler","version":"0.3.17","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"982bbc69ddbc172c80ce79dead0e300b7808e85c","_id":"js-crawler@0.3.17","_shasum":"e72c76ce11b9e22573e7189a793008f85746def9","_from":".","_npmVersion":"4.1.2","_nodeVersion":"7.5.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"e72c76ce11b9e22573e7189a793008f85746def9","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.17.tgz","integrity":"sha512-TTkLHnJCum+CzsjMo0KuL2IE+5Yppx51um4Tznj7OkKY9G6fZ6qYgOpbIvJ7fSx38qmOfjMIuAGysNvQqWT0Aw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDRQHUSGHbudvF0m3YA46awSPiyU8YW8WlnRdqwELlZwAIgXUkeqocu0/1TRaYRFUPklYEo/UHlqB26SzB81lfyOzk="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.17.tgz_1487284114293_0.3014371991157532"}},"0.3.18":{"name":"js-crawler","version":"0.3.18","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"136d807bef514a36c2f6da1ffdf7db7e1cd4fa10","_id":"js-crawler@0.3.18","_shasum":"a96caae6ec1d428e1ce94b0497c30fec3169b633","_from":".","_npmVersion":"4.1.2","_nodeVersion":"7.5.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"a96caae6ec1d428e1ce94b0497c30fec3169b633","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.18.tgz","integrity":"sha512-zxKN9WTKdyXuWxv4vZ4nVK+wMsKOX8oxWJbrnvaCOm527nbgxoc367Lhuch+Gx6LzDhkV2scFMrDyv6UeVYzWQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHEjUifam0u+luSDuv0T2BBK1ALovH+CDjEEfLkYlp9uAiADWxdv30T2VCOGDut7eQVFZSc4gWqqJ2lNCRXLIFm6qg=="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.18.tgz_1487626003847_0.6302980480249971"}},"0.3.19":{"name":"js-crawler","version":"0.3.19","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"040abaa95193cdbd69c3bcec25249d91321660a2","_id":"js-crawler@0.3.19","_shasum":"f8f6ab276e4570308740616b04e48cf7120a6653","_from":".","_npmVersion":"4.1.2","_nodeVersion":"7.5.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"shasum":"f8f6ab276e4570308740616b04e48cf7120a6653","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.19.tgz","integrity":"sha512-kyrIwQcYm4BVHaGnZkMKZwvaZpEZ3VvJmz0GofdMuov+CIX+1eZxfHW0BsDAO9OqVnXxQqUW7+s1r+63YMyKAQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCgW6DpeVkX6oS35F8Gxc9eZKWZsw28sUK57GJSrBcabwIhALiWI82d/ZpWr2joGFsYmWbhhUMp+WmUvh04ZHNfGFtn"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/js-crawler-0.3.19.tgz_1493398947824_0.4606659871060401"}},"0.3.20":{"name":"js-crawler","version":"0.3.20","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.55.0","underscore":"~1.8.3"},"devDependencies":{"express":"^4.16.2","browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"b1d26774526d1b059d458d393169d5747e93f97a","_id":"js-crawler@0.3.20","_npmVersion":"5.6.0","_nodeVersion":"8.3.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"integrity":"sha512-K+qjlKi2b66EIttMnc3yo2tmYHxnIWmVYjw1MazaaSyFIbkSxtNhD6ena89YR0Rh3WDZ2u3f31ktwTK7wBkYlw==","shasum":"e45da53abb441ab9dc8c4e43b252e48b21140e87","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.20.tgz","fileCount":35,"unpackedSize":66965,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIB3gRCURJVQ3Kigs3X59SBkLc4SU0JWa2TF9tWV7tBXzAiEAmtkP0x95ZIdy9upK84MprZGvpMKq7spUU4Z9USHPQbs="}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/js-crawler_0.3.20_1519249245548_0.6532559804589329"},"_hasShrinkwrap":false},"0.3.21":{"name":"js-crawler","version":"0.3.21","description":"Web crawler for Node.js","main":"crawler.js","directories":{"example":"examples"},"dependencies":{"request":"~2.87.0","underscore":"~1.8.3"},"devDependencies":{"express":"^4.16.2","browserify":"^13.0.0","grunt":"^0.4.5","grunt-eslint":"^18.0.0","grunt-karma":"^0.12.2","jasmine-core":"^2.4.1","karma":"^0.13.22","karma-browserify":"^5.0.2","karma-chrome-launcher":"^0.2.2","karma-firefox-launcher":"^0.1.7","karma-jasmine":"^0.3.8","karma-jasmine-html-reporter":"^0.2.0","karma-phantomjs-launcher":"^1.0.0","phantomjs-prebuilt":"^2.1.6","watchify":"^3.7.0"},"scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"license":"MIT","bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"homepage":"https://github.com/antivanov/js-crawler","gitHead":"64ba4de566a5e9ecf13a2945902b7d67ed10ff13","_id":"js-crawler@0.3.21","_npmVersion":"6.1.0","_nodeVersion":"8.3.0","_npmUser":{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"},"dist":{"integrity":"sha512-T/UgXlaikrSLEaKfpLUqJuO2DJZ4wF9i5oZxG6os96hGk/Wr/8PIBFKQmOAIWKx4MIjt1sxrFwDnHVvfbAqAjw==","shasum":"c936a45240e911fad8b5c111928438e618804639","tarball":"https://registry.npmjs.org/js-crawler/-/js-crawler-0.3.21.tgz","fileCount":84,"unpackedSize":179805,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJbG+r+CRA9TVsSAnZWagAAaNYQAKGR7EKg+lbHoSIjByeK\nh4dPdyVS9xdSrGhy8vcj9VrquIe+rNscWREEw2ty2DsSPTFN6dJTzbcNAZIu\nmMPXvja+bgnBWr+OV1tv5jm4Xny9n7nHtmCiKkXF92hqFQfkcL0hjItn3Uoy\nlqls8guKGVVt5+ZHCXWpy50PnlWVVBWUCwd6Wj0J1ctJS0s/dfhjHLhXfrw7\nYAT8PVp7HRucc/VMZ0aLX97h/oP3T+pIp0odTIDbgW4bZbAzJyQeKs3r0Z+d\nc0AiGLk2bDwJsxRIcVzDSx8kDsETvIzfAl4bRx6hy2a2Vh+3JVEP+YcpRLVf\nLFUhL3HnRrnF4YM6hXuXCx9NdKBuvXTTIJHAn7qlZ2xLHlZaG2u1MS2hKHp0\neZyiHAUyRGVTtW7XsVHe/oYtgyAI51h2sRkFoZSkFhJ3GXwGmmmOwgSrgODN\nmYF4RMy2piTT+uQ9BxeF3LiWkc5M3MvXeXtz2El9eA4b404X5L5PIYRvI7H/\nN9SdNEsBc7+SlE7uf3bZj7QE4EGZp/DyjFfz/8XGL1hXord3sIPRnZO4R0Us\n/lfit0Awc6rSqAamK5/mv8PzNK+7yJa/SaSvNp71OOxE6OWrqNLrGpxMHIPg\na6BONshTrLqE0UgFEhKhOQ8lkvLiCKYZAb6BlIBZNRKgyZtNocCRwtQMIMqK\nUrZL\r\n=rhhR\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDpynYgxsKpuxJ8D41+yedWO9uKsNztVw+fa7y65OscIwIhAN4b8Nct4Toh5q9GMKxl0B5MOI7oHkm+9srmB1cPtefK"}]},"maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/js-crawler_0.3.21_1528556284648_0.12853337253031838"},"_hasShrinkwrap":false}},"readme":"js-crawler\n==========\n\nWeb crawler for Node.JS, both HTTP and HTTPS are supported.\n\n## Installation\n\n```\nnpm install js-crawler\n```\n\n## Usage\n\nThe crawler provides intuitive interface to crawl links on web sites. Example:\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nnew Crawler().configure({depth: 3})\n  .crawl(\"http://www.google.com\", function onSuccess(page) {\n    console.log(page.url);\n  });\n```\n\nThe call to `configure` is optional, if it is omitted the default option values will be used.\n\n`onSuccess` callback will be called for each page that the crawler has crawled. `page` value passed to the callback will contain the following fields:\n\n* `url` - URL of the page\n* `content` - body of the page (usually HTML)\n* `status` - the HTTP status code\n\nExtra information can be retrieved from the rest of the `page` fields: `error`, `response`, `body` which are identical to the ones passed to the callback of `request` invocation of the [Request](https://github.com/mikeal/request) module.\n`referer` field will reference the url of the page that lead the crawler to the current page.\n\n#### Options-based API\n\nAlternative APIs for passing callbacks to the `crawl` function.\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nvar crawler = new Crawler().configure({ignoreRelative: false, depth: 2});\n\ncrawler.crawl({\n  url: \"https://github.com\",\n  success: function(page) {\n    console.log(page.url);\n  },\n  failure: function(page) {\n    console.log(page.status);\n  },\n  finished: function(crawledUrls) {\n    console.log(crawledUrls);\n  }\n});\n```\n\n#### Handling errors\n\nIt is possible to pass an extra callback to handle errors, consider the modified example above:\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nnew Crawler().configure({depth: 3})\n  .crawl(\"http://www.google.com\", function(page) {\n    console.log(page.url);\n  }, function(response) {\n    console.log(\"ERROR occurred:\");\n    console.log(response.status);\n    console.log(response.url);\n    console.log(response.referer);\n  });\n```\n\nHere the second callback will be called for each page that could not be accessed (maybe because the corresponding site is down). `status` may be not defined.\n\n#### Knowing when all crawling is finished\n\nExtra callback can be passed that will be called when all the urls have been crawled and crawling has finished.\nAll crawled urls will be passed to that callback as an argument.\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nnew Crawler().configure({depth: 2})\n  .crawl(\"http://www.google.com\", function onSuccess(page) {\n    console.log(page.url);\n  }, null, function onAllFinished(crawledUrls) {\n    console.log('All crawling finished');\n    console.log(crawledUrls);\n  });\n```\n\n#### Limiting the rate at which requests are made\n\n##### maxRequestsPerSecond option\n\nBy default the maximum number of HTTP requests made per second is 100, but this can be adjusted by using the option `maxRequestsPerSecond` if one wishes not to use too much of network or, opposite, wishes for yet faster crawling.\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nvar crawler = new Crawler().configure({maxRequestsPerSecond: 2});\n\ncrawler.crawl({\n  url: \"https://github.com\",\n  success: function(page) {\n    console.log(page.url);\n  },\n  failure: function(page) {\n    console.log(page.status);\n  }\n});\n```\n\nWith this configuration only at most 2 requests per second will be issued. The actual request rate depends on the network speed as well, `maxRequestsPerSecond` configures just the upper boundary.\n\n`maxRequestsPerSecond` can also be fractional, the value `0.1`, for example, would mean maximum one request per 10 seconds.\n\n##### maxConcurrentRequests option\n\nEven more flexibility is possible when using `maxConcurrentRequests` option, it limits the number of HTTP requests that can be active simultaneously.\nIf the number of requests per second is too high for a given set of sites/network requests may start to pile up, then specifying `maxConcurrentRequests`\ncan help ensure that the network is not overloaded with piling up requests.\n\n##### Specifying both options\n\nIt is possible to customize both options in case we are not sure how performant the network and sites are. Then `maxRequestsPerSecond` limits how many requests the crawler is allowed to make and `maxConcurrentRequests` allows to specify how should the crawler adjust its rate of requests depending on the real-time performance of the network/sites.\n\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nvar crawler = new Crawler().configure({\n  maxRequestsPerSecond: 10,\n  maxConcurrentRequests: 5\n});\n\ncrawler.crawl({\n  url: \"https://github.com\",\n  success: function(page) {\n    console.log(page.url);\n  },\n  failure: function(page) {\n    console.log(page.status);\n  }\n});\n```\n\n##### Default option values\n\nBy default the values are as follows:\n\n`maxRequestsPerSecond` 100\n\n`maxConcurrentRequests` 10\n\nThat is, we expect on average that 100 requests will be made every second and only 10 will be running concurrently, and every request will take something like 100ms to complete.\n\n#### Reusing the same crawler instance for repeated crawling: forgetting crawled urls\n\nBy default a crawler instance will remember all the urls it ever crawled and will not crawl them again. In order to make it forget all the crawled urls the method `forgetCrawled` can be used. There is another way to solve the same problem: create a new instance of a crawler. Example https://github.com/antivanov/js-crawler/blob/master/examples/github_forgetting_crawled_urls.js\n\n#### Supported options\n\n* `depth` - the depth to which the links from the original page will be crawled.\nExample: if `site1.com` contains a link to `site2.com` which contains a link to `site3.com`, `depth` is 2 and we crawl from `site1.com` then we will crawl `site2.com` but will not crawl `site3.com` as it will be too deep. \n\nThe default value is `2`.\n\n* `ignoreRelative` - ignore the relative URLs, the relative URLs on the same page will be ignored when crawling, so `/wiki/Quick-Start` will not be crawled and `https://github.com/explore` will be crawled. This option can be useful when we are mainly interested in sites to which the current sites refers and not just different sections of the original site.\n\nThe default value is `false`.\n\n* `userAgent` - User agent to send with crawler requests.\n\nThe default value is `crawler/js-crawler`\n\n* `maxRequestsPerSecond` - the maximum number of HTTP requests per second that can be made by the crawler, default value is 100\n\n* `maxConcurrentRequests` - the maximum number of concurrent requests that should not be exceeded by the crawler, the default value is 10\n\n* `shouldCrawl` - function that specifies whether a url should be crawled/requested, returns `true` or `false`,\nargument is the current `url` the crawler considers for crawling\n\n* `shouldCrawlLinksFrom` - function that specifies whether the crawler should crawl links found at a given url, returns `true` or `false`, argument is the current `url` being crawled\n\nNote: `shouldCrawl` determines if a given URL should be requested/visited at all, where as `shouldCrawlLinksFrom` determines if the links on a given URL should be harvested/added to the crawling queue. Many users may find that using `shouldCrawl` is sufficient, as links from a page cannot be crawled if the page is never visited/requested in the first place. A common use case for having these functions separated: if a user would like to check external links on a site for errors without crawling those external links, the user could create a `shouldCrawlLinksFrom` function that restricts crawling to the original url without visiting external links.\n\n\n**Examples:**\n\nThe following will crawl the specified URL, but not allow external URLs to be visited/requested, and therefore not search for additional links to crawl on the external URLs:\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nvar crawler = new Crawler().configure({\n  shouldCrawl: function(url) {\n    return url.indexOf(\"reddit.com\") < 0;\n  }\n});\n\ncrawler.crawl(\"http://www.reddit.com/r/javascript\", function(page) {\n  console.log(page.url);\n});\n```\n\nThe following will crawl the specified URL, allow external URLs to be visited/requested, but will not search for additional links to crawl on the external URLs:\n```javascript\nvar Crawler = require(\"js-crawler\");\n\nvar crawler = new Crawler().configure({\n  shouldCrawlLinksFrom: function(url) {\n    return url.indexOf(\"reddit.com\") < 0;\n  }\n});\n\ncrawler.crawl(\"http://www.reddit.com/r/javascript\", function(page) {\n  console.log(page.url);\n});\n```\n\nThe default value for each is a function that always returns `true`.\n\n#### Development\n\nInstall dependencies\n\n`npm install`\n\nRunning the build\n\n`grunt`\n\n##### Unit tests\n\n`grunt karma:unit`\n\nlaunches unit tests in the console mode\n\n`grunt karma:unit_browser`\n\nlaunches a browser in which unit tests can be debugged\n\n##### End-to-end tests\n\n`jasmine-node` and `express` are used to setup and run end-to-end tests\n\nInstall jasmine-node and express globablly\n\n```\nnpm install -g jasmine-node\nnpm install -g express\n```\n\nStart the end-to-end target server\n\n```\ncd e2e\nnode server.js\n```\n\nNow the server runs on the port 3000.\nRun the end-to-end specs:\n\n```\njasmine-node e2e/\n```\n\n## License\n\nMIT License\n(c) [Anton Ivanov](http://smthngsmwhr.wordpress.com/)\n\nCredits\n---------------\n\nThe crawler depends on the following Node.JS modules:\n\n* [Underscore.js]\n* [Request]\n\n[Underscore.js]: http://underscorejs.org/\n[Request]: https://github.com/mikeal/request","maintainers":[{"name":"ant-ivanov","email":"anton.al.ivanov@gmail.com"}],"time":{"modified":"2022-06-19T05:32:08.589Z","created":"2014-02-15T17:24:55.922Z","0.1.0":"2014-02-15T17:24:55.922Z","0.2.0":"2015-05-21T19:51:36.937Z","0.3.0":"2015-05-26T16:29:55.924Z","0.3.1":"2015-06-01T20:29:58.587Z","0.3.2":"2015-07-07T20:50:30.541Z","0.3.3":"2015-09-06T17:19:22.873Z","0.3.4":"2015-09-08T20:31:42.213Z","0.3.5":"2015-09-11T20:22:15.987Z","0.3.6":"2015-09-11T20:36:15.511Z","0.3.7":"2015-09-14T21:20:57.979Z","0.3.8":"2015-10-26T20:09:09.977Z","0.3.9":"2016-04-02T22:07:00.393Z","0.3.10":"2016-07-08T16:39:36.642Z","0.3.11":"2016-07-18T20:54:44.576Z","0.3.13":"2016-11-13T21:25:57.283Z","0.3.14":"2016-11-17T21:32:46.710Z","0.3.15":"2016-11-24T21:42:06.735Z","0.3.16":"2017-02-15T22:05:47.371Z","0.3.17":"2017-02-16T22:28:37.586Z","0.3.18":"2017-02-20T21:26:47.042Z","0.3.19":"2017-04-28T17:02:31.357Z","0.3.20":"2018-02-21T21:40:45.713Z","0.3.21":"2018-06-09T14:58:04.773Z"},"readmeFilename":"README.md","homepage":"https://github.com/antivanov/js-crawler","keywords":["web-crawler","crawler","scraping","website-crawler","crawling","web-bot"],"repository":{"type":"git","url":"git+https://github.com/antivanov/js-crawler.git"},"author":{"name":"Anton Ivanov, anton.al.ivanov@gmail.com"},"bugs":{"url":"https://github.com/antivanov/js-crawler/issues"},"license":"MIT","users":{"junjiansyu":true,"dragonken":true,"quocnguyen":true,"zairon87":true,"bhaskarmelkani":true,"etservicesde":true,"nicknaso":true,"nelix":true}}