{"_id":"huntsman","_rev":"61-bc496009c74d2004c11d53c53a5de6f4","name":"huntsman","description":"Super configurable async web spider","dist-tags":{"latest":"0.3.0"},"versions":{"0.0.1":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.0.1","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","cheerio":"~0.11.0","coffee-script":"~1.6.2"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.0.1","dist":{"shasum":"983e292c8fd71e27dea1fb40c193fa8d61f021ea","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.0.1.tgz","integrity":"sha512-2Bo+MUDG+AVb5N/xyTV49AxTRDinUH6JPwinsH4UZeVpFzsYzfC1UjmKSCF8mD9REtv/9SHYmgpVDyRVAT+RYg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCn3vdWejAdQH0QWiquUH/NKjVuLQXq4nKEWUIOqBw/yAIhAO4uJO+1dYoqNekwII2q0oZtcef1HFE4+Gk082dWCIfo"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.0":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.0","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.0","dist":{"shasum":"8e07d4d5b964790125e6507ab1b03d6c10532a00","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.0.tgz","integrity":"sha512-BG0mpyfOhLtj3/DOQUDoTWRul/cQQG02h6cCUYksxiJZ9UoSHRYGHk8rfP/bcY5Bx2TiIO0//vwV9Tnv6jWwaw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCEX5xOJH6a+wSFNhTZAN3yArSrzJxJN2nmx3XzVCeU1wIhANRpH5rRySJYN4JvZHIwIO9kKksiy1n0Yqs4vc7RPQXj"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.1":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.1","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.1","dist":{"shasum":"1227604e189c5e7b94e3ed6089b94d2cfe18f49c","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.1.tgz","integrity":"sha512-O7d3R/8SupA6JKLwxmTLTUSYnwGtFdCmQBGw8P3JScpjNJGRgbBW3cMjynAWKIjCRFsNeO3RyDSn1CVgYpwPDw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDqT3MtS+JOcuLIo1JlFRA3ec3J4o3yGQrY0kSWOBEj9AIgVYGTJS54McxO7p41GZzF4xUPbQi8hhYyndy5Ke7kA+M="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.2":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.2","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.2","dist":{"shasum":"0c86037cf052d92f3dd82cbfb8d14c3c635ea557","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.2.tgz","integrity":"sha512-Uy+c7OJnnytblp7UhmnGbZNe9v03auuCIHnbDo/UqFXuPxveBJkSk1+hIWWjQXyUtCClHPOiRcx83IUc1MZfEg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDV0++rJ3ltynRGgeUeWtDHlcJ4uBsHypYLyRyc8sEwjAIhAKIU4sDr0o1O/NMnUtMTwjwV4y94NXu4b6sQvQYLtsKb"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.3":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.3","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.3","dist":{"shasum":"c03a48ca8f7a14a424e61c2b9d0f0c1256834c94","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.3.tgz","integrity":"sha512-sSV9RlTn6Z6sxipkowrZcXY53S4Z8Swz3larfv0xYjVCM1/FfD73VrOtazAHnW5+xQ8D+ELulRPdruxk5bSryg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIColoD3+vd7lwMVJhgIZOiLqmYVyZHRxU0V7Qfd/o7O+AiEAztOemL+Gie5zFzJvJggQgHpKycH3Gqs2V4oGMdtYhAY="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.4":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.4","description":"Super simple web spider","contributors":[],"dependencies":{"request":"~2.21.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.4","dist":{"shasum":"3e15d46c1b5d29000febec9b31b4655bceabf141","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.4.tgz","integrity":"sha512-6fN8ZA/t6H/he8o0ueZl+NMDq3U5lkHYisJOXEmOehN+Y8QSRg71ks+3JW1gRSl9Eiw52v1uISppajQiR8GGkA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDYAV95sMPNysKHIr0ORyYOBPxcw91JL1vUwKW2QeeoowIhALgeAbPcHwtjjyrPEk2QojnQ2oHspO3Fd8zL1vhgFiXX"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.5":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.5","description":"Super simple web spider","contributors":[],"dependencies":{"regexemitter":"0.1.0","request":"~2.21.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.5","dist":{"shasum":"175c9d0cc06096f08f963e979fd3f20c0fb8f039","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.5.tgz","integrity":"sha512-8yi8fYVZVYCAPyFSS6DVnqEFpukyl8A+Kmt1JgwGRwxYPeTKY7rcJHXnK6jRmRtKZeGatuwbLfnib9kkkw7VGA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD3Q79brL8ivagczaY9A0hLPLAZmTZxNDni9+VxVmEzQwIhAOsZjBxgWeyFAxQeuGlCZ2n9EnjTFmPAKR9TIFEi6Ugr"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.6":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.6","description":"Super simple web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","request":"~2.22.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.6","dist":{"shasum":"d085ddfe18daf815e36daadc3516608918f14f1f","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.6.tgz","integrity":"sha512-ZQL6Qdmv5Nsi8ESFAlpqZkYVWeKJeiO6XkZgpqJDFivIw0UHA5DrRSdkbrzHfmW9Ow14BbQYwZD+ffVwqGthIQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCvyCiATWqbcKGLgV+FlcyDjooBU4TeNJD9z0vw13OCsgIhANo3ZpRrCnZPC0oOimkWZGnkVywoBDZsQD6tjrMHJx10"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.7":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.7","description":"Super simple web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","request":"~2.22.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.7","dist":{"shasum":"4fcd231a75566c6031129fbb46369f36fd0e45be","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.7.tgz","integrity":"sha512-ZqlZxLI6y/0FFUOMD9MruzrrhXTiahp5lOHkyee99tKXUzE6MUWoc+3QRA6dVsJb5c/JT4lnAx4daOvjCl8MCA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGPYQ/GOH5/ljPRUnicyyThrR45boinFPUFGBM8VFi6LAiEAi1mVdtv7tLR7/SaX1bxb26FZZPtOkJ7L0MtdhZvKf0M="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.8":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.8","description":"Super simple web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","request":"~2.22.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.8","dist":{"shasum":"6e05938053c5cb3d026e728baeb569eb0c49b6e3","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.8.tgz","integrity":"sha512-q/aKJPzYYRaxHcaF91nRAxDp2/sF+YCYkp4u2c1thMrQwxu/AJ2gTj3vkL9jzDESKbuIuuVTrT76aUvoGAZdvA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDkoH9NHY8zLEQDZsUYOkexea+ZKuZBl8ynsl0LM5GT2AIgIaU2xuQVMn4esrrhbKel+km8Qj9ikUgqR3TNMl734hY="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.1.9":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.1.9","description":"Super simple web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","request":"~2.22.0","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.1.9","dist":{"shasum":"3953aaed42605ee5bfd666c3fba74ff6bf1e6a9e","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.1.9.tgz","integrity":"sha512-jGOAV/Leq1+cANbBbQ5S84OXutSW+7Xpi+1bwW9MIJsaPS0EKF5GmwNlXOaZQloGyYogHK9SEbjfyUycoXzWEA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCoYj9M7j0FcnnUvik0Hs0VihPzTkTY+Eq7n6PPv1DzEAIhALGzzOCptMjC74CYKXwwKKyCAqtUNigzJYskQm2ysgta"}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.0":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.0","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.2.0","dist":{"shasum":"d5659c2cf78a04561bb261df9a4188c569ccb8ca","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.0.tgz","integrity":"sha512-+NOkCbJXqg9XYfKNDl7otFv+fpfrKdEvTMKzmPDwEhO/Pv5qcdecGjx9V8p3jvWhuy7eSCOGIT65k5gQA/jgAA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIE9xIu8k7Zl8kgpPG1SkXRKkfI1f6C7Mv8PpyKkqIe5AAiBI1Ym4LiBHNDQIbQfKuIgrM+rEGFOdPd5xKkxZ/S9rGg=="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.1":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.1","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.1.2","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"_id":"huntsman@0.2.1","dist":{"shasum":"04e35a1ebc5079dee16dba7afd7154a4fb5a980a","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.1.tgz","integrity":"sha512-eBjhL7E82lmRWqFFZ4BG+2Ejtsmu3VUUWpVPdes7GMcjJCJ0k/A3U+hpr1J3hjR8deUpAJ8r15ghL4II1CaWRw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHZ+HaWNOzZT8oQi4dtg1d7AZXCrRdE8JWyUqJCDkZC4AiAoVSnGM0wvSq38Vq9gJhvwPA2PN5fNmrseVOgwNCjEiA=="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.2":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.2","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.2.2","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"bugs":{"url":"https://github.com/missinglink/huntsman/issues"},"_id":"huntsman@0.2.2","dist":{"shasum":"b9ef53f7b47d03f6e8cef0aa24fb31a21ddc9295","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.2.tgz","integrity":"sha512-xZY2C50e1OKiwg/8hwLNp8ccuKkgTfeMCFU+OOV/xGQLANpkSK4xMvRqSeU0/UBnq8TtWcRdV2yznHlxXAb6tw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICarF/2lZ6xq7KveKYFHCcaaKS84ztwhuQhM4QWgpZHdAiAodY5gyf5Bw3aoXrqa1sVVQX/gsclFA2VQaVDNHe2Zhg=="}]},"_from":".","_npmVersion":"1.3.2","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.3":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.3","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.2.2","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"bugs":{"url":"https://github.com/missinglink/huntsman/issues"},"_id":"huntsman@0.2.3","dist":{"shasum":"b3dbad1c29b41cb34f09b891b0ecef238ec5de5f","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.3.tgz","integrity":"sha512-+bmMxra2tfBUYsqzp2aMfCEIGCWL5ggGJLYkdIeyGaIHbxDXszKhi1UcQ8gV4LquPuZ4YfF9txjEza9fBqGuHA==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHiWhGbvnlgNOd8YvCf6aKm7Qb4nNYIvGE+oFwefS/e0AiEA6SPcu2zHr/pUalOiZ/N1rmcz3HZEfXtDCOg4h3oBhL8="}]},"_from":".","_npmVersion":"1.3.2","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.4":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.4","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.2.2","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"bugs":{"url":"https://github.com/missinglink/huntsman/issues"},"_id":"huntsman@0.2.4","dist":{"shasum":"b52d8a29a606c9962db12d3f97672ebf8eeedc46","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.4.tgz","integrity":"sha512-LIu7WP40zGD9iJEJ4K4OIkCZ/9IDSXGkr4hmuQmyZbPS5o032VPsIgCiMA8AWbP9NsrnDTLoRtVh9aNshWEybg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCLueE3vWxLNPGMgwx3moK1ish5lbiPMq9Qq4e+ODVgSAIhAIWmDP9Dyah2zni52W+kSVLaX2qQOx+n0com7V2Cno5i"}]},"_from":".","_npmVersion":"1.3.7","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.5":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.5","description":"Super configurable async web spider","contributors":[],"dependencies":{"regexemitter":"0.2.3","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"repository":{"type":"git","url":"https://github.com/missinglink/huntsman.git"},"bugs":{"url":"https://github.com/missinglink/huntsman/issues"},"_id":"huntsman@0.2.5","dist":{"shasum":"e80470bb1a304a517d30496bd1699ebaea0dc49d","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.5.tgz","integrity":"sha512-rGotl27YiAQBGpOdO/htAU7v1YgbMSs/pA/ZGxLyDM6xBbOA6jsj/TRdDu8CAewv7FeI1ADEzcDxGs1U9cEnng==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCUaqrEQY6vYfnHZHX68IQ+LzopKeoMUygwbqkB9MZ3fAIhAJT517LIK/N2ZrxIxrTALO3nl2pg5qshBZfIEZeFmoIL"}]},"_from":".","_npmVersion":"1.3.7","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.6":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.6","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"regexemitter":"0.2.3","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"_id":"huntsman@0.2.6","dist":{"shasum":"49136b0ac4e4c7a1466c2e45b7fd4c9d7d7e15d6","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.6.tgz","integrity":"sha512-T1sV6NVpeztMp017Ml+w6VXDJZIucPHogMYsZ3oanh0pQKmGzCMPpdTF8tzxXfh+WtbtkqmsglSDXvc4o184Pw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGcoQjf1HXKHDluZJhurRDis1/KvbQRXXSGU0jfpMsIjAiEA03cqCBgotWV04CA6zqBl9OvsVevqWz0g7UUVKouwEdA="}]},"_from":".","_npmVersion":"1.3.7","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.7":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.7","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"regexemitter":"0.2.3","superagent":"~0.15.4","async":"~0.2.9","cheerio":"~0.11.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.7","dist":{"shasum":"7e1d54cfac101df2b94c031d2c696b14df7629a2","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.7.tgz","integrity":"sha512-FkBAtsRt+xn97cZzYJLI+yDpVriOZ/2LbgTKaOqzi4XDwr6BrWuq5ku3eNlbNiVVhuvswulqMDL/iMtgwVu+/w==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIEWwHYdAv+zUir3cgmHcwqdh5z58tS6yJfXA8vBYwgpDAiEArCUFNsWthU/fAO3IL8lEn+zXL2xmT4xsCk/trFXe62E="}]},"_from":".","_npmVersion":"1.3.14","_npmUser":{"name":"missinglink","email":"peter@missinglink.co.nz"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.8":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.8","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"regexemitter":"0.2.3","superagent":"0.17.0","async":"0.7.0","cheerio":"0.15.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.8","dist":{"shasum":"35aaf49bdfdbc8242f34ae814775528f52bc74af","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.8.tgz","integrity":"sha512-w/leCBRDS9ZXwTEzA1EiMe2yyFrKcINaxhdSyVu7svdxEY4UkSLSezUIsou3AtKy3azjNgzMGSlWWTVfgoQ+Vg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCuxOJAZapmIZpTtAxudxsZOr/ByMh78qX1TgcoHgajvwIhAPib78YRTjpIQSgsWJ5FlpfHWg6xw5/Sy/1TwXPrlFeX"}]},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.9":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.9","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"regexemitter":"0.2.3","superagent":"^0.18.0","async":"0.7.0","cheerio":"0.15.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.9","dist":{"shasum":"55a7723c80859c3cbcf19a3503c3c4e3a51bcd51","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.9.tgz","integrity":"sha512-LMLQfu6ch7x6l0TtktkUM3Hdo6cWlFGIeIwZG35iZLluOAIRkYDdH4wm03S4onMfud31RUrXGKZSBKBwba354Q==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCtOvjTwDHl777znaD+hsJlyjfrOlcm9D5e4OMqBu/m6AIgRjz9WJ+9zDaITkvAfnENW3sV1qjRlyZ+OOI5qbEYGfs="}]},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}]},"0.2.10":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.10","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"regexemitter":"0.2.3","superagent":"^0.18.0","async":"0.7.0","cheerio":"0.15.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.6.0","npm":">1.1.x"},"gitHead":"407f64d80906719c8216d4a4d2dc350a96ada0ac","homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.10","_shasum":"3f6eed71103e0e178198ab97e2805a96fe851c15","_from":".","_npmVersion":"1.4.23","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}],"dist":{"shasum":"3f6eed71103e0e178198ab97e2805a96fe851c15","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.10.tgz","integrity":"sha512-xXcKssXFKWa2WB/6DVQhpQ10biIhdYJHdXMvVA5jbs0P7lZLsiJLSDqJtA7eglEkzGZjl8YCi00M+iMCIojOGg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHpMkSWY1s5mPlfg8dqFvMavwhGFXzMN2v5VTlYm6A4sAiEA7TSni2Orkpjh4C0UJ9tzXhoeZE/h/aD2boYrCjTXuHU="}]}},"0.2.11":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.11","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"async":"0.7.0","cheerio":"0.15.0","regexemitter":"0.2.3","semver":"3.0.1","superagent":"0.18.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.8.0","npm":">1.1.x"},"gitHead":"88e77fd53530d65784ed539afaacfb2fc409374d","homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.11","_shasum":"3a00f742bc83aae1bcee2432d1433f887e111e41","_from":".","_npmVersion":"1.4.23","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}],"dist":{"shasum":"3a00f742bc83aae1bcee2432d1433f887e111e41","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.11.tgz","integrity":"sha512-xm3S9aoqdyBa4YALF680+CyXkGhcBkBBB8mQag4LjVQtYI0g4e7kx0Epx2T6pNhvgKErfh1dsJGj5RnGtsBSHQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD+e2zR7v1xuPfkuapbk6TjoKkrWpPPqydMTouQAOQpkQIhAKcuYa1rP/8HESn+/9oY5Ahx1o/u0FgXFs+8RxZiQJUL"}]}},"0.2.12":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.2.12","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"async":"0.7.0","cheerio":"0.15.0","regexemitter":"0.2.3","semver":"3.0.1","superagent":"0.18.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.8.0","npm":">1.1.x"},"gitHead":"83635e9b9d1f871f43b9e431f141af8116114ce4","homepage":"https://github.com/missinglink/huntsman","_id":"huntsman@0.2.12","_shasum":"5ee8245d4b8b281f81b05cb089a9ad4fa187cec2","_from":".","_npmVersion":"1.4.28","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}],"dist":{"shasum":"5ee8245d4b8b281f81b05cb089a9ad4fa187cec2","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.2.12.tgz","integrity":"sha512-X2w7nUridOr9SB4/AuZwrXJL9StOQaB8SKHYv9dWlrJsCui39ig8iDtPk6xBSLhlWKlaeIieKPyuK5hAii2UNQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHF8ggeCLyHqpICV0QvV0c+RCGc1MUGZVmh+Jra4Jf10AiBoy+3vBNqup6ew/mq/ZVwJAgyIYuyhWwdLl+PnjcmP/A=="}]}},"0.3.0":{"name":"huntsman","preferGlobal":false,"private":false,"version":"0.3.0","description":"Super configurable async web spider","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"licenses":[{"type":"MIT","url":"http://www.opensource.org/licenses/MIT"}],"dependencies":{"async":"0.7.0","cheerio":"^0.19.0","regexemitter":"0.2.3","semver":"3.0.1","superagent":"0.18.0"},"scripts":{"test":"node node_modules/mocha/bin/mocha --recursive --reporter spec --compilers coffee:coffee-script/register test"},"devDependencies":{"mocha":"*","should":"*","coffee-script":"*","breakdown":"*"},"engines":{"node":">0.10.0","npm":">1.1.x"},"license":"MIT","gitHead":"38ef6b92a1f14bd761a5de5b2612283d7b3f6ded","homepage":"https://github.com/missinglink/huntsman#readme","_id":"huntsman@0.3.0","_shasum":"f11858152c6e5f3b5c6bee51da2c901e3968681c","_from":".","_npmVersion":"2.11.3","_nodeVersion":"0.12.7","_npmUser":{"name":"missinglink","email":"npm@rcpt.at"},"maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}],"dist":{"shasum":"f11858152c6e5f3b5c6bee51da2c901e3968681c","tarball":"https://registry.npmjs.org/huntsman/-/huntsman-0.3.0.tgz","integrity":"sha512-cFMnmU7SvLJvvr/L9VhyAM3hxbekvi0I9fK7fSVDVArfxCWvq8fLidG1nlpBjJtN3JRu41mEaW4s101c0qJe8A==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQC1RW317sbfUxRibSUu7J2jlaxM80gkkDZO4XBM2S+F9gIgJyuCfFhjj9Ljtruj5pgQEZQ4FocL4O+5ShHLbCRo8UM="}]}}},"readme":"### Install\n\n```bash\nnpm install huntsman --save\n```\n\n[![NPM](https://nodei.co/npm/huntsman.png?downloads=true&stars=true)](https://nodei.co/npm/huntsman/)\n\n### Example Script\n\n```javascript\n/** Crawl wikipedia and use jquery syntax to extract information from the page **/\n\nvar huntsman = require('huntsman');\nvar spider = huntsman.spider();\n\nspider.extensions = [\n  huntsman.extension( 'recurse' ), // load recurse extension & follow anchor links\n  huntsman.extension( 'cheerio' ) // load cheerio extension\n];\n\n// follow pages which match this uri regex\nspider.on( /http:\\/\\/en\\.wikipedia\\.org\\/wiki\\/\\w+:\\w+$/, function ( err, res ){\n\n  // use jquery-style selectors & functions\n  var $ = res.extension.cheerio;\n  if( !$ ) return; // content is not html\n\n  // extract information from page body\n  var wikipedia = {\n    uri: res.uri,\n    heading: $('h1.firstHeading').text().trim(),\n    body: $('div#mw-content-text p').text().trim()\n  };\n\n  console.log( wikipedia );\n\n});\n\nspider.queue.add( 'http://en.wikipedia.org/wiki/Huntsman_spider' );\nspider.start();\n```\n\n### Example Output\n\n```bash\npeter@edgy:/tmp$ node examples/html.js \n{\n  \"uri\": \"http://en.wikipedia.org/wiki/Wikipedia:Recent_additions\",\n  \"heading\": \"Wikipedia:Recent additions\",\n  \"body\": \"This is a selection of recently created new articles and greatly expanded former stub articles on Wikipedia that were featured on the Main Page as part of Did you know? You can submit new pages for consideration. (Archives are grouped by month of Main page appearance.)Tip: To find which archive contains the fact that appeared on Did You Know?, return to the article and click \\\"What links here\\\" to the left of the article. Then, in the dropdown menu provided for namespace, choose Wikipedia and click \\\"Go\\\". When you find \\\"Wikipedia:Recent additions\\\" and a number, click it and search for the article name.\\n\\nCurrent archive\"\n}\n\n... etc\n```\n\nMore examples are available in the [/examples](https://github.com/missinglink/huntsman/tree/master/examples \"Title\") directory\n\n---\n\n## How it works\n\nHuntsman takes one or more 'seed' urls with the `spider.queue.add()` method.\n\nOnce the process is kicked off with `spider.start()`, it will take care of extracting links from the page and following only the pages we want.\n\nTo define which pages are crawled use the `spider.on()` function with a string or regular expression.\n\nEach page will only be crawled once. If multiple regular expressions match the uri, they will all be called.\n\n**Page URLs which do not match an `on` condition will never be crawled**\n\n---\n\n## Configuration\n\nThe spider has default settings, you can override them by passing a settings object when you create a spider.\n\n```javascript\n// use default settings\nvar huntsman = require('huntsman');\nvar spider = huntsman.spider();\n```\n\n```javascript\n// override default settings\nvar huntsman = require('huntsman');\nvar spider = huntsman.spider({\n  throttle: 10, // maximum requests per second\n  timeout: 5000 // maximum gap of inactivity before exiting (in milliseconds)\n});\n```\n\n---\n\n## Crawling a site\n\nHow you configure your spider will vary from site to site, generally you will only be looking for for pages with a specific url format.\n\n### Scrape product information from amazon\n\nIn this example we can see that amazon product uris all seem to share the format `'/gp/product/'`.\n\nAfter queueing the seed uri `http://www.amazon.co.uk/` huntsman will follow all the product pages it finds recursively.\n\n```javascript\n/** Example of scraping products from the amazon website **/\n\nvar huntsman = require('huntsman');\nvar spider = huntsman.spider();\n\nspider.extensions = [\n  huntsman.extension( 'recurse' ), // load recurse extension & follow anchor links\n  huntsman.extension( 'cheerio' ) // load cheerio extension\n];\n\n// target only product uris\nspider.on( '/gp/product/', function ( err, res ){\n\n  if( !res.extension.cheerio ) return; // content is not html\n  var $ = res.extension.cheerio;\n\n  // extract product information\n  var product = {\n    uri: res.uri,\n    heading: $('h1.parseasinTitle').text().trim(),\n    image: $('img#main-image').attr('src'),\n    description: $('#productDescription').text().trim().substr( 0, 50 )\n  };\n\n  console.log( product );\n\n});\n\nspider.queue.add( 'http://www.amazon.co.uk/' );\nspider.start();\n```\n\n### Find pets for sale on craigslist in london \n\nMore complex crawls may require you to specify hub pages to follow before you can get to the content you really want. You can add an `on` event without a callback & huntsman will still follow and extract links from it.\n\n```javascript\n/** Example of scraping information about pets for sale on cragslist in london **/\n\nvar huntsman = require('huntsman');\nvar spider = huntsman.spider({\n  throttle: 2\n});\n\nspider.extensions = [\n  huntsman.extension( 'recurse' ), // load recurse extension & follow anchor links\n  huntsman.extension( 'cheerio' ), // load cheerio extension\n  huntsman.extension( 'stats' ) // load stats extension\n];\n\n// target only pet uris\nspider.on( /\\/pet\\/(\\w+)\\.html$/, function ( err, res ){\n\n  if( !res.extension.cheerio ) return; // content is not html\n  var $ = res.extension.cheerio;\n\n  // extract listing information\n  var listing = {\n    heading: $('h2.postingtitle').text().trim(),\n    uri: res.uri,\n    image: $('img#iwi').attr('src'),\n    description: $('#postingbody').text().trim().substr( 0, 50 )\n  };\n\n  console.log( listing );\n\n});\n\n// hub pages\nspider.on( /http:\\/\\/london\\.craigslist\\.co\\.uk$/ );\nspider.on( /\\/pet$/ );\n\nspider.queue.add( 'http://www.craigslist.org/about/sites' );\nspider.start();\n```\n\n---\n\n## Extensions\n\nExtensions have default settings, you can override them by passing an optional second argument when the extension is loaded.\n\n```javascript\n// loading an extension\nspider.extensions = [\n  huntsman.extension( 'extension_name', options )\n];\n```\n\n### recurse\n\nThis extension extracts links from html pages and then adds them to the queue.\n\nThe default patterns only target anchor tags which use the http protocol, you can change any of the default patterns by declaring them when the extension is loaded.\n\n```javascript\n// default patterns\nhuntsman.extension( 'recurse', {\n  pattern: {\n    search: /a([^>]+)href\\s?=\\s?['\"]([^\"'#]+)/gi,\n    refine: /['\"]([^\"'#]+)$/,\n    filter: /^https?:\\/\\//\n  }\n})\n```\n\n- `search` must be a `global` regexp and is used to target the links we want to extract.\n- `refine` is a regexp used to extract the bits we want from the `search` regex matches.\n- `filter` is a regexp that must match or links are discarded.\n\n\n```javascript\n// extract both anchor tags and script tags\nhuntsman.extension( 'recurse', {\n  pattern: {\n    search: /(a([^>]+)href|script([^>]+)src)\\s?=\\s?['\"]([^\"'#]+)/gi, // <a> or <script>\n  }\n})\n```\n\n```javascript\n// ignore query segment of uris (exclude everything from '?' onwards)\nhuntsman.extension( 'recurse', {\n  pattern: {\n    search: /a([^>]+)href\\s?=\\s?['\"]([^\"'#\\?]+)/gi // charlist for end of uri [^\"'#\\?]\n  }\n})\n```\n\n```javascript\n// avoid some file extensions\nhuntsman.extension( 'recurse', {\n  pattern: {\n    filter: /^https?:\\/\\/.*(?!\\.(pdf|png|jpg|gif|zip))....$/i, // regex lookahead\n  }\n})\n```\n\n```javascript\n// avoid all uris with three letter file extensions\nhuntsman.extension( 'recurse', {\n  pattern: {\n    filter: /^https?:\\/\\/.*(?!\\.\\w{3})....$/, // exclude three letter file extensions\n  }\n})\n```\n\n```javascript\n// stay on one domain\nhuntsman.extension( 'recurse', {\n  pattern: {\n    filter: /^https?:\\/\\/www\\.example\\.com/i, // uris must be prefixed with this domain\n  }\n})\n```\n\nBy default `recurse` converts relative urls to absolute urls and strips fragment identifiers and trailing slashes.\n\nIf you need even more control you can override the `resolver` & `normaliser` functions to modify these behaviours.\n\n### cheerio\n\nThis extension parses html and provides jquery-style selectors & functions.\n\n```javascript\n// default settings\nhuntsman.extension( 'cheerio', { lowerCaseTags: true } )\n```\n\nThe `res.extension.cheerio` function is available in your `on` callbacks when the response body is HTML.\n\n```javascript\nspider.on( 'example.com', function ( err, res ){\n\n  // use jquery-style selectors & functions\n  var $ = res.extension.cheerio;\n  if( !$ ) return; // content is not html\n\n  console.log( res.uri, $('h1').text().trim() );\n\n});\n```\n\n`cheerio` reference: https://github.com/MatthewMueller/cheerio\n\n### json\n\nThis extension parses the response body with `JSON.parse()`.\n\n```javascript\n// enable json\nhuntsman.extension( 'json' )\n```\n\nThe `res.extension.json` function is available in your `on` callbacks when the response body is json.\n\n```javascript\nspider.on( 'example.com', function ( err, res ){\n\n  var json = res.extension.json;\n  if( !json ) return; // content is not json\n\n  console.log( res.uri, json );\n\n});\n```\n\n### links\n\nThis extension extracts links from html pages and returns the result.\n\nIt exposes the same functionality that the `recurse` extension uses to extract links.\n\n```javascript\n// enable extension\nhuntsman.extension( 'links' )\n```\n\nThe `res.extension.links` function is available in your `on` callbacks when the response body is a string.\n\n```javascript\nspider.on( 'example.com', function ( err, res ){\n\n  if( !res.extension.links ) return; // content is not a string\n\n  // extract all image tags from body\n  var images = res.extension.links({\n    pattern: {\n      search: /(img([^>]+)src)\\s?=\\s?['\"]([^\"'#]+)/gi, // extract img tags\n      filter: /\\.jpg|\\.gif|\\.png/i // filter file types\n    }\n  });\n\n  console.log( images );\n\n});\n```\n\n### stats\n\nThis extension displays statistics about pages crawled, error counts etc.\n\n```javascript\n// default settings\nhuntsman.extension( 'stats', { tail: false } )\n```\n\n---\n\n## Custom queues and response storage adapters\n\nI'm currently working on being able to persist the job queue via something like redis and potentially caching http responses in mongo with a TTL.\n\nIf you live life on the wild side, these adapters can be configured when you create a spider.\n\nPull requests welcome.\n\n---\n\n## Build Status\n\n[![Build Status](https://travis-ci.org/missinglink/huntsman.png?branch=master)](https://travis-ci.org/missinglink/huntsman)\n\n[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/missinglink/huntsman/trend.png)](https://bitdeli.com/free \"Bitdeli Badge\")\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2013 Peter Johnson &lt;@insertcoffee&gt;\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","maintainers":[{"name":"missinglink","email":"peter@missinglink.co.nz"}],"time":{"modified":"2022-06-18T23:13:43.632Z","created":"2013-05-28T11:29:49.970Z","0.0.1":"2013-05-28T11:29:59.644Z","0.1.0":"2013-07-13T03:46:47.121Z","0.1.1":"2013-07-17T11:40:44.315Z","0.1.2":"2013-07-17T13:18:34.211Z","0.1.3":"2013-07-17T14:04:01.441Z","0.1.4":"2013-07-19T17:28:44.032Z","0.1.5":"2013-07-19T20:23:43.889Z","0.1.6":"2013-07-20T00:28:44.799Z","0.1.7":"2013-07-20T02:52:52.615Z","0.1.8":"2013-07-20T02:58:07.317Z","0.1.9":"2013-07-20T03:17:38.412Z","0.2.0":"2013-07-20T06:44:47.611Z","0.2.1":"2013-07-20T07:01:07.154Z","0.2.2":"2013-07-21T17:18:27.275Z","0.2.3":"2013-07-22T14:41:55.578Z","0.2.4":"2013-09-22T15:47:52.702Z","0.2.5":"2013-09-22T16:16:07.460Z","0.2.6":"2013-11-30T02:51:47.547Z","0.2.7":"2014-01-30T07:09:18.600Z","0.2.8":"2014-04-19T07:36:06.810Z","0.2.9":"2014-05-10T04:08:55.182Z","0.2.10":"2014-09-03T08:36:09.533Z","0.2.11":"2014-09-03T09:18:26.745Z","0.2.12":"2015-02-24T19:36:32.293Z","0.3.0":"2015-09-21T16:34:34.702Z"},"repository":{"type":"git","url":"git://github.com/missinglink/huntsman.git"},"author":{"name":"Peter Johnson","email":"@insertcoffee","url":"https://github.com/missinglink/"},"readmeFilename":"README.md","homepage":"https://github.com/missinglink/huntsman#readme","keywords":["spider","crawler","crawl","huntsman","robot","aync"],"bugs":{"url":"http://github.com/missinglink/huntsman/issues"},"users":{"mimmo1":true,"leonning":true,"augbog":true,"detj":true,"nguru":true,"goose":true,"timdp":true,"lwdthe1":true},"license":"MIT"}