{"_id":"crawlme","_rev":"25-ec1049c6b2c8fd96782fc7112eb5262d","name":"crawlme","description":"Makes your ajax web application indexable by search engines by generating html snapshots on the fly. Caches results for blazing fast responses and better page ranking.","dist-tags":{"latest":"0.0.7"},"versions":{"0.0.1":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application crawlable by search engines","keywords":["ajax","crawling","google","indexing"],"version":"0.0.1","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.1","dist":{"shasum":"6102281e767f93a2fdbaed3d536eecbe1eead05b","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.1.tgz","integrity":"sha512-6VW/LYzxwJLFgn0v4Nfp+Up3b9HRFxFwZNggwXBdEAuXvVvWYv3H1hvTDdYpC7FR7L8yfhnpIiSP8Vc1/z4QEw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGzFWSr2Nn0k8WXkzPDJ7WMB93PUyqtqk2ioK7BE1jA+AiEA6tpHLym2fUdQh10YvytrEY8Pss6x3lIs5MI5wsnmQi0="}]},"_npmVersion":"1.1.62","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}],"directories":{}},"0.0.2":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application crawlable by search engines","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.2","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.2","dist":{"shasum":"f66ec6dd88dee6d880bc634ab9de328cb429b829","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.2.tgz","integrity":"sha512-Sqx9ePS10Oo8k3FY9Vmb+K4771VA6IHeCMnwOjzeP+C6fUZfrA99/hCYd8+GwPz54z4lHewyDhOxJfYkODbGDQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCID4gIilDNmnE0Mb9ioUF/e6VRyzRpvmuA1Vt+SFFknpPAiB1tyQT/S41zTlqpdHA9rD9JCZ4O05Z08jI0cN98XOoiw=="}]},"_npmVersion":"1.1.63","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}],"directories":{}},"0.0.3":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application crawlable by search engines by generating html snapshots on the fly","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.3","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.3","dist":{"shasum":"c31624962ed63e41b33b961f5033596679a29ffa","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.3.tgz","integrity":"sha512-F2aGeVle1GpDLMlixq1G+d6qppww2cFP/90zKadg1+5/FtjBV/hmFIjLVZLP+TYCCpleqqDWdPnPfJpuSclO9A==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHPYURz0PMH975ptVv5ds4MRZ3XEpJ+uX/EItU4ESO59AiEAsDxH9U3gZNXx7js4EZZmYxAinysKmgxMKpeLmxGcPr4="}]},"_from":".","_npmVersion":"1.2.2","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}],"directories":{}},"0.0.4":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application crawlable by search engines by generating html snapshots on the fly","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.4","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.4","dist":{"shasum":"a093afdd6a0ec7775be2bfdc7f044be79c5458fc","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.4.tgz","integrity":"sha512-j/ybCbx2yWbn4rQ+gZR5TDIpSeYO/+0g0a8tXzuwfi+2iQMDGmYMI+wOcKbqaeJqAgep8HJNsVsrNWHJyRLueg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQChD5eY4f2ssYKYCIgh2IEjJNWbe3w9hfhqT/d5Zb8QawIgSw3uJpM38RIIGhdutu7HQsLq/RkibTCvlaW0q66sGbE="}]},"_from":".","_npmVersion":"1.2.2","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}],"directories":{}},"0.0.5":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application indexable by search engines by generating html snapshots on the fly. Caches results for blazing fast responses and better page ranking.","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.5","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x","lru-cache":"2.3.x","async":"0.2.6"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.5","dist":{"shasum":"c5ee90a914d0e5a5255c0bb6c8d0cecff647291d","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.5.tgz","integrity":"sha512-ZJc1/fYb8mQ8F/nLKg5J5e/ewwQGa1NJgXhiZsh+XGA3a8PHbxOINjoXEbOvNxFb8tvxD1ijhPrPENpq5Mi9vg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDpK3fY/3Hxp+T/hN0wp+APLl7noosXIwmFZDFIlzFJGAIgfRsQJxVEBbM9C8iM3QvffN2hF3scK4YUhh/kKBvZzE8="}]},"_from":".","_npmVersion":"1.2.14","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}]},"0.0.6":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application indexable by search engines by generating html snapshots on the fly. Caches results for blazing fast responses and better page ranking.","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.6","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x","lru-cache":"2.3.x","async":"0.2.6"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.6","dist":{"shasum":"1037c78954cb08b449b495e7902c2bd6815298fb","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.6.tgz","integrity":"sha512-gE5+VS9nmqYd+9EerSaf3arxcGSqdPq8FhGW/lPbNC2bF7yM0w7EJ59DFQt0BW99Ws7/nL/DVgCoEmfYft2eIw==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIE+XYSEW3gcf7JdOiai5soTi0Y3KptxAbUbctbBFRVFdAiBbBiClEWAo6J9AuYOUAwvRxe+t8zDOzJTvZ+g0aDYvBw=="}]},"_from":".","_npmVersion":"1.2.2","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}]},"0.0.7":{"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"name":"crawlme","description":"Makes your ajax web application indexable by search engines by generating html snapshots on the fly. Caches results for blazing fast responses and better page ranking.","keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"version":"0.0.7","main":"./index.js","engines":{"node":">=0.6.10"},"dependencies":{"zombie":"2.x.x","lru-cache":"2.3.x","async":"0.2.6"},"devDependencies":{"connect":"2.x.x","express":"3.x.x","mocha":"1.6.x","request":"2.11.x","should":"1.2.x"},"scripts":{"test":"mocha --reporter spec --bail"},"_id":"crawlme@0.0.7","dist":{"shasum":"c3bc4970f84e21fa0b5e41d997caf5df87228ccd","tarball":"https://registry.npmjs.org/crawlme/-/crawlme-0.0.7.tgz","integrity":"sha512-vXMCd1Q20YAhNKlOqwc1wivIWQx2jGT/FR9cnHxGzk/w1Zrwuw/gjK/TO0Fid+9JAXVPsf9JksXBMhquqm5XIg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDxDLSDFyjh08MoAd4XLYgV44OEAEL6Pff9rHhJ10fzAgIhAOkzsudU5i1bzxFQErsuUc0241wGQ01lpqThvZhxWGrm"}]},"_from":".","_npmVersion":"1.3.21","_npmUser":{"name":"arokor","email":"aron@kornhall.se"},"maintainers":[{"name":"arokor","email":"aron@kornhall.se"}]}},"readme":"#Crawlme\nA Connect/Express middleware that makes your node.js web application indexable by search engines. Crawlme generates static HTML snapshots of your JavaScript web application on the fly and has a built in periodically refreshing in-memory cache, so even though the snapshot generation may take a second or two, search engines will get them really fast. This is beneficial for SEO since response time is one of the factors used in the page rank algorithm.\n\nMaking ajax applications crawlable has always been tricky since search engines don't execute the JavaScript on the web sites they crawl. The solution to this is to provide the search engines with pre-rendered HTML versions of each page on your site, but creating those HTML versions has until now been a tedious and error prone process with many manual steps. Crawlme fixes this by rendering HTML snapshots of your web application on the fly whenever the Googlebot crawls your site. Apart from making the process of more or less manually creating indexable HTML versions of your site obsolete, this also has the benefit that Google will always index the latest version of your site and not some old pre-rendered version.\n\nFollow [optimalbits](http://twitter.com/optimalbits) for news and updates regarding this library.\n\n##How to use\n1. Make you ajax app use the hashbang #! instead of just the # in urls. This tells Google that those urls support ajax crawling and indexing.\n2. Insert the Crawlme middleware before your server in the chain of Connect/Express middlewares.\n3. Sit back and relax. Crawlme takes care of the rest. :)\n\n##Example\n    var\n      connect = require('connect'),\n      http = require('http'),\n      crawlme = require('crawlme');\n\n    var app = connect()\n      .use(crawlme())\n      .use(connect.static(__dirname + '/webroot'));\n\n    http.createServer(app).listen(3000);\n\n##Install\n    npm install crawlme\n\n##How it works\nGoogle detects that your page *your.server.com/page.html#!key=value* is ajax-crawlable by the hashbang #! in the url. The Googlebot doesn't evaluate JavaScript so it can't index the page directly. Instead it tries to get the URL *your.server.com/page.html?_escaped_fragment_=key=value* and expects to find an HTML snapshot of your page there. Crawlme will catch all requests to this kind of URLs and generate a HTML snapshot of the original ajax page on the fly.\n\nMore info on Google's ajax crawling can be found [here](https://developers.google.com/webmasters/ajax-crawling/docs/getting-started)\n\n##Test\n###Running the unit tests\n1. Install dev-dependencies\n    npm install\n2. Run the test suite\n    npm test\n\n###Testing that your ajax web application is crawlable\nPick an ajax url to some part of your web application like for example *your.server.com/page.html#!key=value*\nNow replace the hashbang with *?_escaped_fragment_=*. The new URL will be *your.server.com/page.html?_escaped_fragment_=key=value*\nNow go to that URL. Crawlme should intercept the request and render an HTML snapshot of your page.\n\nYou can also use Googles [Fetch as Googlebot](http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=158587) tool.\n\n##Reference\n\n    crawlme(options)\n\n__Arguments__\n \n    options  {Object} options object\n    \n##Options\nCrawlme provides the following configuration options:\n- `waitFor`   The time (in ms) crawlme waits before it assumes that your ajax page has finished loading and takes an HTML snapshot. Set this high enough to make sure that your page loads completely before the snapshot is taken. Defaults to 1000ms.\n- `protocol`  The protocol crawlme should use to get the ajax pages. If crawlme runs under express this is determined automatically. Under connect this option is used. (defaults to http)\n- `cacheSize`  The size of the cache that crawlme should use to cache the snapshots. String.prototype.length is used to determine the \"size\" of each snapshot. A cache size of 0 means no cache. Defaults to 2^20.\n- `cacheRefresh`  The number of seconds between cache refreshes. Defaults to 15 minutes.\n\n##Under the hood\nCrawlme uses the excellent headless browser [zombie.js](http://zombie.labnotes.org/) to render the HTML snapshots.\n\n##License \n\n(The MIT License)\n\nCopyright (c) 2012 Optimal Bits Sweden AB (http://optimalbits.com)\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","maintainers":[{"name":"arokor","email":"aron@kornhall.se"}],"time":{"modified":"2022-06-14T02:44:26.201Z","created":"2012-10-09T21:25:40.591Z","0.0.1":"2012-10-09T21:25:42.633Z","0.0.2":"2012-11-04T19:31:27.150Z","0.0.3":"2013-01-31T08:03:43.645Z","0.0.4":"2013-03-07T15:35:04.352Z","0.0.5":"2013-04-03T16:13:21.213Z","0.0.6":"2013-06-27T09:45:54.361Z","0.0.7":"2014-04-07T20:18:17.955Z"},"author":{"name":"Aron Kornhall","email":"aron@optimalbits.com","url":"http://optimalbits.com"},"keywords":["ajax","crawling","google","indexing","SEO","Search Engine Optimization"],"readmeFilename":"README.md"}