{"_id":"puppeteer-infinite-scroller","_rev":"2-fe040d068ed120d40eaabe06ac8ea71a","name":"puppeteer-infinite-scroller","dist-tags":{"latest":"1.0.2"},"versions":{"1.0.0":{"name":"puppeteer-infinite-scroller","version":"1.0.0","description":"Provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.","main":"index.js","scripts":{"test":"jest","example":"node examples/index.js","release:major":"npm version $(semver $npm_package_version -i major) && npm publish --tag latest","release:minor":"npm version $(semver $npm_package_version -i minor) && npm publish --tag latest","release:patch":"npm version $(semver $npm_package_version -i patch) && npm publish --tag latest"},"repository":{"type":"git","url":"git+https://github.com/DulanHewage/puppeteer-infinite-scroller.git"},"license":"MIT","author":{"name":"Dulan Hewage","email":"dulanhewage2@gmail.com"},"bugs":{"url":"https://github.com/DulanHewage/puppeteer-infinite-scroller/issues"},"homepage":"https://github.com/DulanHewage/puppeteer-infinite-scroller#readme","keywords":["headless-chrome","puppeteer","scraping","scroll","auto-scroller","infinite-scroll"],"devDependencies":{"jest":"^29.6.2","puppeteer":"^21.0.1"},"_id":"puppeteer-infinite-scroller@1.0.0","gitHead":"6d70e30dd435c8e0331b94fc1257ad67d00a22e5","_nodeVersion":"18.11.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-09wS2jQFJJSpI3JGzClGq8aVuMaTsRjfKVdEIulMkHLXbsXxAo5NRZUWLiQqWAHCk/85KRxt/8dVIEyCtb3bZg==","shasum":"9b465a9e7231ea1e246065341afa846e9825bea7","tarball":"https://registry.npmjs.org/puppeteer-infinite-scroller/-/puppeteer-infinite-scroller-1.0.0.tgz","fileCount":10,"unpackedSize":24589,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCy2Rh/oq63KrbmLdcVKsbGsMvryp3SYG2ntqWsXtXO3gIgSBfVL/7Rbz4DFkelPBEeiPILBaSd8Wr8XWw09EIzkUs="}]},"_npmUser":{"name":"dulanh","email":"dulanhewage2@gmail.com"},"directories":{},"maintainers":[{"name":"dulanh","email":"dulanhewage2@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/puppeteer-infinite-scroller_1.0.0_1692640867618_0.8600636881868411"},"_hasShrinkwrap":false},"1.0.1":{"name":"puppeteer-infinite-scroller","version":"1.0.1","description":"Provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.","main":"lib/index.js","scripts":{"test":"jest","build":"esbuild index.js --bundle --platform=node --target=node10.4 --outfile=out.js","example":"node examples/index.js","release:major":"npm version $(semver $npm_package_version -i major) && npm publish --tag latest","release:minor":"npm version $(semver $npm_package_version -i minor) && npm publish --tag latest","release:patch":"npm version $(semver $npm_package_version -i patch) && npm publish --tag latest"},"repository":{"type":"git","url":"git+https://github.com/DulanHewage/puppeteer-infinite-scroller.git"},"license":"MIT","author":{"name":"Dulan Hewage","email":"dulanhewage2@gmail.com"},"bugs":{"url":"https://github.com/DulanHewage/puppeteer-infinite-scroller/issues"},"homepage":"https://github.com/DulanHewage/puppeteer-infinite-scroller#readme","keywords":["headless-chrome","puppeteer","scraping","scroll","auto-scroller","infinite-scroll"],"devDependencies":{"esbuild":"^0.19.2","jest":"^29.6.2","puppeteer":"^21.0.1"},"_id":"puppeteer-infinite-scroller@1.0.1","gitHead":"c080723396f2f73151fa511d7dff37c944516388","_nodeVersion":"18.11.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-ulQhrSKEiTVAiT+FXrvt0jHlYCPogLCMzB2Zo47WEy+NTkdW4UagFTIX/Y8lAZHMs9gnMcLV2HwHsqZVCUYZPw==","shasum":"e9509708f2a51a578e35339023ee2483a9e1f293","tarball":"https://registry.npmjs.org/puppeteer-infinite-scroller/-/puppeteer-infinite-scroller-1.0.1.tgz","fileCount":6,"unpackedSize":7749,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDqlkwuuUvbYbFNNgjYtI4PyWopg5Zj2LoPKz4pJvwAZAIhAIg980s50jenjalvboNTUVO8JRladM5xR1GXYv2Csqc+"}]},"_npmUser":{"name":"dulanh","email":"dulanhewage2@gmail.com"},"directories":{},"maintainers":[{"name":"dulanh","email":"dulanhewage2@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/puppeteer-infinite-scroller_1.0.1_1693170369022_0.8449723085338248"},"_hasShrinkwrap":false},"1.0.2":{"name":"puppeteer-infinite-scroller","version":"1.0.2","description":"Provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.","main":"lib/index.js","typings":"lib/index.d.ts","scripts":{"test":"jest","build":"node build/index.js","example":"node examples/index.js","release:major":"npm version $(semver $npm_package_version -i major) && npm publish --tag latest","release:minor":"npm version $(semver $npm_package_version -i minor) && npm publish --tag latest","release:patch":"npm version $(semver $npm_package_version -i patch) && npm publish --tag latest"},"repository":{"type":"git","url":"git+https://github.com/DulanHewage/puppeteer-infinite-scroller.git"},"license":"MIT","author":{"name":"Dulan Hewage","email":"dulanhewage2@gmail.com"},"bugs":{"url":"https://github.com/DulanHewage/puppeteer-infinite-scroller/issues"},"homepage":"https://github.com/DulanHewage/puppeteer-infinite-scroller#readme","keywords":["headless-chrome","puppeteer","scraping","scroll","auto-scroller","infinite-scroll"],"devDependencies":{"esbuild":"^0.19.2","jest":"^29.6.2","npm-dts":"^1.3.12","puppeteer":"^21.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"_id":"puppeteer-infinite-scroller@1.0.2","gitHead":"394c01ff48efe1d65197ef374162d29f2bab8a2e","_nodeVersion":"18.11.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-EBZzRbtBrr80rTKpEqjjn4sefsOJax/vJaU8TVH6iHapq3EKtZ3CBGjWa7/cQ9gEQF4KiZGBsQvo3Mrqnp60Bg==","shasum":"721755015d1ea0f765d7a1e7615eeeb519c234f0","tarball":"https://registry.npmjs.org/puppeteer-infinite-scroller/-/puppeteer-infinite-scroller-1.0.2.tgz","fileCount":5,"unpackedSize":7446,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHgpDKD+4RQKeTSFEnUSjaKDCXNQbzzKNzWKf2tselG/AiBwUMCKLWG1wQz2SWnwmLeJ87AsvOgcpWfJf1BXdQf31g=="}]},"_npmUser":{"name":"dulanh","email":"dulanhewage2@gmail.com"},"directories":{},"maintainers":[{"name":"dulanh","email":"dulanhewage2@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/puppeteer-infinite-scroller_1.0.2_1699023089149_0.5968032781078081"},"_hasShrinkwrap":false}},"time":{"created":"2023-08-21T18:01:07.617Z","1.0.0":"2023-08-21T18:01:07.779Z","modified":"2023-11-03T14:51:29.531Z","1.0.1":"2023-08-27T21:06:09.206Z","1.0.2":"2023-11-03T14:51:29.337Z"},"maintainers":[{"name":"dulanh","email":"dulanhewage2@gmail.com"}],"description":"Provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.","homepage":"https://github.com/DulanHewage/puppeteer-infinite-scroller#readme","keywords":["headless-chrome","puppeteer","scraping","scroll","auto-scroller","infinite-scroll"],"repository":{"type":"git","url":"git+https://github.com/DulanHewage/puppeteer-infinite-scroller.git"},"author":{"name":"Dulan Hewage","email":"dulanhewage2@gmail.com"},"bugs":{"url":"https://github.com/DulanHewage/puppeteer-infinite-scroller/issues"},"license":"MIT","readme":"# Puppeteer Infinite Scroller\n[![NPM Version](https://badge.fury.io/js/puppeteer-infinite-scroller.svg?style=flat)](https://www.npmjs.com/package/puppeteer-infinite-scroller)\n[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/DulanHewage/puppeteer-infinite-scroller/issues)\n\nPuppeteer-Infinite-Scroller provides a simple and efficient solution for scraping data loaded through infinite scrolling on web pages using Puppeteer.\n\n## Installation\n\nYou can install the package using npm:\n\n```bash\nnpm install puppeteer-infinite-scroller\n```\n\n## Usage\nImport the puppeteerInfiniteScroller function from the package and use it to scrape data from infinite scrolling web pages.\n\n```javascript\nconst puppeteer = require(\"puppeteer\");\nconst puppeteerInfiniteScroller = require('puppeteer-infinite-scroller');\n\n(async () => {\n  const pageUrl = \"https://infiniteajaxscroll.com/examples/blocks/\";\n  const browser = await puppeteer.launch();\n  const page = await browser.newPage();\n  await page.setViewport({\n    width: 1200,\n    height: 800,\n  });\n\n  await page.goto(pageUrl);\n  await page.waitForSelector(\".blocks .blocks__block\");\n\n  const options = {\n    scrollDelay: 1000,      // Milliseconds between scrolls\n    itemCount: 50,          // Number of items to scrape\n    selector: '.blocks .blocks__block',      // CSS selector for items\n    // OR\n    // pageFunction: () => { /* Custom page function for scraping */ }\n  };\n\n  const scrapedData = await puppeteerInfiniteScroller(page, options);\n\n  console.log(scrapedData);\n\n  await browser.close();\n})();\n```\n\n## Options\n\nThe following options can be configured when using the `puppeteerInfiniteScroller` function:\n\n- `scrollDelay` (optional): Milliseconds between scrolls. Default is 1000ms.\n- `itemCount` (optional): Number of items to scrape. Default is 10.\n- `selector` (optional): CSS selector for the items. Either this or `pageFunction` must be provided.\n- `pageFunction` (optional): Custom function for scraping data from the page. Either this or `selector` must be provided.\n\n## Example with pageFunction option\n\n```javascript\nconst puppeteer = require(\"puppeteer\");\nconst puppeteerInfiniteScroller = require('puppeteer-infinite-scroller');\n\n(async () => {\n  const pageUrl = \"https://infiniteajaxscroll.com/examples/blocks/\";\n  const browser = await puppeteer.launch();\n  const page = await browser.newPage();\n  await page.setViewport({\n    width: 1200,\n    height: 800,\n  });\n\n  await page.goto(pageUrl);\n  await page.waitForSelector(\".blocks .blocks__block\");\n\n  function extractElements() {\n    const items = [];\n    const extractedElements = document.querySelectorAll(\".blocks .blocks__block\");\n    for (let element of extractedElements) {\n      items.push({\n        class: element.getAttribute(\"class\"),\n        id: element.getAttribute(\"id\"),\n        tagName: element.tagName,\n      });\n    }\n    return items;\n  }\n\n  const options = {\n    scrollDelay: 1000,      // Milliseconds between scrolls\n    itemCount: 50,          // Number of items to scrape\n    pageFunction: extractElements\n  };\n\n  const scrapedData = await puppeteerInfiniteScroller(page, options);\n\n  console.log(scrapedData);\n\n  await browser.close();\n})();\n```\n\nLicense\nThis project is licensed under the MIT License - see the LICENSE file for details.\n","readmeFilename":"README.md"}