{"_id":"@charleswong28/bearer-api-scraping","_rev":"7-5a1ad69e15c61453ddf23641df23641b","name":"@charleswong28/bearer-api-scraping","dist-tags":{"latest":"0.8.0"},"versions":{"0.1.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.1.0","description":"Scraping Library with auto-retry","main":"index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"dcd29255d5c7cd5329c9b928e7606ed3a648ca13","_id":"@charleswong28/bearer-api-scraping@0.1.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-x785L/UfUOWDHhgKJeKY0G25FDoj0rLqimbcc3olHwRFv9jR6k2/chjoSc5Vwrmf4nKxxQ+p3asSJaqsdyHL8g==","shasum":"87c747d0fe51866891374186fff46013ced97fe0","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.1.0.tgz","fileCount":6,"unpackedSize":13359,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIEXaQyKRM0uV8Gi17d1W54DoFCQLpDoMiQoTM29WEDHLAiEA7PloaKzjWvUcTvZCprRtJ8c8HoNQjh7Uvm4PwXyN1Ms="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.1.0_1683485389116_0.010065022957935765"},"_hasShrinkwrap":false},"0.2.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.2.0","description":"Scraping Library with auto-retry","main":"index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"dafbb38397ab7383c8e30b0c09339f04a1071de8","_id":"@charleswong28/bearer-api-scraping@0.2.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-Q0PxDpYooxbreez/odEPrqN2Y+Hd3Dy9vAdTtiHk/9/l6Xsub4LgppMi8+R0gZqYrSHS80klgHEXqBWFlHVEOw==","shasum":"f59331f151840031900d5544f4a39bf2130a240e","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.2.0.tgz","fileCount":6,"unpackedSize":13799,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDPqYRCFcqFIDZlXnACKAK3/g/z7hgtMQjj/ayEkA6BDAiAB36JeMgFNX+CE7x5DlmXIoVAQJl+P//bg7zFGZvsSDw=="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.2.0_1683534145967_0.8317316354796933"},"_hasShrinkwrap":false},"0.3.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.3.0","description":"Scraping Library with auto-retry","main":"index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"6c5e7206821997d229c42f7ec6cd7db0887fb5e7","_id":"@charleswong28/bearer-api-scraping@0.3.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-kJdFmzDh6kTgwBw0mAv9RJCWJuEJH9b7+QoO2Suo5r0KF0VuGN2cUb8lG3ursnMj7f0vhBiXH165sg4Zb5LonA==","shasum":"6b3b2cc88c0d9ead37d7d06f0bc780761e60ff6c","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.3.0.tgz","fileCount":6,"unpackedSize":13986,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGfYni1uxmBw8lQympzgNmZwzKqOxKQIJ1S3Jv0tUyUBAiEA4EjWNR20gz+Ghl4ogGW7seTXDuUV8yfU5sXK/AioWxc="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.3.0_1683534409063_0.9379239845062921"},"_hasShrinkwrap":false},"0.4.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.4.0","description":"Scraping Library with auto-retry","main":"index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"fe617fc5194f5f29f2e8053fded39a46c7d1c481","_id":"@charleswong28/bearer-api-scraping@0.4.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-evIApDbdHH/dm7fS/JByxRiLPUnIU0c+ZlX1eX/s8OZ+LO5gz2/xaw1+LuO0iI4+QtnYoNcfd2ZD7R6p8Wb80Q==","shasum":"d19aa1eee69a36224fa5ac72a21403b08f3be19d","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.4.0.tgz","fileCount":6,"unpackedSize":14172,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGGPgymSIBmVjECHoIk6xzi3yJzaTcO7B2hVvAtdZZwCAiEA3s+ZQ42KxlIeLNXnWoufd5fLko4CoxGYogLmEkdvYoM="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.4.0_1683534922539_0.47474861034033866"},"_hasShrinkwrap":false},"0.5.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.5.0","description":"Scraping Library with auto-retry","main":"dist/index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"72e92aca715351ca4946e5ff0c3933ab349b108f","_id":"@charleswong28/bearer-api-scraping@0.5.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-1veNcWd1IXRNbLEuoPDR2KFsNMYvf899QQt0y/4gmzoLINg+AT5kX6BZL/eS2hkWzUsmgPOsqd9c5EfglgYS1A==","shasum":"4b79541790499e4a9339e7459bdd5658da871f17","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.5.0.tgz","fileCount":10,"unpackedSize":26948,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHmr4/Dwam9DM1MFy76IVeBs1K3egwouprTp05/xEzcVAiEAi5vonvxWcIWwMD/bc0JBv91BR9262FbbwMJieOEjk+0="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.5.0_1683535114122_0.36845249259999435"},"_hasShrinkwrap":false},"0.6.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.6.0","description":"Scraping Library with auto-retry","main":"dist/index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"99614439311a3d8e4c5f08e5c180ea4fc58a0c16","_id":"@charleswong28/bearer-api-scraping@0.6.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-1dE7ahCWOtmXXww4Tk9xyhlbmZ2n6DRtHABvrvZxt9SeUAzxgyGshTNYeCmwDICuL0CwXszwf0SO4Tphk+7StA==","shasum":"abc432bede9c3ae2d5d9cc296cae35d851f551d0","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.6.0.tgz","fileCount":10,"unpackedSize":37357,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIADaBOA4SNEJZNu8/xRhU4PyrDo39h0StNcJvBdEbFf2AiB/eDbv67cVBDcxWuf/HhdU8cMbh0WgmPF3sd6WcB8RAg=="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.6.0_1683628170221_0.1990141757384607"},"_hasShrinkwrap":false},"0.7.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.7.0","description":"Scraping Library with auto-retry","main":"dist/index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"ba490551028514f2be327d1388519f9ca3ddefe8","_id":"@charleswong28/bearer-api-scraping@0.7.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-6nu6FzAFhs9dufSnul9NJ9ZNh+IUB7hhl5CQZ4080e618fNMkQLXDahcPmCFaS/Vgr4pVNCz6sdqzTm7FXQ+PA==","shasum":"ed3f56b5a38ddf991bd28666203152899ddf015c","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.7.0.tgz","fileCount":10,"unpackedSize":37523,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIAN9IS/QL4RosisF0tmkSfuwe1ZF/WMY/X+C57HAbdoOAiEA8aTbywOeNlr6ox5J9mrI2x3eIYzrZl2k8mgKYTB7olc="}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.7.0_1683629433331_0.3912022621192308"},"_hasShrinkwrap":false},"0.8.0":{"name":"@charleswong28/bearer-api-scraping","version":"0.8.0","description":"Scraping Library with auto-retry","main":"dist/index.js","scripts":{"test":"jest","tsc":"tsc"},"keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"author":{"name":"Charles Wong"},"license":"MIT","devDependencies":{"@types/jest":"^26.0.23","@types/node":"^20.1.0","jest":"26.6.3","ts-jest":"^26.5.5","tsc-watch":"^4.2.9","typescript":"^4.9.4"},"dependencies":{"axios":"^1.4.0"},"jest":{"transform":{"^.+\\.ts?$":"ts-jest"},"testRegex":"(/__tests__/.*|(\\.|/)(test|spec))\\.ts?$","moduleFileExtensions":["ts","js"],"moduleNameMapper":{"axios":"axios/dist/node/axios.cjs"}},"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","gitHead":"ba490551028514f2be327d1388519f9ca3ddefe8","_id":"@charleswong28/bearer-api-scraping@0.8.0","_nodeVersion":"16.13.0","_npmVersion":"8.3.0","dist":{"integrity":"sha512-XldPgybxe5zWOFWgxSdGEHMoQVSHXt+4QmDyGYWG33N+SMtuENiS6b6nNSatgmNDqYDV/VI0zSIEDKBEHp5f0Q==","shasum":"334272214c843b8fd89c9cfd38ab04986284bcc8","tarball":"https://registry.npmjs.org/@charleswong28/bearer-api-scraping/-/bearer-api-scraping-0.8.0.tgz","fileCount":10,"unpackedSize":38162,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCNujKHPXp8r1ncz1jXFDqi8GG1LzvBrZqX596HUGZY7gIhAN19VKTYqbfesql57vDdJskQ/z61qn6SGEC2Bdlu5UY7"}]},"_npmUser":{"name":"charleswong28","email":"chung.triniti@gmail.com"},"directories":{},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/bearer-api-scraping_0.8.0_1683631408545_0.5738916991875065"},"_hasShrinkwrap":false}},"time":{"created":"2023-05-07T18:49:49.057Z","0.1.0":"2023-05-07T18:49:49.312Z","modified":"2023-05-09T11:23:28.845Z","0.2.0":"2023-05-08T08:22:26.130Z","0.3.0":"2023-05-08T08:26:49.221Z","0.4.0":"2023-05-08T08:35:22.677Z","0.5.0":"2023-05-08T08:38:34.318Z","0.6.0":"2023-05-09T10:29:30.419Z","0.7.0":"2023-05-09T10:50:33.551Z","0.8.0":"2023-05-09T11:23:28.693Z"},"maintainers":[{"name":"charleswong28","email":"chung.triniti@gmail.com"}],"description":"Scraping Library with auto-retry","homepage":"https://github.com/charleswong28/bearer-api-scraping#readme","keywords":["scraping","crawl","curl","nodejs","typescript","javascript","scraper"],"repository":{"type":"git","url":"git+https://github.com/charleswong28/bearer-api-scraping.git"},"author":{"name":"Charles Wong"},"bugs":{"url":"https://github.com/charleswong28/bearer-api-scraping/issues"},"license":"MIT","readme":"# Bearer API Scraping tool\n\nSimple scraping library with auto-retry of API with structure of `Bearer ${token}` in header.\n\n## Logic flow\n\n1. Fetch the access token api and store it\n2. Fetch the API with access token in header\n3. If the response status code is not 2xx, retry with `numberOfRetryBeforeRefetchAccessToken` times.\n4. If the request failed with more than `numberOfRetryBeforeRefetchAccessToken` times, renew access token and try again with `numberOfAccessTokenRetry`.\n5. If access token api failed `numberOfAccessTokenRetry` times or request failed with `numberOfAccessTokenRetry + accessToken == null ? 0 : 1` x `numberOfRetryBeforeRefetchAccessToken` times, throw an error.\n\n## Usage\n```\nimport Scraper from 'api-scaping';\n\nfor (let i = 0; i < 1000; i++) {\n  const result = await Scraper.get(`some_uri/${i}/item`, {\n    numberOfRetryBeforeRefetchAccessToken: 5,\n    refetchAccessTokenUri,\n    getAccessToken: (response) => response.accessToken,\n    numberOfAccessTokenRetry: 5,\n  });\n\n  console.log('result', result);\n}\n\n```\n\n### Method\n| Name | parameters |\n| --- | ----------- |\n| get | uri - uri to fetch <br> opts - fetch options |\n| post | uri - uri to fetch <br> opts - fetch options |\n\n## Structure\n- Code located in [`index.ts`](./src/index.ts)\n- Tests located in [`index.test.ts`](./src/index.test.ts)\n\n## Limitation\n- Proxy can be implemented by passing requestConfig to Axios.\n- Access Token API is only supported as get request. PR is welcomed.","readmeFilename":"README.md"}