{"_id":"@opentr/cuttlecat","_rev":"85-aacdedea6bf8b3ad81cbc24e5df4b6e3","name":"@opentr/cuttlecat","dist-tags":{"latest":"0.0.10","next":"0.0.11-dev-snapshot.db42b3f.20240124224534"},"versions":{"0.0.1":{"name":"@opentr/cuttlecat","version":"0.0.1","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://github.com/OpenTRFoundation/cuttlecat","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"echo \"Error: no test specified\" && exit 1","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","envalid":"^8.0.0","mocha":"^10.2.0","nodemon":"^3.0.1","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1"},"_id":"@opentr/cuttlecat@0.0.1","gitHead":"b20cb554d348defec59c0b70f2d2c655cc5a9cc4","bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-vdtmIZOA3AHOZjx7p6vJzcqDgb3HaYjkvrbnkeNF2f77RraOxQjCtpE93tJEwGFwqaWB+XImjYEtygcnt9eSjQ==","shasum":"78e46b50a51c1c5ab72a42c17729547ab8ea601c","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.1.tgz","fileCount":13,"unpackedSize":246542,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDWSJ2YVaim8DT2j+9t+rgpzE+kznidiPfO94vwasBfJQIhAMKtlplup8UptW5xCLArAUf917O0sMpl4qLogsq5smpw"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.1_1697408852162_0.9858564037349111"},"_hasShrinkwrap":false},"0.0.3-snapshot.8cbfc23.20231016.0":{"name":"@opentr/cuttlecat","version":"0.0.3-snapshot.8cbfc23.20231016.0","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.3-snapshot.8cbfc23.20231016.0","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"8cbfc23e9d514313ffc55d11d75ca49fac4db838","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-mkbcUE7MT6J+hqEmg7h+WYmigg5o8XP2NivT6PPd0eYOqoG2hWnKv3NNrrhBqvySy2VuFI/sE3TcNb7wArIbyQ==","shasum":"9096e76d8fa173cf4cc8acc7ccd806c213c1cd56","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.3-snapshot.8cbfc23.20231016.0.tgz","fileCount":13,"unpackedSize":246968,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIE4wXCGeU9pHGOieWY1iW2eLfRc2zQNhU8YyLAVMnekRAiEAg8D93VqaJSrE/PMIH+tGqVYrYEko49YDcZBAKFqaxbw="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.3-snapshot.8cbfc23.20231016.0_1697445277795_0.6562371075587494"},"_hasShrinkwrap":false},"0.0.2-snapshot.deeab89.20231016085517":{"name":"@opentr/cuttlecat","version":"0.0.2-snapshot.deeab89.20231016085517","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.2-snapshot.deeab89.20231016085517","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"deeab893e4205cc154513acb0f69b6c792cbf28f","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-yBOUt2SDM6gnFPtNDbS/fdeP7VtK6srEc15z9AHOhz/4oLw0fQ/70HujUOe9P0rezNkrrJ0gXjNB7WJtBjQXxA==","shasum":"62b7cdf2ed2beb696db17293318ab8df5b1f643e","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.2-snapshot.deeab89.20231016085517.tgz","fileCount":13,"unpackedSize":246972,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD9p8y4R7o3uNfMGZMofER35U2VY+6IdVQBCJSmeCEKIAIgSyY5U0iZ5VsOeoyYFTUhTMAqBDQH1fK6H8+PVbpSDsM="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.2-snapshot.deeab89.20231016085517_1697446576078_0.0762220734016208"},"_hasShrinkwrap":false},"0.0.2":{"name":"@opentr/cuttlecat","version":"0.0.2","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.2","gitHead":"deeab893e4205cc154513acb0f69b6c792cbf28f","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-Hur8pQx95ydYhYfygYFv/gRdj4GgCGlorU1uDSM18t2GirekOPRMgpwcgp2joCsYv2f3nm1YGRh4R/h9+hF0Yg==","shasum":"bc4c3c8f338fef0a4f8a5a2548c9c5cee60b04a9","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.2.tgz","fileCount":13,"unpackedSize":246940,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBrOigDlNsZAe0bYlhsociUyyLzizoKYH4d9Sz913BwrAiBru3y26TErHllQj8rV/TzooRgyZOw0xaNJxxoSIHwiTA=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.2_1697446802757_0.9669657044251592"},"_hasShrinkwrap":false},"0.0.2-snapshot.3197c7b.20231016201323":{"name":"@opentr/cuttlecat","version":"0.0.2-snapshot.3197c7b.20231016201323","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.2-snapshot.3197c7b.20231016201323","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"3197c7bbb9b907acb79eae6cbffcee3afa72fa0e","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-4AO9RnsldTqHdb2HKm9cYCh2GMtC9CRNZuX8VO7Je+Gwy4PCObXdKyK2B1YAbpdX323rS0F8Ppo+/zOtxuoJGA==","shasum":"ff61cda5b2bc35df87aa8fdfdae799a20c812b5a","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.2-snapshot.3197c7b.20231016201323.tgz","fileCount":13,"unpackedSize":247002,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHN22YQW0DBhOU/Ov2u8h4eS/qH6s73ojbZq+NHZc2TVAiEAoeBALQcGc5RofyHmAsx4xc6pECY7UmZI6LtY3bLW17w="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.2-snapshot.3197c7b.20231016201323_1697487264577_0.051125700994950174"},"_hasShrinkwrap":false},"0.0.2-snapshot.20889d0.20231016201809":{"name":"@opentr/cuttlecat","version":"0.0.2-snapshot.20889d0.20231016201809","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.2-snapshot.20889d0.20231016201809","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"20889d022b20919852972f36b4d9bc6bb6c195bb","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-JZJrabITbkDp4LyDEhFH+BFMgmFM+o6nYp8b9QdHUs8APLTOpZ1TuPyrwpDG0FflVgIXhIDQA5LWXQa/jULJVQ==","shasum":"c9b0ca2685d3c0f6248904a03c3b272c074aed12","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.2-snapshot.20889d0.20231016201809.tgz","fileCount":13,"unpackedSize":247028,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIFNjRFbfu8Lo+d/rkRRhl6XowfjwbtFwB80vGpnbVfSOAiEAwbsLCxlkwwvqXtMBwyzuykU4ZsjW1LvFPjsmmBB4lWg="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.2-snapshot.20889d0.20231016201809_1697487536010_0.8302328473621976"},"_hasShrinkwrap":false},"0.0.3-snapshot.d815331.20231016211437":{"name":"@opentr/cuttlecat","version":"0.0.3-snapshot.d815331.20231016211437","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.3-snapshot.d815331.20231016211437","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"d81533146e5c1fdfe3a5987c15764e196e65d04b","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-OEkILvAqe+BVFSezCazyGGgVXBv2sJepY1sA1LzINs+e/Sct+k7fXTqG9n5KEN6ki/PSu5qlvM2nrzI3WCa6wA==","shasum":"9db4f090e729306d3521992b0fedda1a7e33019d","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.3-snapshot.d815331.20231016211437.tgz","fileCount":13,"unpackedSize":247028,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQC8SQB2kabZMY08lOmi0emxm1/3KTjTaUFRJsrjUsiSKwIgD6Bvf49wktS55xPfU2TbqMwsXzrDpqZvYFx31vJfip8="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.3-snapshot.d815331.20231016211437_1697490924960_0.5060054924629873"},"_hasShrinkwrap":false},"0.0.3":{"name":"@opentr/cuttlecat","version":"0.0.3","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.3","gitHead":"d81533146e5c1fdfe3a5987c15764e196e65d04b","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-4E8rTOG28oPjsiCJsUJpIGcMWK9PVtsDloG1AmncB8GVLLVrBLs64ZT6i1d+6X9zVgIkBZHqcdIaxZNjCvyT5Q==","shasum":"95c180eb45c75f24193cb5a4479e45338050efac","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.3.tgz","fileCount":13,"unpackedSize":246996,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCID3y4lNAVFhYulNyXzAZzSridme/2rzwTAwWNWCiG4SqAiBwORklK7Y5lB1ELHNTEP9y06LQuBT1auLrXkVzrxp8mw=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.3_1697490930653_0.6959801618805062"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.a94c76f.20231016212717":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.a94c76f.20231016212717","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.a94c76f.20231016212717","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"a94c76f6a568500eb51fbedc649e6f58ac223fe4","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-siWAVAxZ2+Bl80uKHSBLRfJfWGUpYrQsYe0UVCrkGhtFO9nl1Iq7uEL6x0g5cGBqHMP2M3cqVQ+hA5O25Q9ICg==","shasum":"2887253b9a13049f200e83692b7d1fea35e2d317","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.a94c76f.20231016212717.tgz","fileCount":13,"unpackedSize":247032,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD6AvMXwK0fyXFbaqxK1FYgpltH2eZWNDs3VIYScDHArAIhAIjcRVrd+TLps4mclGBhesBWxWe/KMMU5BGiHHK4OLfN"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.a94c76f.20231016212717_1697491690879_0.11284713111893252"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.997eb38.20231017150700":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.997eb38.20231017150700","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.997eb38.20231017150700","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"997eb387a514639acc0ff14b0209e5cc9dcb53cf","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-aez7W3aDV2s27DdsXVyqJZECksH2EUmg4qpYdK7Pfyy588iWpHez6HCaK35DglqlJR3BdkgIMBa+2lTeOXH/Gw==","shasum":"333de68be69666c85ffc887ea250cfbe782c2810","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.997eb38.20231017150700.tgz","fileCount":13,"unpackedSize":247887,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDQi7z6OTL+Uf8+Rb2fwNvuKzvcJNU2dwWTCEcDHvG85AIhAPx9aEpxDnbSugfRty9MSJDTgM1I2RjEQCMPH0YTI8Fh"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.997eb38.20231017150700_1697555264000_0.3500922074254078"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.dc46962.20231017154942":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.dc46962.20231017154942","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.dc46962.20231017154942","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"dc469629878f5f1ec815bd15b68713056626109a","_nodeVersion":"18.18.0","_npmVersion":"9.8.1","dist":{"integrity":"sha512-tgHU16P5/S3R2V9DjnkHuZvTkG5qdSc8Ek7K/fPARD3Sh3zbOAUWW0tM0n96CxWJyos0Te4vwCqlJ91F67zlcg==","shasum":"9c86e11f9b8d6cd4c873a7fba49900292e1c85b2","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.dc46962.20231017154942.tgz","fileCount":13,"unpackedSize":252216,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDoEAWwhfjz8hKsTzWnXeUaiISV385bslGDuWRyDyqmugIhAPbedDhmNd9Ryi4Lc6LG9Vj95W++IoSgu/XZVx+iKAHN"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.dc46962.20231017154942_1697557845307_0.5610613099858031"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.774bf0b.20231018193446":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.774bf0b.20231018193446","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","p-queue":"^7.4.1","uuid":"^9.0.1","graphql-tag":"^2.12.6","lodash":"^4.17.21"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.774bf0b.20231018193446","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults:\n\n```shell\n# store the results in a temporary directory\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"774bf0b4b5ad353699db36fa97bcdb148eb6f03f","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-fddx+7XuCfH6qNNS2p2BsWKOWXqLVMeweDGZZ6b/Mz2YSzKVCbEIJWXA+OaPsy2HBfPf7qOaKmFi3W3OogBU9A==","shasum":"464b14c32d8e296357b7a749b54ab66ce97a90a6","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.774bf0b.20231018193446.tgz","fileCount":15,"unpackedSize":250073,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDhT4SeqipiRJrzjXhjyGMnvwm5hz5oClFzf7DwfPA06AiEApexp8Q1qa3Pi819NihtEhKNJlYRDnP+MnnAKWv3Ua1w="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.774bf0b.20231018193446_1697657745396_0.5200620705662236"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.47a82d2.20231018193701":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.47a82d2.20231018193701","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.47a82d2.20231018193701","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"47a82d2e475dd65e20ef09f7a10212d7a51e7027","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-tD/u7rkvtvXxkRnhamOOGxLfr/znInVYYZjJQP1WedYxK/EXCiB14930NwOMMG2KPMLyBCwqDjnAzuJchJH63g==","shasum":"4041cf31e805dd1d74f10f4d33b6aa522b44c0c8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.47a82d2.20231018193701.tgz","fileCount":15,"unpackedSize":252353,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDXVokCoTZYD0nr5t4gm54t3Y2pzFQ6JnsPYJJPatD+rAiEA+MSqpzMYgC989YJR+IrXYXRGuPbV+glQpxNqru2uq/Y="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.47a82d2.20231018193701_1697657863192_0.22025044626757806"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.c0e0969.20231018193825":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.c0e0969.20231018193825","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.c0e0969.20231018193825","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"c0e09690b0ceb34801e87899222d3a068d85a154","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-sRRffND8zXRliiEEHU22NQUoQCx9HvMz/cbO3KigNfKp+LTdmAsrdCLfsGHhYEPnBq9JvahE16TW7/ak/YTQiQ==","shasum":"884751af116cc7dc61105c8ad01502d0106f39c2","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.c0e0969.20231018193825.tgz","fileCount":15,"unpackedSize":252554,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBT/wGJNXmtecq7IIiZZvEHTAqeXjwoIWK61CPOY1dK2AiAZseoTVSaoR5smw3pAbPktDKXBgiHrhee6j+Hl0prSVw=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.c0e0969.20231018193825_1697657955334_0.9920641480039007"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.b8ffb37.20231019165513":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.b8ffb37.20231019165513","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.b8ffb37.20231019165513","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n","readmeFilename":"README.md","gitHead":"b8ffb37c5929e4f474c6da44fe26133b69d6f3e1","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-njgE5A8cJiKKWjRXxr4+rt7S8dlqo/XR1qTA96b6f+/Blrx7Zo/6agxFayvSMvN+Pc4kWeql9sN5w1cNBji7hA==","shasum":"606594aa7e2c39a95937ef7a544f2f90f6ed2b1b","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.b8ffb37.20231019165513.tgz","fileCount":15,"unpackedSize":252658,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCID8xNZ9VFowK7Zjs190MtT6vKi0hbU+rC5YLV/NW0LkBAiBH5KaGG12UPKVlHi/QoI6YuOLyFzn/HGFtdNEK6wyIWA=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.b8ffb37.20231019165513_1697734559710_0.0540247299462"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.41dd1dc.20231019165638":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.41dd1dc.20231019165638","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.41dd1dc.20231019165638","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"41dd1dcf60d258b47fbc79277e579e070098b941","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-8zqgyvgzgiz3lacdE+VrZlsh3MFAx5IKm4TjM53OEcYbPifb52Gur0H/WeYlj7XYs+uLSzQI1ZLBtIVJ/P6c5Q==","shasum":"4fcbb00165c4d2daa54cb9bea7fa216fe6867bad","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.41dd1dc.20231019165638.tgz","fileCount":15,"unpackedSize":252880,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHOM0O0yc42tJQ/iJDfODuka90NyugMEZNl7G8PJ7rbiAiEA+HbEUxExMKDa0PctRLuDiPN3TF0ephPopFHv4ECdNg8="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.41dd1dc.20231019165638_1697734637590_0.4030795319511933"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.7140b6f.20231019170040":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.7140b6f.20231019170040","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.7140b6f.20231019170040","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"7140b6fd65c8d464982b90743c532df8332e04af","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-u19EferFuITPJOBx13SKyvcfnezgBkavs4f/d9630M+EguvQEjrJrEWLEQYkSVNMxeMIlUhmwWIL0vHfHKCsqw==","shasum":"e5307a9d522834399fea807476fcbee5d1854689","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.7140b6f.20231019170040.tgz","fileCount":17,"unpackedSize":281845,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDRNDgh6hqWomDo4P4Jr1GpBKzHKUgyGZ34VZFYzyKQ4wIhAJQAqVRI4S26PEYGXVrJjdUfbndJ650oGcNSiK1dGkNB"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.7140b6f.20231019170040_1697734881963_0.8356186559169618"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.667fc8c.20231019170119":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.667fc8c.20231019170119","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.667fc8c.20231019170119","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_SEARCH` - search for projects that match the criteria\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"667fc8cea6702190be1b6b71017eba214c81817c","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-VfB+kwAWuYmgt5I65DntLVP97GqIEbd+6rPh2DvU0ksNu/uWS3f5/Pj2LTC1syrk7NflyiaVQ56cmke6iOrYyA==","shasum":"16c308368dc34eb5589f73df1234371b29e0b7fb","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.667fc8c.20231019170119.tgz","fileCount":18,"unpackedSize":290865,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCldfXi/4GPQuIpZSJbo7f1hOfydNpXd1c1HCWFH7oElwIhAKirSNxnMJVLEk8kNdRprIQFTQ/RyO3kjFeVltbV2NyM"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.667fc8c.20231019170119_1697734961451_0.18122943589492935"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.79d3a63.20231019170437":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.79d3a63.20231019170437","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npx tsc && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.79d3a63.20231019170437","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"79d3a633ecfd0c3e5c7dae358268b9c3e833248e","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-v7OeZXoJEiTVIEtOfI+DhXFAZav1iSxGwd4vwOTUNwLxM/FkdPnlWp963FC2mZHmK6KHVkqAwRSE2GguyidBOQ==","shasum":"0954f740b254a423c6b8d16d4cf5920e4af2626b","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.79d3a63.20231019170437.tgz","fileCount":18,"unpackedSize":290584,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCID4Fgdwz6+bnwg3hHbwTMxcve9KCx7S0OixTBkTmvmQdAiEAha15p0v3KocYUADNAOMOaI3ENx+mzal0aZuN2Qd1PhE="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.79d3a63.20231019170437_1697735155819_0.024536318916243527"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.94589a6.20231019171914":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.94589a6.20231019171914","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.94589a6.20231019171914","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nThe output will be written to the `./data/focus-project-search` directory.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"94589a6650205778d3791fcbf182688d9723859f","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-njhEy2CVpeBsCgJKrgYl/xOZbWU0XRr3ud6P4bfWQMYpU2uNTVZPo9PIeRcz8s8c84djYN7jWG3Da/ZBMwx+8Q==","shasum":"d0d9af7cc27df282bca0cec7522ba9828ce003e9","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.94589a6.20231019171914.tgz","fileCount":18,"unpackedSize":290699,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHnnE8U5t8avb8sYE/hBYLx53udMq5bvVTDi4cp0z8m8AiAEmfoUoZ6gq6CjZNQDZG8IWA85HTFxKgf23wm/1UTeiQ=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.94589a6.20231019171914_1697736044574_0.03442280240362394"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.91a1a23.20231019172909":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.91a1a23.20231019172909","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.91a1a23.20231019172909","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Enabling debug logging.\n\nSet `ENABLE_DEBUG_LOGGING=true` to enable debug logging.\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"91a1a23619765677d625790f9806633b27a8eae2","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-XaIiTspoApMv8I25w5wJjfSY7syylnbbV97l0CGEZxfiWUE/Wan2EaYNbHEagSs4CZq/FfNcnfrigmIdjcavOQ==","shasum":"846e77a3dcac5ce7f2c0634e92444016ebd4e113","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.91a1a23.20231019172909.tgz","fileCount":19,"unpackedSize":294549,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDJb7wIwqPL8GZ+moaWbxrHeCfq7DEd0nbaKxwoOKjedwIhAJ/LMlhXy6Bdil5hF62vHzvpJLtN7KoOhKhL9Bayf/xi"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.91a1a23.20231019172909_1697736645903_0.4142984653523183"},"_hasShrinkwrap":false},"0.0.4-dev-snapshot.a9fbb71.20231019180105":{"name":"@opentr/cuttlecat","version":"0.0.4-dev-snapshot.a9fbb71.20231019180105","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4-dev-snapshot.a9fbb71.20231019180105","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Enabling debug logging.\n\nSet `ENABLE_DEBUG_LOGGING=true` to enable debug logging.\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n -s ACTIONS_RUNNER_DEBUG=\"true\" \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"a9fbb718e1bbf6f36aa3753b28f26f55ada21823","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-7YwPNxyrwCEklveqhDH2/yN9cO/vpysMfnDT26iKyOxkLUDCK2VhiN5XoDmLOGxAqVYC1v++BFJp2NKkRtgHnQ==","shasum":"c47150fb05c7943f0b2b2c725401e90be0434556","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4-dev-snapshot.a9fbb71.20231019180105.tgz","fileCount":19,"unpackedSize":294584,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCiaeQx9ZJNAIdv0G67nl1o350e3AHFg6S7VogaNGmRXgIgc4RBHKvSZVat97jfDRQPhkktCdUdiBAyU6CAlmi0SuM="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4-dev-snapshot.a9fbb71.20231019180105_1697738557985_0.34989286908693207"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.ba02839.20231019180528":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.ba02839.20231019180528","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.ba02839.20231019180528","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Enabling debug logging.\n\nSet `ENABLE_DEBUG_LOGGING=true` to enable debug logging.\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nENABLE_DEBUG_LOGGING=true \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n -s ACTIONS_RUNNER_DEBUG=\"true\" \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"ba028396549318e5b819288ca893dab46a38ad86","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-DTSifMhyU6Z+E7l9D9gmqLNfnKLZP14wisJeOF/ndCziEOpvLDanI+6qedOIDSDh4qxf6nRIvklp2jVijKfPSA==","shasum":"2718394fb18d963c782be4fbbd76cecd4ac85877","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.ba02839.20231019180528.tgz","fileCount":19,"unpackedSize":294584,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC0/jGPU6Gre6oSuczn9Nv9EcdSbQc6Veth7CYlPCPFBQIhAIEy6++fEhvLVKemCW6CSzFHC8569SBstHl1oBZwWhUY"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.ba02839.20231019180528_1697738827895_0.4792517903836291"},"_hasShrinkwrap":false},"0.0.4":{"name":"@opentr/cuttlecat","version":"0.0.4","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.4","gitHead":"816fb4569273805a1f3b29ad8b96d04c365ecf3c","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-uLzJUtWcKjLbbpLg9YoMjX1icJ7JTyzdl0eB9cOTMeJZICMhFSWBIFTxVtgxg3DW9Y2LdXbsqrVj1Iq2uPCYvg==","shasum":"5a0e12d7653c8178de7901ab317061a09e231dd0","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.4.tgz","fileCount":19,"unpackedSize":294548,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCRBhCara965vqIfX+y2TN7ksumB5C52YIXB7ZY9MP4mAIgUmQlFJMdB+Ds6SGGBWvtYtLhMuFMQ+do4qVCf07v9DU="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.4_1697738842385_0.0760245887498967"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.090e653.20231019220422":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.090e653.20231019220422","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","envalid":"^8.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.090e653.20231019220422","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Usage\n\n```shell\nPROCESS= \\\nOTHER_PARAMETERS=... \\\nnpm run start\n```\n\n`PROCESS` can be one of the following:\n\n- `FOCUS_PROJECT_CANDIDATE_SEARCH` - search for projects that can be used to identify the organizations that have focus projects\n- `FOCUS_PROJECT_CANDIDATE_SEARCH_LATEST_FILE_COMPLETE` - if the latest file for `FOCUS_PROJECT_CANDIDATE_SEARCH` is complete, print \"true\" in the console, otherwise print \"false\"\n\n### Recording HTTP calls\n\nSet `RECORD_HTTP_CALLS=true` to enable recording of HTTP calls using nock. \"Nock back\" will be used in `record` mode\nwhere the new records will be created.\n\nThe calls will be stored in the `./nock-records/${PROCESS}_${timestamp}` directory.\n\n### Enabling debug logging.\n\nSet `LOG_LEVEL=\"debug\"` to enable debug logging. Default log level is \"info\".\n\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------------------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to read and store the output. | N/A | No |\n| `RENEW_PERIOD_IN_DAYS` | if previous queue is completed, create the next one after RENEW_PERIOD_IN_DAYS days | 7 | No |\n| `CONCURRENCY` | number of concurrent tasks | 6 | No |\n| `PER_TASK_TIMEOUT_IN_MS` | timeout for each task | 30000 | No |\n| `RATE_LIMIT_STOP_PERCENT` | if rate limit remaining is less than RATE_LIMIT_STOP_PERCENT * rate limit (typically 1000) / 100, stop the queue. | 10 | No |\n| `INTERVAL_CAP` | max number of tasks to execute in one interval | 4 | No |\n| `INTERVAL_IN_MS` | interval for the cap in milliseconds | 20000 | No |\n| `RETRY_COUNT` | number of retries for each task before giving up | 3 | No |\n| `REPORT_PERIOD_IN_MS` | period to print the queue state (0 for disabled) | 5000 | No |\n| | | | |\n| `MIN_STARS` | minimum number of stars | 50 | Yes |\n| `MIN_FORKS` | minimum number of forks | 50 | Yes |\n| `MIN_SIZE_IN_KB` | minimum size in KB | 1000 | Yes |\n| `MAX_INACTIVITY_DAYS` | maximum number of days since last commit; ignore projects that have been inactive for longer than this | 90 | Yes |\n| `EXCLUDE_PROJECTS_CREATED_BEFORE` | exclude projects created before this date (format: YYYY-MM-DD) | 2008-01-01 | Yes |\n| `MIN_AGE_IN_DAYS` | minimum number of days since the project was created; ignore projects younger than this | 365 | Yes |\n| `SEARCH_PERIOD_IN_DAYS` | Number of days to search for projects in one call | 5 | Yes |\n| `PAGE_SIZE` | Max number of projects to return in one call | 100 | Yes |\n\nThe options marked as `Yes` in the `Persisted` column are persisted in the process state file. This means, when the\nprocess is run again for the same process file again later, it will use the values from the file. This is to continue a\nsearch process. Those values will only be used if there's a new process file created.\n\nThe options marked as `No` in the `Persisted` column will always be used from the environment variables.\n\nTo start the process with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5700 \\\nLOG_LEVEL=\"debug\" \\\nnpm run start\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/tmp/foo/bar\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH\" \\\nMIN_AGE_IN_DAYS=5750 \\\nRECORD_HTTP_CALLS=true \\\nLOG_LEVEL=\"debug\" \\\nnpm run start\n```\n### Process `FOCUS_PROJECT_CANDIDATE_SEARCH_LATEST_FILE_COMPLETE`\n\nSupports the following environment variables:\n\n| Name | Description | Default value | Persisted |\n|-----------------------------------|-------------------------------------------------------|---------------|-----------|\n| `GITHUB_TOKEN` | GitHub API token. Token doesn't need any permissions. | N/A | No |\n| `DATA_DIRECTORY` | Data directory to check the latest file. | N/A | No |\n\nYou will want to use \"LOG_LEVEL=error\" to see the output and only the output.\n\n```shell\nGITHUB_TOKEN=\"$(gh auth token)\" \\\nDATA_DIRECTORY=\"/Users/aliok/go/src/github.com/opentrfoundation/state-of-oss-contribution/focus-project-candidate-search\" \\\nPROCESS=\"FOCUS_PROJECT_CANDIDATE_SEARCH_LATEST_FILE_COMPLETE\" \\\nLOG_LEVEL=\"error\" \\\nnpm run start\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n -s ACTIONS_RUNNER_DEBUG=\"true\" \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"090e65387e00720b11b0409032fd5e0e30eacc06","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-Jbkbt69vnJtiXqeigaq623Oadj/mvQqwRkAGz6YXVyN45Hjdo5tBPmjZNhcBgbe0nm41qzsYoj1OzMh5olMAKw==","shasum":"8250e76793ebab7586b8c3d49fc5a6b85665bb14","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.090e653.20231019220422.tgz","fileCount":19,"unpackedSize":296648,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCICTVoCjdhpZB9zzytU1kcYPSUxnlXtBVJAG5FIay/adWAiEA39Faf7r/cthGshrXM76jEJPDLOh+g1MB5dCRaUrcGRo="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.090e653.20231019220422_1697753145771_0.13584287071162704"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.e84d36d.20231021215056":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.e84d36d.20231021215056","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.e84d36d.20231021215056","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n## Running tests\n\n```shell\nnpm run test\n```\n\n## Testing GitHub Actions workflows locally\n\n### Test publish release\n\n```shell\n act --job=publish-release-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test publish snapshot\n\n```shell\n act --job=publish-snapshot-on-npm \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n -s NPM_TOKEN=\"FAKE TOKEN\" \\\n --reuse=true \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Test HTTP call recording\n\n```shell\n act --job=record-sample-http-calls-focus-project \\\n -s GITHUB_TOKEN=\"$(gh auth token)\" \\\n --reuse=true \\\n -s ACTIONS_RUNNER_DEBUG=\"true\" \\\n --use-gitignore=true \\\n --remote-name=origin\n```\n\n### Downloading HTTP call recording\n\nRun the workflow.\n\nThen download the output, such as:\n\n```shell\ngh run view 6565769484 --job=17835015583 --log > foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"e84d36d503c978b50a0a71368a16362828640845","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-xvrtJKdFnwtgGP+mR4n/4cIX5F4yvaAvtXRYkKJDJwWIt/7R350MpZKreHNmxKmv/so6TO2wtGgO3LgcZcBiyw==","shasum":"9ccfa46831acf3854226f775b72c425c5a6340d8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.e84d36d.20231021215056.tgz","fileCount":23,"unpackedSize":303285,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDg7ann7wYWwyrpMjmCyN2VM0whw8KLJtSJSVld+MGdfQIhANc5GifBtNQvM6ePO8naeIYW/RphWuBkoh4BQwTXGljG"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.e84d36d.20231021215056_1697925146752_0.30032565030014813"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.cccd644.20231021225559":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.cccd644.20231021225559","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.cccd644.20231021225559","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/tasks/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/tasks/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/tasks/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/tasks/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"cccd644560740dd61439f361b89f2d047ea1fd95","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-wHBlDJI2Sh5HEovDdkVmcJexDbGQbIBY4TLTKkVsJtMuqCaR+L5Xm88h1sFQ6fMPC6o5WJoa1b13RLJTTE61xg==","shasum":"2b950076462cfd8fb5448269c2a7244b8d939a8a","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.cccd644.20231021225559.tgz","fileCount":31,"unpackedSize":344126,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDPEN2X50m6SIOcSD/EnCZl4qH2+1YOfG6rmfxmW8OmZAiBZsh1T+X7lOPYHFGtKXjBpH4fP+2HdCrMZz1xdm6IiIg=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.cccd644.20231021225559_1697929031416_0.5580686814071596"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.c0c51c7.20231022203840":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.c0c51c7.20231022203840","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.c0c51c7.20231022203840","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"c0c51c7767b2cd42d9370be045c851d1d399e373","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-OFVJSdcPmfY0ZOfjp8zYbddRti8s51k4jPeDCTRnZ4VlnJI6pUo0BsT8VQkfjefLBv/MYeyd6GT/ICtVwhtDUQ==","shasum":"56f52b426cac2e2c7d064215b20920b4bbfa956c","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.c0c51c7.20231022203840.tgz","fileCount":36,"unpackedSize":335128,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIGZNTvVjo+DrlTcxs9U7V/bflbBJqpiAhmhyWkXdF0X2AiBhXTEYHVFVnDbTLHtA5maF3x4aFu0BR6pgK2BTOFhc+w=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.c0c51c7.20231022203840_1698007200598_0.1392406478873336"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.ec4ecab.20231022205323":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.ec4ecab.20231022205323","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.ec4ecab.20231022205323","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"ec4ecab1482b88daa51e049c69529dabe3e932a7","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-LLny1j7AH7tJZfag74wyJgklrQtvHdImC2CRCkMT4P61TjdT63doOL8jwsDnRDDyGUJNS8V54ABZ3iN1gRi86w==","shasum":"64bd928a8fc337f10dc408e3814f6661c43ea029","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.ec4ecab.20231022205323.tgz","fileCount":34,"unpackedSize":334974,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHEkbuihFbm5Rt7BUiScm4E9c5MgVbBXPk3ZuqTs85tXAiAknDC1NKzAxuWoIvV/V0XUXj0XVBMbbNefzM9vzGjBVg=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.ec4ecab.20231022205323_1698008103497_0.9470948764944911"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.bfd164a.20231022214929":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.bfd164a.20231022214929","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.bfd164a.20231022214929","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"bfd164a7b404cee9c49440c4ed35ce94b37b6d76","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-vKNDBqFMMWfdzdDS1rfh+vU9GusbxaTAws2BE0WUb8UBwbObUcJJUcVzDjF3v693h7KnPdxeGu5qX/o/xTU6zw==","shasum":"e437ff3aa7e10ee9c4a5e810f84adc6320875218","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.bfd164a.20231022214929.tgz","fileCount":34,"unpackedSize":334715,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBAg6zkE/T/ArwEtFIpd9vHKh37rb2a/VQ+q1Nnr5KBhAiAfTNLlWVguZsBMciJ0GskUDaY/zTZeXFZSi01zbgIyXg=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.bfd164a.20231022214929_1698011450676_0.6208013339275551"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.2860701.20231022215348":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.2860701.20231022215348","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.2860701.20231022215348","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"2860701b8dea1c98f7126f556bf1c23dab89d18f","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-dsPXxNNLBOeSzaS/LEAiAXyzncuLhMa+FgmEil5byLsTWkXDCAV9b6WaAgJe35U1uks+cZ+5ZFk9Eo5xpOxQHQ==","shasum":"2be69bed61fdbbae58dca09678d41fd47fe3bd32","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.2860701.20231022215348.tgz","fileCount":34,"unpackedSize":334413,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHiSAI0mLgQ/kIOnWGvOr/zszhYC5av9gB/UE9KZ5ujzAiAtlyTmqe6+Eef5IC879VTm4UH0lVcvBAtSypB/axJGDw=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.2860701.20231022215348_1698011717995_0.46128239882843713"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.541d5bb.20231022221405":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.541d5bb.20231022221405","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc","test":"npm run build && mocha --require ts-node/register ./src/**/*.test.ts","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.541d5bb.20231022221405","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"541d5bb5cfda4b2de3159cf4b258d7c1cdc6ed3c","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-szByG9beS2agiyWg354dZYBwP3gWfU8y1DM/K1ADuMm73yf5fDc42vR65bEziiYr2sp47ot5yYbdW+KnMetLpQ==","shasum":"1a395eac385fcdb499abfd15806de892cebe4aea","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.541d5bb.20231022221405.tgz","fileCount":34,"unpackedSize":334152,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDpJaVA1rTxgtTg3OnN1+ZDG7eutwOZYuDtNHjX95nbSQIgGXW0mcr47K8i4gSomA07qpJyj5P+/Q4NEpw3XWlo2lI="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.541d5bb.20231022221405_1698012925810_0.8593415132553934"},"_hasShrinkwrap":false},"0.0.5-dev-snapshot.9369c32.20231023185022":{"name":"@opentr/cuttlecat","version":"0.0.5-dev-snapshot.9369c32.20231023185022","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-dev-snapshot.9369c32.20231023185022","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"9369c32500466ef257b25aa23e18c2aa942e4904","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-E46w4PQ39litlHLmkU0sS/v/TzmNE+redRYpQo9S7Broi20ImjIIPH5RpXL9npfAE/Fq+ZnTghO1mh7tZeFyGQ==","shasum":"248b5e46163fbec5b0c44b3d212a2c2b0c2fbd03","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-dev-snapshot.9369c32.20231023185022.tgz","fileCount":60,"unpackedSize":428766,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIG7941vywkLf/7RusDwoZxW9f+1kzesNQxLt5p85m4F/AiEAg8CsEX6cdPHN6DGD5OOy7Rw96yVGixUANQFaK30UJTs="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-dev-snapshot.9369c32.20231023185022_1698087136286_0.8142713749186086"},"_hasShrinkwrap":false},"0.0.5-snapshot.a62d082.20231023185209":{"name":"@opentr/cuttlecat","version":"0.0.5-snapshot.a62d082.20231023185209","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5-snapshot.a62d082.20231023185209","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `focus-project-candidate-search-complete`\n\n```shell\nUsage: index.js focus-project-candidate-search-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest focus project candidate search is complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the focus project candidate search files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js \\\n --command=\"focus-project-candidate-search-complete\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"a62d08246e2b48745f29c730646dbe822edb0a31","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-zyhyxeQoghMHF+dmP6yYQiFUVta3FV0k4mXlhRLtAuIZubBDlu9g8gL2P8umYzJBJQwRljxFws/p0hoVjMFT6A==","shasum":"cfca3ccc18f2e0d535bc91c0561061b04c83cf20","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5-snapshot.a62d082.20231023185209.tgz","fileCount":60,"unpackedSize":428762,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCFOYOOE2l0Twv83FYDMEjU3IlK/JWfXm114FYE/BMRUQIhAI1zUHzUPa4WywSq01Kc5hMDtBvKt4IFhsjxRuc+B8Pp"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5-snapshot.a62d082.20231023185209_1698087229552_0.21048577424067672"},"_hasShrinkwrap":false},"0.0.5":{"name":"@opentr/cuttlecat","version":"0.0.5","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.5","gitHead":"a62d08246e2b48745f29c730646dbe822edb0a31","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-aazHNSjAw9dHgX9OIg65jGjcHEwxCYGpWP3UlOwJPmG9RU2VZ3D3cDgfHkgKxBu17FdH30jsMSGQqTDM3EP1Pw==","shasum":"7f1353925fe0a3dcd6e7d86f3a12e2da74888564","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.5.tgz","fileCount":60,"unpackedSize":428730,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIG3V3vEJ6R4ZcJXL5GZZAu01cBYenNSuOPWFVzfTOuDCAiEAnnXGsrZs8xX/BakB5AD/a2+ClupfgA3uzLHdYlV+Iz4="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.5_1698087270983_0.11901019484217579"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.7dda9b8.20231023191844":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.7dda9b8.20231023191844","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.7dda9b8.20231023191844","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js --command=focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js --command=generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js --command=user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"7dda9b8d8f8cbe87b69d297418e77366486d5275","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-fuzLR+GCvkER7ID3e1gCYyM3YeMfPaNP1YZ2Zu+J2UjxCpgrbiRNcODub+usvb/smJISfVqTAtaJ9efVSeL1+Q==","shasum":"0e97c2970ff8b211902d2c4d226daf46207bb662","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.7dda9b8.20231023191844.tgz","fileCount":61,"unpackedSize":427850,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCLrGn0qF4Q1KeR8EOEY6krsfx9zYzJerSoXDzcolva8AIgY8T5lpaaSXqdpMLQZJXPKpyXNP/6fiUW+XRI9/gJ1PE="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.7dda9b8.20231023191844_1698088844786_0.1272414088136007"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.d3d7d39.20231023211835":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.d3d7d39.20231023211835","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.d3d7d39.20231023211835","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js --command=focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js --command=generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js --command=user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"d3d7d39d4e673d86e434d1c0fd52818aa51ad2e3","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-/g0KqG3/6PQo2Zys3A0wz+FKioSEjrqFyOSixciwbQXd7d2qBTiD/wulZAbkybVYmqpmGVsavYiK3hlimUaFlA==","shasum":"c05cb467d842008d526b083fa49cfc945b0020bf","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.d3d7d39.20231023211835.tgz","fileCount":61,"unpackedSize":429973,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCt5HE1OuRu381mnMPlvctuGxCrjBk0D3I+62/aeo9g8gIhANjarVggnrYHlCjLwNElP/E6JJLPScwMf1+R+F4A/nrl"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.d3d7d39.20231023211835_1698096027771_0.49933772526671283"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.27534b7.20231023221314":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.27534b7.20231023221314","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"start":"npm run build && node dist/index.js","build":"rimraf ./dist && npm run codegen && tsc && npm run postbuild","postbuild":"cpy 'src/**/*.json' 'dist'","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","codegen":"graphql-codegen --config codegen.ts"},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.27534b7.20231023221314","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js --command= [options] [--help]\n\nOptions:\n --version Show version number [boolean]\n --command Command to run [string] [required] [choices: \"focus-project-candidate-search\", \"focus-project-candidate-search-complete\", \"generate-locations\", \"user-count-search\"]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js --command=focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js \\\n --command=\"focus-project-candidate-search\" \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js --command=generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js \\\n --command=\"generate-locations\" \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js --command=user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"27534b70a46bc424c63273780c8cf28e0bf65772","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-w9mDx+hkmBEQhK7GG+NVG1ToYUkKdNYRLvYw+Zi0I5qDyUQi8spG9BlG1+DGpIgC3znOMG0KD9njenc9oN9ZVg==","shasum":"19504af9daa7af4fd3f4e90d22afc4ed442f5de8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.27534b7.20231023221314.tgz","fileCount":61,"unpackedSize":432777,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHKc36dPXYxnlVn6jDv5+2v83GJB2L4/t3uzMd4e5TWtAiEAirpjs/mmT9GN3MCztq3jiCFN8R1SN5EnCepMcC7v+XU="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.27534b7.20231023221314_1698099280705_0.4066912040825945"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.c2d41db.20231025192157":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.c2d41db.20231025192157","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.c2d41db.20231025192157","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"c2d41dbfabb98b8d84c47c800dc73acefb4b25b3","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-0QX2D38cMaClHnev+Z4jy3eLKI2ZIp6a0BXlu5reQb4RJeUy1glQzGjUuC28ZaTeWZ8Py36XgZzxNWv9CuQLiA==","shasum":"f0e6fe2cd199d1de7b7c2483a5bc94fcef70c682","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.c2d41db.20231025192157.tgz","fileCount":68,"unpackedSize":430048,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDSmDfT6qzfQzMqNqv0/0DY55SgOack5zamVb0Ou06ylAiAelxBT6htXsIhjqEeOn1b4LORaHYU823hw4behwby8iQ=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.c2d41db.20231025192157_1698261842965_0.6855552889796035"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.270c625.20231025211804":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.270c625.20231025211804","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.270c625.20231025211804","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-orgs Extract GitHub organizations from the focus project candidate search results.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-orgs`\n\n```shell\nUsage: index.js extract-focus-orgs [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-orgs \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `focus-project-candidate-search`\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-count-search`\n\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"270c6251bc882ce5029108964d7c82f8e06b8d61","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-W7EbiEbGNmUvLauqC/EIDkV/5jFESTMHutcrcJBz4vre5Cg4+F6swR1I3U+n+HnqpeU8A45ooXKOeR2RUOj5TA==","shasum":"705fa68ed7ce5ac21f88c03e0f7eedfe9116654f","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.270c625.20231025211804.tgz","fileCount":75,"unpackedSize":442465,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIEovZDFkDNfp1dy9Lii67JqBU8NhoGJmvYRF1RE4KXQEAiEA6DzFIUTNjhFAv1T6CZRI7SGaHpyfSOazVXhjb5I6Mas="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.270c625.20231025211804_1698268808620_0.004887416879245876"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.d6f808c.20231028105414":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.d6f808c.20231028105414","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.d6f808c.20231028105414","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `'read:user'` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"d6f808c6a8a6e88fa320756f6bdaae6735be9f22","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-0gTiqU5gSq47h/FYwo51IzfANgPRDJgU9S3xWixqjupS2xPnmgypKgJLzlFQQFBvmDuCpaUZoRB5ro3+7lj82A==","shasum":"0a7f7f032f7dd239b2cdaf79006efb199025b2a9","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.d6f808c.20231028105414.tgz","fileCount":99,"unpackedSize":696161,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDvQZi6KS8c/Rswy/hgmTRpm8HpUm3bTgKTxlrAsO4w6gIgEu1w9VQP1fO53hG/FLRef8b3Tc6OGJFGtyshSX2dqJU="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.d6f808c.20231028105414_1698490556636_0.19291359494611982"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.b59de4b.20231028161715":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.b59de4b.20231028161715","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.b59de4b.20231028161715","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"b59de4b8926bc7693531d32ca4ac835c49a7d9ef","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-g8a5ugv8AAbqpMX6HGQR/f+SFISQ0onhtNfTLomcH/q6cArYNRvx1lLO+HS7q073dJrxKpJjZTX5Hk2kg5TCMA==","shasum":"b6a041d726627d257867e1d09fec3d2c019d31b1","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.b59de4b.20231028161715.tgz","fileCount":104,"unpackedSize":702806,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDI/YdWtE6plGTjdBcns/pTggyAfTHAKIuYFGhzBg9iaAiEAroK27lhVJfKdWQDHQN0ZmlbX307Igu0DAvxWFkrJgOg="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.b59de4b.20231028161715_1698509927198_0.3722737624876251"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.f8daaa2.20231028193624":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.f8daaa2.20231028193624","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.f8daaa2.20231028193624","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"f8daaa22338acf2488615f344236a8c24d27b0c0","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-g4suJj9g9i8yXINgv3r7f2/+bz46aoU7T//04aVteAjvph4S/dfCu+6WbbPvUIgirhFMeY7ciep4JZt1YWebTw==","shasum":"6e2fc37a0fd08b625e386448bf8e212d2b959454","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.f8daaa2.20231028193624.tgz","fileCount":104,"unpackedSize":702802,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIFG+JTM8FobbHEhDdRZLwetxCE8AkA9Mgu2Qw31/DT/xAiAST4+e5nQ9X9E0FeKSCXSHN902TFzdbwAA2T6pVnHTvw=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.f8daaa2.20231028193624_1698521893480_0.21192726322622857"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.edd45e8.20231028205944":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.edd45e8.20231028205944","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.edd45e8.20231028205944","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"edd45e851179a408e471456d78aba8db6c75039f","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-XnEad0/BntGpXsxUX75rKGGdyRusfMNuhAYj1S7uYk0Vf7VRsYwjPFOpWTfkKEuLVPtHosBhWJrK59RYQODdfg==","shasum":"85cdcab277dc12ef59b695f664f91183a2895970","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.edd45e8.20231028205944.tgz","fileCount":104,"unpackedSize":702801,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIFXsG3o96PMbqrPEBbxSKN2ZBPp/SJngSNgImhSaVUO+AiEAlZ4ZHjZz3qHHN4qGg7krjY0K2s2/wDJRiayDOE1l4Yg="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.edd45e8.20231028205944_1698526876313_0.022849576683054762"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.b12e77e.20231028214151":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.b12e77e.20231028214151","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.b12e77e.20231028214151","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"b12e77e6ad08295197d64744f1afc1ed31260800","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-7v9s1rPh1zxyGIb9T6ybMpPDhJmRcg3Bh3zm2exZlIgqHNOmFteJ+SKb1MI8GbFrSgEg5zWEsBJxOMWBGuP/7A==","shasum":"5db41d0fb00c4e7ce50354e9507884b2723aae3c","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.b12e77e.20231028214151.tgz","fileCount":104,"unpackedSize":702801,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICqaBZ4Vv8ZbT8wK30Hns23cb5tAcWnQgqEhSxxbC4QCAiAZ4JgTbjbiPlyGvUmc5z6665CY8+7i4n1Vvy76Os5ZPQ=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.b12e77e.20231028214151_1698529415982_0.034531682498367244"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.bb1b2be.20231028214304":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.bb1b2be.20231028214304","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.bb1b2be.20231028214304","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-file-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-organizations-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named 'focus-repositories-.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-file-complete`\n\n\n```shell\nUsage: index.js latest-file-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-file-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-file Path to the file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-file=/path.json Use the JSON file that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/user-count-search.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"bb1b2bee9b455495f2ac0d5c35c50601e63a2ced","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-LGaahOhLi/FcTdvL/LQIlSVHIzpiKkhWpWsHRdra/aJfO2rwGj9XQyzRuxM0OHt3rnxfT1+VIlJem0fs/YsOKg==","shasum":"25fe8ab308712f0c7195a06e880afcd87856f2d4","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.bb1b2be.20231028214304.tgz","fileCount":104,"unpackedSize":702865,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCLyHWTB9l+9R31pM5CBrSNOz/DLH96tJQJtCLh91isVAIgc52fklHGwwNhRITSp0p+MnBYfG4CuONsMacaOoXD7o4="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.bb1b2be.20231028214304_1698529487990_0.6138847153608415"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.06b31af.20231031172229":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.06b31af.20231031172229","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/seedrandom":"^3.0.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.06b31af.20231031172229","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-process-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-organizations.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data/output\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-repositories.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data/output\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-process-complete`\n\n\n```shell\nUsage: index.js latest-process-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-process-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-files-dir Path to the directory that contains files with user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-files-dir=/path Use the JSON files from the given the directory that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nrm -rf /tmp/foo/baz\nmkdir -p /tmp/foo/bar\nmkdir -p /tmp/foo/baz/0001\n\n# create a state file and an output file under that dir\ncat > /tmp/foo/baz/0001/state.json < /tmp/foo/baz/0001/output-0001.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"06b31af16778ce9d0cfa311fcc429b7ca871ec2a","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-DSTEzYxxsNkp/kEl2m1h1gUGKSzuMNGBnvDoejC8xshOJrL2v8wK62RxK/mfeGybPirR61c9sYK3qWmRcdEENQ==","shasum":"922d88511de8e69f10534ed3b9e1918c97db394e","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.06b31af.20231031172229.tgz","fileCount":105,"unpackedSize":726862,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIEa/hY79XLOPL1NIRXdg8fzZfNhLpwnKnhPRJLXqaJgxAiAE81DMOYf5WNI6pvS+KRuREzTxdmZvsSpGAQ0WfEPqXg=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.06b31af.20231031172229_1698773037059_0.920005223257099"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.74a86a1.20231102211924":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.74a86a1.20231102211924","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/seedrandom":"^3.0.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.74a86a1.20231102211924","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-process-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js requeue-tasks Manually requeue tasks for trying them again.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nYou need to specify a command.\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-organizations.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data/output\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-repositories.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data/output\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-process-complete`\n\n\n```shell\nUsage: index.js latest-process-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-process-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `requeue-tasks`\n\n```shell\nUsage: index.js requeue-tasks [options]\nRun index.js --help for help on common options.\nManually requeue tasks for trying them again.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample call:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-files-dir Path to the directory that contains files with user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-files-dir=/path Use the JSON files from the given the directory that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nrm -rf /tmp/foo/baz\nmkdir -p /tmp/foo/bar\nmkdir -p /tmp/foo/baz/0001\n\n# create a state file and an output file under that dir\ncat > /tmp/foo/baz/0001/state.json < /tmp/foo/baz/0001/output-0001.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"74a86a1ecc6506961298d87c42d8e938d0136d69","types":"./dist/index.d.ts","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-q2zQ0q5TfvSiCDCG71WJF6GWk++B6ykji63OHzB5wHJkH6qXcCaEfr5g2Kj6ceDaJuhbV6j/Wm7M84fPG/LNEQ==","shasum":"0c6ed871a6ad8157b31a031070a656bd55c8e4f4","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.74a86a1.20231102211924.tgz","fileCount":166,"unpackedSize":1069691,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDGrKT19o017zyGB4kb9Oq3570Vv9i9eVHILrUr631m6AiEApDGdh3JXk9DthNAKTlniiDVmqkly8lS1aXm77Ee4XiQ="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.74a86a1.20231102211924_1698960070586_0.6305847890070493"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.34410ae.20231103083416":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.34410ae.20231103083416","description":"CuttleCat collects information from GitHub about open source contributors and their contributions.","keywords":["github","graphql","open source","contributors","contributions"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"main":"dist/index.js","scripts":{"build":"npm run clean && npm run codegen && npm run compile && npm run postbuild && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha --require ts-node/register './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","codegen":"graphql-codegen --config codegen.ts","compile":"tsc","postbuild":"cpy 'src/**/*.json' 'dist'","lint":"eslint ."},"devDependencies":{"@graphql-codegen/cli":"5.0.0","@graphql-codegen/named-operations-object":"^2.3.1","@graphql-codegen/typescript":"4.0.1","@graphql-codegen/typescript-document-nodes":"4.0.1","@graphql-codegen/typescript-operations":"^4.0.1","@graphql-codegen/typescript-resolvers":"4.0.1","@octokit/graphql-schema":"^14.32.0","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.9","@types/chai-as-promised":"^7.1.7","@types/mocha":"^10.0.2","@types/mock-fs":"^4.13.3","@types/node":"^20.6.3","@types/node-fetch":"^2.6.7","@types/seedrandom":"^3.0.7","@types/uuid":"^9.0.4","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","cpy-cli":"^5.0.0","eslint":"^8.52.0","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.1","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2"},"dependencies":{"@octokit/graphql":"^7.0.1","@types/lodash":"^4.14.199","date-fns":"^2.30.0","date-fns-tz":"^2.0.0","eventemitter3":"^5.0.1","graphql":"^16.8.1","graphql-tag":"^2.12.6","lodash":"^4.17.21","nock":"^13.3.4","node-fetch":"^2.7.0","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.34410ae.20231103083416","readme":"# CuttleCat - Collect information from GitHub\n\nTODO: Project description, purpose, etc.\n\n## Building the project\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the project\n\n```shell\nnode dist/index.js\n```\n\n## Usage\n\n\n```shell\nUsage: index.js [options]\n\nCommands:\n index.js extract-focus-organizations Extract GitHub organizations from the focus project candidate search results.\n index.js extract-focus-repositories Extract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n index.js focus-project-candidate-search Search for repositories that can be used to identify focus organizations and projects.\n index.js generate-locations Generate a JSON file with location information that is to be used in various searches and processes.\n index.js latest-process-complete Checks if the latest process is marked as complete and prints the result in the stdout.\n index.js requeue-tasks Manually requeue tasks for trying them again.\n index.js user-and-contrib-search Search for users and their contributions.\n index.js user-count-search Search for user counts for given search criteria.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nYou need to specify a command.\n```\n\n## Command `extract-focus-organizations`\n\n\n```shell\nUsage: index.js extract-focus-organizations [options]\nRun index.js --help for help on common options.\nExtract GitHub organizations from the focus project candidate search results.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-organizations.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub organizations. For each output file in /path, a focus-organizations-.json file will be generated with the GitHub organizations that are found.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-organizations \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusOrgExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusOrgExtraction/test-data/output\"\n```\n\n## Command `extract-focus-repositories`\n\n\n```shell\nUsage: index.js extract-focus-repositories [options]\nRun index.js --help for help on common options.\nExtract GitHub repositories from the focus project candidate search results, that do not belong to a GitHub organization.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --focus-project-candidate-search-data-directory Path of the directory that holds focus project candidate search files. [string] [required]\n --exclude-list-file Path to the exclude list file. [string] [required]\n --output-directory Path to output directory. The output files will be named '/focus-repositories.json'. Timestamps will be taken from the focus candidate project search output file names. [string] [required]\n --min-stars Minimum number of stars for a repository to be considered. [number] [default: 500]\n --min-forks Minimum number of forks for a repository to be considered. [number] [default: 300]\n --min-mentionable-users Minimum number of mentionable users for a repository to be considered. [number] [default: 200]\n --min-pull-requests Minimum number of pull requests for a repository to be considered. [number] [default: 500]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --focus-project-candidate-search-data-directory=/path When focus-project-candidate-search is run and it produced output files to /path, this command will use those files to extract GitHub repositories that **do not belong to a GitHub organization** and match the criteria. For each output file in /path, a focus-repositories-.json file will be generated with the GitHub repositories.\n --exclude-list-file=/path/to/exclude-list.json Path to the exclude list file. This file should be a JSON array of repository full names (owner/repo). Even though some repositories may match the criteria, they are not really open source projects. This option allows you to exclude those repositories from the output.\n --min-stars=500 --min-forks=300 --min-mentionable-users=200 --min-pull-requests=500 Minimum number of stars, forks, mentionable users, and pull requests for a repository to be considered.\n```\n\nExample call with test data:\n```shell\nnode dist/index.js extract-focus-repositories \\\n --focus-project-candidate-search-data-directory=\"./src/commands/focusRepositoryExtraction/test-data/input\" \\\n --output-directory=\"./src/commands/focusRepositoryExtraction/test-data/output\" \\\n --exclude-list-file=\"./src/commands/focusRepositoryExtraction/test-data/exclude-list.json\" \\\n --min-stars=\"50\" \\\n --min-forks=\"50\" \\\n --min-mentionable-users=\"50\" \\\n --min-pull-requests=\"50\"\n```\n\n## Command `focus-project-candidate-search`\n\n\n```shell\nUsage: index.js focus-project-candidate-search [options]\nRun index.js --help for help on common options.\nSearch for repositories that can be used to identify focus organizations and projects.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --min-stars Minimum number of stars for a repositories to search for. [number] [default: 50]\n --min-forks Minimum number of forks for a repositories to search for. [number] [default: 50]\n --min-size-in-kb Minimum size of the repositories in KB to search for. [number] [default: 1000]\n --max-inactivity-days Maximum number of days since last commit; ignore repositories that have been inactive for longer than this [number] [default: 90]\n --exclude-repositories-created-before The earliest date of repository creation to search for the repositories (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-age-in-days Minimum number of days since the repository was created; ignore repositories younger than this [number] [default: 365]\n --search-period-in-days Length of the date range in days to search for repositories in one call [number] [default: 5]\n --page-size Maximum number of repositories to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --min-stars=50 --min-forks=50 --min-size-in-kb=1000 --max-inactivity-days=90 Find repositories that have at least 50 stars, 50 forks, 1000KB size, had an activity in the past 90 days.\n --exclude-repositories-created-before=2008-01-01 --min-age-in-days=365 Exclude repositories created before 2008-01-01 and that are created within last year.\n --search-period-in-days=5 --page-size=100 Search for repositories in 5 day-periods and ask for 100 repositories in one call (change these to avoid GitHub API timeouts).\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\"\n```\n\nTo start the process with recording:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js focus-project-candidate-search \\\n --github-token=\"$(gh auth token)\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --min-age-in-days=\"5700\" \\\n --log-level=\"debug\" \\\n --record-http-calls=\"true\"\n```\n\n### Command `generate-locations`\n\n\n```shell\nUsage: index.js generate-locations [options]\nRun index.js --help for help on common options.\nGenerate a JSON file with location information that is to be used in various searches and processes.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --locations-master-file Path to the master locations file. [string] [required]\n --locations-additional-file Path to the additional locations file. [string] [required]\n --locations-exclude-file Path to the file that contains locations to exclude. [string] [required]\n --output-file Path to the output file. [string] [required]\n```\n\nExample call with test data:\n```shell\nnode dist/index.js generate-locations \\\n --locations-master-file=\"./src/commands/locationGeneration/test-data/locations-master.json\" \\\n --locations-additional-file=\"./src/commands/locationGeneration/test-data/locations-additional.json\" \\\n --locations-exclude-file=\"./src/commands/locationGeneration/test-data/locations-exclude.json\" \\\n --output-file=\"./src/commands/locationGeneration/test-data/locations.json\"\n```\n\n### Command `latest-process-complete`\n\n\n```shell\nUsage: index.js latest-process-complete [options]\nRun index.js --help for help on common options.\nChecks if the latest process is marked as complete and prints the result in the stdout.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you run another process that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous process was done.\n```\n\nYou will want to use `--log-level=\"error\"` to see the output and only the output.\n\n```shell\nnode dist/index.js latest-process-complete \\\n --data-directory=\"/tmp/foo/bar\" \\\n --log-level=\"debug\"\n```\n\n### Command `requeue-tasks`\n\n```shell\nUsage: index.js requeue-tasks [options]\nRun index.js --help for help on common options.\nManually requeue tasks for trying them again.\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample call:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n### Command `user-and-contrib-search`\n\n\n```shell\nUsage: index.js user-and-contrib-search [options]\nRun index.js --help for help on common options.\nSearch for users and their contributions.\nGiven GitHub token must have 'user:email' and 'read:user' scopes.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --user-count-per-location-files-dir Path to the directory that contains files with user counts per location. Search will be done for the locations that has some users in it. Also, the number of users in a location will help with the batch size to avoid making search calls that take huge amount of time. [string] [required]\n --search-period-in-days-for-10000-users Length of the search date range per 10000 users to search for users for a location in one call. [number] [default: 5]\n --min-repositories Minimum number of repositories for users to search for. [number] [default: 1]\n --min-followers Minimum number of followers for users to search for. [number] [default: 0]\n --exclude-users-signed-up-before The earliest date of user signed up to search for the users (format: YYYY-MM-DD) [string] [default: \"2008-01-01\"]\n --min-user-age Minimum number of days since user was signed up; ignore users who signed up after this date. [number] [default: 0]\n --contrib-max-age Start date of the date range to search for user contributions. The task will search for contributions between this [now()-`contrib-max-age`, now()-`contrib-min-age`]. [number] [default: 365]\n --contrib-min-age End date of the date range to search for user contributions. [number] [default: 0]\n --contrib-search-period-parts The contrib search will be split into this many parts. Since GitHub API returns max 100 repositories for finding contributions between a date range, this option will help to split the search into smaller date ranges. [number] [default: 1]\n --page-size Maximum number of users to find in one call [number] [default: 100]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --user-count-per-location-files-dir=/path Use the JSON files from the given the directory that contains user counts per location. Search will be done for the locations that has some users in it. Also, the user count in a location will help with the batch size to avoid making search calls that take huge amount of time.\n --exclude-users-signed-up-before=2008-01-01 --min-user-age=365 Fetch users who signed up between 2008-01-01 and 365 days ago.\n --search-period-in-days-for-10000-users=5 For each location, use a max date range length of <5*10000/userCount> in one call. Assume these parameters: --exclude-users-signed-up-before=2010-01-01 --min-user-age=365. Also assume today is 2021-01-01. The user sign-up date range to search for will be [2010-01-01, 2020-01-01] and will have a length of 10 years (~3650 days). For example, if there are 1 user in location A, 100 users in location B, 1000 users in location C and 100000 users in location D, users in those locations will be fetched like this: A: Date range length = ceil(5*10000/1) = 50000, ceil(3650/50000) = 1 calls --- B: Date range length = ceil(5*10000/100) = 500, ceil(3650/500) = 8 calls --- C: Date range length = ceil(5*10000/1000) = 50, ceil(3650/50) = 73 calls --- D: Date range length = ceil(5*10000/100000) = 1, ceil(3650/1) = 3650 calls. --- Even when the user count information is not collected with the same criteria (min followers, min repositories, etc.), that count gives an idea of how many users will be in a location that information can be used to generalize.Please note that these are _initial_ calls. If there are more users than the page size, subsequent calls will be made. Similarly, contrib search will be split into smaller date ranges and this will also affect the number of calls as bothuser search and contribution search are done at the same time in one call.\n --page-size=100 Fetch max 100 users in one call. Combined with other options that might affect the batch size, things can get complicated. However, this option will help to avoid making search calls that take huge amount of time and also avoid GitHub API timeouts. Lowering the page size will not solve all problems! Even when one uses a page size of 1, GitHub API may time out when using large date ranges because it needsto search through a lot of data and only return 1 result.\n --contrib-max-age=365 --contrib-min-age=0 --contrib-search-period-parts=2 Assume -search-period-in-days-for-10000-users=365 and --exclude-users-signed-up-before=2010-01-01 --min-user-age=365 and today is 2012-01-01. Then the process will only have a single date range for user sign up date range: [2010-01-01, 2011-01-01]. If --contrib-search-period-parts was 1,the whole contribution search would be done in one call. However, since it is 2, the contribution search would be split into 2 parts and each part would be searched separately. So, these will be the queries: 1. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-01-01 and 2011-07-01. 2. Get users who signed up between 2010-01-01 and 2011-01-01 and find their contributions between 2011-07-01 and 2012-01-01. This split will help with 2 things: getting more results (GitHub API returns max 100 contributed repositories) and avoiding GitHub API timeouts. However, it is advised to search for contributions for a short range. For example, for user contributions in last 6 months.\n --min-repositories=50 --min-followers=50 Find users that have at least 50 followers and 50 repositories.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\nPlease note that you would need `'user:email' and `read:user` scopes for the GitHub token.\nFor local testing, you can use `gh auth login --scopes='user:email','read:user'` to create a token with those scopes.\nOr, to refresh the token, you can use `gh auth refresh --scopes='user:email','read:user'`.\n\nTo start the command with defaults but with a short search date range:\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nrm -rf /tmp/foo/baz\nmkdir -p /tmp/foo/bar\nmkdir -p /tmp/foo/baz/0001\n\n# create a state file and an output file under that dir\ncat > /tmp/foo/baz/0001/state.json < /tmp/foo/baz/0001/output-0001.json < /tmp/foo/bar/user-count-search.json <\n```shell\nUsage: index.js user-count-search [options]\nRun index.js --help for help on common options.\nSearch for user counts for given search criteria.\n\nFollowing options are persisted in the process state file. This means, when the process is run for the same process file again later, it will use the values from the file. This is to continue an existing search process. The values passed as arguments will be ignored in that case.\n --location-json-file Path of the location file. Contents of this file will be used to pass location information in the search query. [string] [required]\n --min-repositories Minimum number of repositories that the users should have. [number] [default: 0]\n --min-followers Minimum number of followers that the users should have [number] [default: 0]\n\nFollowing options are not persisted in process file. They will always be used from the environment variables.\n --github-token GitHub API token. Token doesn't need any permissions. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nGlobal options\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/${command}_${timestamp}` directory. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --location-json-file=/path/to/location.json For each location in the location.json file, make a call to the GitHub API to get the number of users in that location. Output of this process will be a JSON file with the number of users in each location. Output can be used to make optimized search queries to get users in those locations.\n --min-repositories=10 --min-followers=100 When collecting user counts per location, only consider users who have at least 10 repositories and 100 followers.\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n```shell\n# store the results in a temporary directory\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\n# create a location file under that dir\ncat > /tmp/foo/bar/location.json < foo.txt\n```\n\nThen manually copy paste some cases to the test fixtures.\n","readmeFilename":"README.md","gitHead":"34410aeb52b03bef83e38facddb43cd287d46d0e","types":"./dist/index.d.ts","_nodeVersion":"18.18.2","_npmVersion":"9.8.1","dist":{"integrity":"sha512-RNRGQtyHclLZuymHIT3x3t6fA7xv8boi3ra5uzYbmaT5IX5zrlPZQ8+zi3NGewVNOAI/tBUkHBEzvaHpXQH1cA==","shasum":"28af071838ee99d4b10251960ef5fd2ba40331bb","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.34410ae.20231103083416.tgz","fileCount":166,"unpackedSize":1069912,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIG+arB3lz1kmBMxsCEDcHhb9sCWEtFh+3kgZMUjRlVNtAiEA2yE1L69E3NfXISuHKeoYNOypp5ot+I/4YDcKEVSZcnc="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.34410ae.20231103083416_1699000528808_0.11481272918979202"},"_hasShrinkwrap":false},"0.0.1-dev-snapshot.58de206.20231226161434":{"name":"@opentr/cuttlecat","version":"0.0.1-dev-snapshot.58de206.20231226161434","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.1-dev-snapshot.58de206.20231226161434","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n","readmeFilename":"README.md","gitHead":"58de206f6c081adc5cb0464609d9b77ead084523","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-yjbBYRel0TNIjh+nqeZKtqoonkcGKREeVR7zgwDnKAXDtcuBKHxq8Hf6Ryc6+BsCpyUW35ib0u2yAzpX4zaOeQ==","shasum":"89f7dd8bebc42bd78d81991042184032b078fd66","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.1-dev-snapshot.58de206.20231226161434.tgz","fileCount":48,"unpackedSize":145046,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCHqg46wQSjEGvY/2jIyg3SHIIeHctagK7e1px2a792xwIhAKGHCmjnvKrv6ShfDx5AvFyczQ7CUbwGVNRVSpcKW8U+"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.1-dev-snapshot.58de206.20231226161434_1703607344173_0.5749458348356802"},"_hasShrinkwrap":false},"0.0.6-dev-snapshot.dc0c3e1.20231226171638":{"name":"@opentr/cuttlecat","version":"0.0.6-dev-snapshot.dc0c3e1.20231226171638","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-dev-snapshot.dc0c3e1.20231226171638","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n","readmeFilename":"README.md","gitHead":"dc0c3e1326f00f5d9bda71b6f75f30f99fb158c6","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-pbMhC11GtLSN38TQlP5smRdzVsCLapvHmxVVIqL3hBWN6BY3DfhFUv0fWWOS2LmiRWB13hbZmIlouP3Ye3jbTQ==","shasum":"3ac19bcc8e2b1c70672be490ea4d599d77a2eb50","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-dev-snapshot.dc0c3e1.20231226171638.tgz","fileCount":48,"unpackedSize":145046,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICF6eBiLdaFqJSUyBG4UvSqa+jsg2TfWsmw+JfBj2fz0AiAELcozQD5u/4fqcHIvCAEMxN6e8LZdZhtOsNAdeCgJ4A=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-dev-snapshot.dc0c3e1.20231226171638_1703611067706_0.46657801914832464"},"_hasShrinkwrap":false},"0.0.6-snapshot.fd78434.20231226172447":{"name":"@opentr/cuttlecat","version":"0.0.6-snapshot.fd78434.20231226172447","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6-snapshot.fd78434.20231226172447","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"v0.0.6\" -m \"Release 0.0.6\" \ngit push --follow-tags\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\n```\n","readmeFilename":"README.md","gitHead":"fd784340fc4b3027c9585f95d679ce694dc06ef7","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-wdQeaJhuRC6oyUT+MyaxIU5Mg/j16OZm46yBcvMEMnK8zzSWYX54AN5Jl5KXQthqtgojLxufQw5lYE4m7nB88Q==","shasum":"5df65499dc1867111320d240cbd48e1ea3311d13","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6-snapshot.fd78434.20231226172447.tgz","fileCount":48,"unpackedSize":145383,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICGsInqbd/NQgO+7wFLFLEb2rrW+b5Ror0siB+YOmp8/AiByaT88cTUPV2KCaplS3IfztBkR6waGsgbUgajkj/dLqQ=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6-snapshot.fd78434.20231226172447_1703611555666_0.9965982185281468"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.3d73036.20231226172527":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.3d73036.20231226172527","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.3d73036.20231226172527","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\n```\n","readmeFilename":"README.md","gitHead":"3d730360dd38dca0e4b78a803c144d998aaef48e","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-gsTGR6hXawE9/SP6g03PJMjYy4x6DKViBiA/V/44FY50VQxlV01wkBxRl1auWQc3YMKYk5r3msW8sMTCSUPvJw==","shasum":"25ed84f86ef63be4318e15beb067497c97d537cb","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.3d73036.20231226172527.tgz","fileCount":48,"unpackedSize":145385,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD9zbsNPJG5Dohb+QSHvWpBqkMQoPHM2Gw9Zk/VaMnK0gIgRzjI3XUNpVw80x4TadW3I2+JnlkW9/2gnbwb/ERdGgY="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.3d73036.20231226172527_1703611596111_0.34643339884267244"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.7c01c3f.20231226172540":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.7c01c3f.20231226172540","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.7c01c3f.20231226172540","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"7c01c3f84220627e98a9def607d822f088cc617e","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-jSuYSrWWi4BvL+4e/ATz/XGP2e8LwxcUmYg0q4JyzVYW+QOmnQD6yzg6TIqInbsFYo43EAY63L3mdYCvKeRoBw==","shasum":"49bbf58dd91af810dfaa8a5db38b1b35fae9db79","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.7c01c3f.20231226172540.tgz","fileCount":48,"unpackedSize":145394,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQC9hD1ZNe/864btb94ww8vEcxzcNtEMTe5Koi3z0XtCCgIgIvbzAHoXDYGC+3rj0YqFEc9XxKcNIKHify0prL0zxuE="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.7c01c3f.20231226172540_1703611609482_0.8575171766654424"},"_hasShrinkwrap":false},"0.0.6":{"name":"@opentr/cuttlecat","version":"0.0.6","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.6","gitHead":"fd784340fc4b3027c9585f95d679ce694dc06ef7","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-OzupMx24qBohtjYBb3LnJN73EhiK8GlndgDOZ5NfCNEZVwsb/cPva+YpONitfadkAr8RRj0E8j90O03lUGlBQg==","shasum":"29d56c328d641ea21c7d4d9c17eed813e2eddec2","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.6.tgz","fileCount":48,"unpackedSize":145351,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIFEak5tIYjs0UmDeA6anmNAw8m7lyVM8lR1Q/b3+N4ltAiAvSUvVZNADnwlLTCjtIuircErhEqIJwr9E4d2ltpRKqA=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.6_1703611887091_0.2262694171548303"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.bbe8930.20231226173134":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.bbe8930.20231226173134","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.bbe8930.20231226173134","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"bbe8930b4a79a4a7160a6a005ac424269c48591e","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-DbEhDvEtso6qjEJb6JPg5QOfbl3IOqV+qXthFpsp/JjGt8AuwCLQ6sOgMJd65K9FBtbnoOfYsG/FZqiMEfEkRQ==","shasum":"365cbc30699a7c1cdf24ad497bc3dc3182ae9cdd","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.bbe8930.20231226173134.tgz","fileCount":48,"unpackedSize":145446,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD+4AyiNBD07LTIwNGUfvaz1kIGO5/eV0gW1/4SZn3o6gIhAJ9VW87fdWQNIit94dbHZHdQq96TNpvw9E8dverjE0+f"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.bbe8930.20231226173134_1703611962700_0.5102004580004027"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.020b413.20231226195109":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.020b413.20231226195109","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"exports":"./dist/index.js","scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.020b413.20231226195109","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"020b413a4de38a443e78d7d6ac37e1658c444fc0","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-UF+NiYgR/d2/siYJa17HKqTjxGhwCBjntjafISOLQS3FnspE29xodysg+02npgsL8uzhKnSxQlWKJuzJl1Zx5w==","shasum":"8bec955a6cc3ebe446b8abb01503600faf4ec838","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.020b413.20231226195109.tgz","fileCount":48,"unpackedSize":146112,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIGhOSu0cCPwW6dBCK95X2xmZrS3Uel5xMoJzwbIwPNjZAiALPSEsrJoJkYaJsc6TBXYzBd4hPS7xCo+hf0AK10jm7w=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.020b413.20231226195109_1703620339302_0.7317937311148892"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.1a23a66.20231226202132":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.1a23a66.20231226202132","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@octokit/graphql":"^7.0.2","@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.1a23a66.20231226202132","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"1a23a6661f2581572d78cf76391c788ad79fe1c0","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-U26gkyuZNBuCDcYLlR1XJksPnMhXr9AqA6kZc2g+lmOIT1leis7nE4wLW5aTtyMLSN/zcjQmhMGz3NFk16wZnw==","shasum":"c387723540114d6f540304b91df42ebd98c3de1e","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.1a23a66.20231226202132.tgz","fileCount":48,"unpackedSize":145414,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCmQMznZDZaf4EeKYVR+kPtw6JImPBF1RXWpLP1rE6F0wIhAOVF99Ba8UG2Epx8NhqNt3wBeX7/nGlal23GkwMngeiG"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.1a23a66.20231226202132_1703622160790_0.015507236836410643"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.75da91f.20231226214747":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.75da91f.20231226214747","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.75da91f.20231226214747","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"75da91fc3c8dd69f37fa65ee8c3495ea9a47681c","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-jk+vXQg5uyj0kTXjSpQfKmF5r1Bww1wsyRacS+GrBc5la8z0sl4Ot11nvkaE5YfveVpmGB5ZZXx4ug2qO9TgTA==","shasum":"9d492e0e87650c463ac9f8961c873ca59c42077a","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.75da91f.20231226214747.tgz","fileCount":48,"unpackedSize":145414,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBuV2gE+FC8i62f52NAABTMtlblkRTbSFIGKeWA2ATCcAiAeUN9xfSvg5Pyv9hEtf2ekGyXgDogzpDTdPSLNI7qd9Q=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.75da91f.20231226214747_1703627336740_0.3455207841140313"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.5a8606e.20231226215058":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.5a8606e.20231226215058","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.5a8606e.20231226215058","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Running cuttlecat\n\n```shell\nnode dist/index.js --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: index.js --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\nnode dist/index.js --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"5a8606e30cd0749250fc2a61399fca2bd8d1d99e","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-zKvD3ux3bzUK/BIQoJlxQdZMLfZGq/hg2Ax9JpChV04UzRZBiNAMOFtQARdW7AWuzGaVdKCHPa7pIznr9OStfw==","shasum":"594433a6253fc05df4cdbf15207183dff24bc3a0","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.5a8606e.20231226215058.tgz","fileCount":48,"unpackedSize":145454,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD61ayAWFHSw+f77iE3ObnhF3lVQNJ0eX10tzfl1pWtqgIgLEUcVheDTWAd+poS1HJHMPQ4AUIQSxgDzqMm97cLlkI="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.5a8606e.20231226215058_1703627526365_0.8059664037972345"},"_hasShrinkwrap":false},"0.0.7-dev-snapshot.f7f92ce.20231226220816":{"name":"@opentr/cuttlecat","version":"0.0.7-dev-snapshot.f7f92ce.20231226220816","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-dev-snapshot.f7f92ce.20231226220816","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Running cuttlecat\n\n```shell\ncuttlecat --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: cuttlecat --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"f7f92cebbce8c8ca56a0f537e0e980d20f8dde67","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-u6DnSdDWXRbfiDVFsrjkw58E1xV3+i0E7Vrcnehir0GuqIsWgnpLc3n/oH2yGqh5yqBM+g/n6TEMYXJtDAARYg==","shasum":"a26c501865dcd6a2806a914f73ecea27113e310a","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-dev-snapshot.f7f92ce.20231226220816.tgz","fileCount":48,"unpackedSize":145509,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIF57AhLjzD8Pr4LmcV6AJp6sIkXu4LVoi6x0fl1VxemlAiEA/Zwy8eayZVfIV9gDWxS75PRLDobfWEN3pJXE844FI6w="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-dev-snapshot.f7f92ce.20231226220816_1703628565202_0.2662238234360188"},"_hasShrinkwrap":false},"0.0.7-snapshot.9fc1f0b.20231226220857":{"name":"@opentr/cuttlecat","version":"0.0.7-snapshot.9fc1f0b.20231226220857","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7-snapshot.9fc1f0b.20231226220857","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Running cuttlecat\n\n```shell\ncuttlecat --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: cuttlecat --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"9fc1f0bcb00ec8dae3127bd41d5936254d7ba6e4","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-A3W9qMSRKx0RasCNTcejyYfo21IQldQFmhZmmhymxzeflxxCQS7/jXoXkYixZti333IYQrRoAStudPhzh2gPbw==","shasum":"3755b69f0c54e793166af10639b8efa778da8cb7","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7-snapshot.9fc1f0b.20231226220857.tgz","fileCount":48,"unpackedSize":145505,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCHnIYDPqA4aE5NqLEY1QzqDUDXbc4HrhX6Uumt3f739AIhAM7sLKCqhlrrXK9PqXn0OxM3on9RBygKa8CH2zvOzGvi"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7-snapshot.9fc1f0b.20231226220857_1703628605979_0.8126272264671801"},"_hasShrinkwrap":false},"0.0.7":{"name":"@opentr/cuttlecat","version":"0.0.7","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.7","gitHead":"9fc1f0bcb00ec8dae3127bd41d5936254d7ba6e4","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-eqGf0zIwneoxZYgNS7T2UVySgKPiacKNhsuFlUfTESZjGtGVK7UJ5BxU8U1zJJL+GwvtECYzUga/d2ZDrVQd+w==","shasum":"d57cdebbef7996d4d43771e74098656f747d1c04","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.7.tgz","fileCount":48,"unpackedSize":145473,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDtmCaKVLqhgB8FN8gFHuZnNvPo4FI6Akd1hnPUJWo/cQIgLnSZzTnD3M21JRqpK/JccYCW91ViFV/fRkhymWf0J8M="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.7_1703628648101_0.25306365867305947"},"_hasShrinkwrap":false},"0.0.8-dev-snapshot.87273cc.20231226221001":{"name":"@opentr/cuttlecat","version":"0.0.8-dev-snapshot.87273cc.20231226221001","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.8-dev-snapshot.87273cc.20231226221001","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Running cuttlecat\n\n```shell\ncuttlecat --command-file= --data-directory= --github-token=\n```\n\n### Usage\n\n\n```shell\nUsage: cuttlecat --command-file= [options]\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat --command-file=\"./test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"87273cc009182c03e779bc1c13bba5b786dfbac4","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-Y4IPSdWyS7vTGhLXVefJH2mwBUnDFKL1FD6kubQ2C7kgfUjtIqK/3yhDQ0AhhhtdYb2we7rk9IjOLeTyhByCyw==","shasum":"b5e5b9e231018625bbf4ff6dcb58368391677e41","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.8-dev-snapshot.87273cc.20231226221001.tgz","fileCount":48,"unpackedSize":145509,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDo8mMFago5b4bwqRIg5/MNbUFp9U4JGgY4t4NHlrzQdgIhAJRejYRDKiHlNEWkXo6RMC+q/xmvN0a2io2ZGW6Gpdck"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.8-dev-snapshot.87273cc.20231226221001_1703628671130_0.520572670682627"},"_hasShrinkwrap":false},"0.0.8-dev-snapshot.2eb6052.20231228193836":{"name":"@opentr/cuttlecat","version":"0.0.8-dev-snapshot.2eb6052.20231228193836","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.8-dev-snapshot.2eb6052.20231228193836","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"2eb60526871171cc8f469187f7f68a9c0d93f1fd","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-XF7rcj881+bO3moFoohTYe824D3DQ/kCgMLzmHkD1LTt4zzJ0euz9XnkALjRm50yueL43Waretj3XUfpdtTNcQ==","shasum":"c0253f6d8c8446492c7fcdfae1c04d5089ea91d7","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.8-dev-snapshot.2eb6052.20231228193836.tgz","fileCount":56,"unpackedSize":158093,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC5K2iIcD+mtOlemsS3L1XxzTFVa3Az8QZB3w+jrX78ogIhANRGeAxqtdgyxbKhfXGoewINbLZE/0drlx47kNlndOdv"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.8-dev-snapshot.2eb6052.20231228193836_1703792387000_0.4927822656209082"},"_hasShrinkwrap":false},"0.0.8-snapshot.6101b0f.20231228203843":{"name":"@opentr/cuttlecat","version":"0.0.8-snapshot.6101b0f.20231228203843","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.8-snapshot.6101b0f.20231228203843","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"6101b0f1e047ab76ece6b3a0ed516f60f0166d2c","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-kiXI8m1N+l71kFVmPCjUW3Ev9yEsRxMhadFazQNQVSx2viJR0Id/aPA5UzH5RXM3vgkCY5ojwQFn0TTsjAvnTw==","shasum":"6f7d162253ca1d8ed17d00bffc307335dcac3948","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.8-snapshot.6101b0f.20231228203843.tgz","fileCount":56,"unpackedSize":158089,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIADiiIgV27eXC5qB96ngOoUTmcvkqnEMPUkRcX4sGfdIAiEAj/nmIzmEGy9rNecgNjGF23kLjAoqAu5TLIozMkw/Xis="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.8-snapshot.6101b0f.20231228203843_1703795993460_0.2736224181647853"},"_hasShrinkwrap":false},"0.0.8":{"name":"@opentr/cuttlecat","version":"0.0.8","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.8","gitHead":"6101b0f1e047ab76ece6b3a0ed516f60f0166d2c","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-Mp+sXGERFXVkpC2vKWEeWEzfi18zLyv8Y7FY9r8ns4z8smKNTdG7nZ1wWdZ5/ZVtvHcgi6954N4aJYF5vPnrkQ==","shasum":"238b87a18b2aebc99cc2b05579fdfadab3e0aa51","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.8.tgz","fileCount":56,"unpackedSize":158057,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDL86Ei+y+aqPoujCzMxBx0iS8qZCQgGw8qL8J8qa2FjwIgLz5FISqmQeybvjNAZFcY9mgkDX6+uQPiZZp99gqkFFk="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.8_1703796019213_0.8667658270401748"},"_hasShrinkwrap":false},"0.0.9-dev-snapshot.f07e3aa.20231228203921":{"name":"@opentr/cuttlecat","version":"0.0.9-dev-snapshot.f07e3aa.20231228203921","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.9-dev-snapshot.f07e3aa.20231228203921","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"f07e3aa0af07862afc34e6bd641518aa92e8f647","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-Sf6z8bdzHZTqvlVOAUwV5O1GIRRQhLqXMu0mb1Ln3x3LqN0PLqKh7CiVuhY6CMGQHnEwPvqNZnweg5CpsMWB9g==","shasum":"2bdc4f9b83b5d660dcca5f850054d0900f50eda2","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.9-dev-snapshot.f07e3aa.20231228203921.tgz","fileCount":56,"unpackedSize":158093,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIH30Bt7w8Fm7BbX3AHjsbKMtBt8iWRKcLdl/YbmprRs4AiEA9XxrGzRy0nF3ItYr4+oB6MZ/3c6BmgFd2Zto+H7SHl4="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.9-dev-snapshot.f07e3aa.20231228203921_1703796030552_0.22717243241015717"},"_hasShrinkwrap":false},"0.0.9-dev-snapshot.43f7434.20231230195745":{"name":"@opentr/cuttlecat","version":"0.0.9-dev-snapshot.43f7434.20231230195745","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.9-dev-snapshot.43f7434.20231230195745","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"43f7434dc7b47723e153cc27250739242c4432e5","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-4TmgccVBCuwPgn2aoq0pdt3BCQ+RIewEna3ZzyEmeEdDvN8Wz/xtQ7P9Kcpth5J8YzyIvMosmAJsbloCYF0qrA==","shasum":"03ba3bc5dea66883df89669b094167cdce37fba8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.9-dev-snapshot.43f7434.20231230195745.tgz","fileCount":56,"unpackedSize":159659,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIAcDLM013CYruVb5GwOH+UcSIYXP2lA7faI7jsd2IdbfAiEA59hY29ay9mJYWFEbpPKJ1v4kvTzA/pq5cJ00bzk3d8M="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.9-dev-snapshot.43f7434.20231230195745_1703966335678_0.9317111440900381"},"_hasShrinkwrap":false},"0.0.9-snapshot.fadb1c6.20231230201106":{"name":"@opentr/cuttlecat","version":"0.0.9-snapshot.fadb1c6.20231230201106","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.9-snapshot.fadb1c6.20231230201106","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"fadb1c65db5dea056fa99c1ef2b64089e5fa3f71","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-V1HnGrA1HTO4e6Q+Vzw1bJL8paRz1TUoHOwTc3do3Y0VqXzFG0KrTjSmV0DtV5j3lmLsSbrsDfwnmN3E52PjnQ==","shasum":"b66b237a34d33e9eb957971a89d79dfa54a2aec6","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.9-snapshot.fadb1c6.20231230201106.tgz","fileCount":56,"unpackedSize":159655,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCH/l3cOco5D6LmGRG3mBCJPOs38kNuNxZrPM2n5BZfcwIhAJUL6twPtjNgDazPLR9AU0CLscyH61u6PLUwyfx8d8Q8"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.9-snapshot.fadb1c6.20231230201106_1703967135284_0.3622700639201646"},"_hasShrinkwrap":false},"0.0.9":{"name":"@opentr/cuttlecat","version":"0.0.9","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.9","gitHead":"fadb1c65db5dea056fa99c1ef2b64089e5fa3f71","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-nsFWuh4rORh9iebKL5z3vx9Yg77B6So6Mwqh+5AwHtbgiyszem0BYvPBkuR7OLWhgsRvEo73sCWRbwyGJqtkHw==","shasum":"618e835d6e25ba6cff5b1bbecb9679260f9db2f4","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.9.tgz","fileCount":56,"unpackedSize":159623,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIB91nR0aRxCkF6DvJA/Db9S+cbgQfMS6n+N6Bcvk0KsLAiBbI3sgCHv7rU+V3ozaiUtTMqoqELFl4ph4Vliyuw3oig=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.9_1703967146855_0.31130080357813994"},"_hasShrinkwrap":false},"0.0.10-dev-snapshot.52b0bd0.20231230201136":{"name":"@opentr/cuttlecat","version":"0.0.10-dev-snapshot.52b0bd0.20231230201136","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10-dev-snapshot.52b0bd0.20231230201136","readme":"TODO:\n- requeue command (need to convert current to `execute` command)\n- release tooling\n- CI\n\n# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nTODO: links to more complicated commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"52b0bd058383a4424ef6898305b86dbc320032ea","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-d8eDPX+VAF8Gj2hUFzhlEknmw+S+iWHH/4+MRdSjw+TtaGSRTofVzg+DbvgbnO0xOXv3xbOBUAT/naF2rUR/zQ==","shasum":"633ec0efcb9b1828f701768383fc0ac227f06db9","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10-dev-snapshot.52b0bd0.20231230201136.tgz","fileCount":56,"unpackedSize":159660,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIByPn0axU15Odr2crYvDTe5XBecTtNvoOyjwIppVUQnKAiBNmgDlC0pvleGi1tDIExz3MojHaD/+3Zq76lKLyBDByw=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10-dev-snapshot.52b0bd0.20231230201136_1703967165119_0.135062656808971"},"_hasShrinkwrap":false},"0.0.10-dev-snapshot.bf9b48f.20240107151838":{"name":"@opentr/cuttlecat","version":"0.0.10-dev-snapshot.bf9b48f.20240107151838","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10-dev-snapshot.bf9b48f.20240107151838","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR's [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nSee the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"bf9b48fddf8e621cfb34b0f976923788e97e46b9","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-TsrknDLArf/IST1lsIVrHzEiU8xmmpc70gLp4EOdxa2sTnItNdeqzSEx1haS8E1NzuAzzaWcaKabU1q2yIvc+g==","shasum":"4c9828a01a9970362f119585aa42a22bfe0a7d56","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10-dev-snapshot.bf9b48f.20240107151838.tgz","fileCount":56,"unpackedSize":160032,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIERD6LTTIxOmKjnY7Bu9foafXTRPyVrcPwxEXqwT6f4XAiEAz6xALHVTXmAqaDj3gLmD3QzGZqLt0act102SqWmcz6g="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10-dev-snapshot.bf9b48f.20240107151838_1704640789491_0.07922229360792699"},"_hasShrinkwrap":false},"0.0.10-dev-snapshot.b377afd.20240107153605":{"name":"@opentr/cuttlecat","version":"0.0.10-dev-snapshot.b377afd.20240107153605","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10-dev-snapshot.b377afd.20240107153605","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"b377afd3a2a930ff594da4fa28df1c5354c52494","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-z1jW3M9BjujG+z5lP7WWh6x8Hh8JuGJnrxwFFIR6sXZV79IK//xhoTrR39i9/k48pr4gRJlF2qRiN3ZRTXAh8g==","shasum":"0e9a1093f27fb808d8e517e6b725e38877d9533b","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10-dev-snapshot.b377afd.20240107153605.tgz","fileCount":56,"unpackedSize":160249,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD8poTcHxxMPPPRG+cSBwPqY5lrrroAGX/1McIlXEY4YgIhALzU1iGjCUqkyrcNTCoHjlepheey/rNa4qJXvDP4VYvF"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10-dev-snapshot.b377afd.20240107153605_1704641835680_0.48830370427437275"},"_hasShrinkwrap":false},"0.0.10-dev-snapshot.2e36ffd.20240114104522":{"name":"@opentr/cuttlecat","version":"0.0.10-dev-snapshot.2e36ffd.20240114104522","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10-dev-snapshot.2e36ffd.20240114104522","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"2e36ffda5bb9c8e8cf4fa0237fc4dcf7942d0c67","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-mAt1EvF/Xc7aGCmXmFXQ0RulS25D4SDO6stEno2CPZ+HjvNRQaNd+gNCnfehZVGJvZSX611GSrPJjdxl4+owWQ==","shasum":"96493bebf7a07beb30dce7a100b9671cf71cd091","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10-dev-snapshot.2e36ffd.20240114104522.tgz","fileCount":56,"unpackedSize":160352,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDkAEb+AkvrGuqQrq9oGpL/h4WEtjlSq7sKgCvRXDcnFAiEA6CLGmZdOAfACvSANnK04sXs2dgql54wO2d9MouAmZ8w="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10-dev-snapshot.2e36ffd.20240114104522_1705229192393_0.22488204468459916"},"_hasShrinkwrap":false},"0.0.10-snapshot.49a80b8.20240114104638":{"name":"@opentr/cuttlecat","version":"0.0.10-snapshot.49a80b8.20240114104638","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10-snapshot.49a80b8.20240114104638","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"49a80b85fbf2959a924be05cdca3b1a0fe34eaaf","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-arE0PxBXmpJnczUo7taBEP09JHd/36rYJSdvf7IeopfE3QEy4IbP74iB7/2AMV8FFr30lwNK1uTbBxYWqMvtig==","shasum":"355fe537d57ad6d96a90d57ef4042c66f230e013","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10-snapshot.49a80b8.20240114104638.tgz","fileCount":56,"unpackedSize":160348,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCN4lHcetbyGUBQ1xq/5kBUoGd1BpLU5/Lf1EIsAy9tUwIgfwDoEphv3ZjCeT79AK8pUO+gNixnldzxLfIDqtzC1wM="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10-snapshot.49a80b8.20240114104638_1705229267111_0.4984927837848514"},"_hasShrinkwrap":false},"0.0.10":{"name":"@opentr/cuttlecat","version":"0.0.10","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.10","gitHead":"49a80b85fbf2959a924be05cdca3b1a0fe34eaaf","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-gByF60aEfL4YpWf+wZwXgfYkpL8MjSwnwcc+UEdRp2gW1qGq6cJu/VP4ZVodZaMnKg9Mf4pIijR3iaV+Uks6Mw==","shasum":"abf88bc930171f17e722860374dd89d49485c787","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.10.tgz","fileCount":56,"unpackedSize":160316,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGCYNsTcZvz8E7/EI1q5CV++/vTLWOEYtfVybJq8Tp3oAiEAmM2bMH4T0zWqI6HaZLhTGd2gBvmOw3BR43+XfS5USQ4="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.10_1705229276772_0.5614270806820836"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.a2b7a07.20240114104704":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.a2b7a07.20240114104704","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.a2b7a07.20240114104704","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"a2b7a070fcc8645aa0c2bda062954ba37d707c9b","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-xsQig4Gv+TxeUuPizhpcr/2iRnW4cUFyVvHFoFxBD+mDz9w/5wcyZkSuidBqRsMkZRu4F0eP8QPlPafoT065kA==","shasum":"8152a44a41a9c1e33fd9fc78a96b48660df80c83","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.a2b7a07.20240114104704.tgz","fileCount":56,"unpackedSize":160352,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIF/ArVLuzgzcFz9e3ay0SZDeiZFhXlx6J5KAdnMCQsIVAiEAv1U0thyijRsXCX/zyewzVNWHjCAyD1SyEvNKnurpiHY="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.a2b7a07.20240114104704_1705229293989_0.4546480075151291"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.c112b3e.20240124202555":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.c112b3e.20240124202555","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.c112b3e.20240124202555","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"c112b3e537fe095db87f2367ff076275dd63c11f","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-Psj6+P61eE/omNlZP0DihlT2Fta1kM4erM1yLlv3ZYGVCf1yLqdDyKgglv64VyREhK4wywSQab/4Pg4u7nBm1Q==","shasum":"b075d3d69910ffc98beda0eee2814135fe7e3aac","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.c112b3e.20240124202555.tgz","fileCount":56,"unpackedSize":160415,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIEUWSh1F81n540/4+CltMJlCmxZHRtRW2BpaN3XMb4mgAiBZnEbRZ8g08MoAP6wGpZqouHD8gcFOQl6GbpMoM7qa7Q=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.c112b3e.20240124202555_1706128025140_0.3497916699972645"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.09c38ea.20240124204513":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.09c38ea.20240124204513","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.09c38ea.20240124204513","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"09c38eafc0240537b13ffab8e0ba68a090554d1e","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-EOcgraf7YNnWTYJs+DGrsWUqS8pezATck8A0iZsshLsS2T26mrs6M3ZDalhLc9fdiJgFe/0aFdHQiFUp9eapeg==","shasum":"71057b2197253290c0a069ffa7336bae06116c53","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.09c38ea.20240124204513.tgz","fileCount":56,"unpackedSize":159916,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCb4gnS4EzEGwq52eT50MoiHejLP0fco+VKWanB6duHVQIgQA69fFIqsyUKH+oJC/aLP0CEwX3Z0pGJJSgUgkphiDw="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.09c38ea.20240124204513_1706129184180_0.5423406679514293"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.c55265e.20240124212359":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.c55265e.20240124212359","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.c55265e.20240124212359","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"c55265ed573fc6f8729e24212dde88f8d1b0d485","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-SzYpnOHe4/nDBz6E4yTJpg+jVKw/D7f/sS2o1x88t32ndef6iL2L4X48rkvDLLMg5Gu2lKkCtxmhQGylo/PRqA==","shasum":"3590278ad8606671daa7aa6f1cec03dfd92cbc2c","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.c55265e.20240124212359.tgz","fileCount":56,"unpackedSize":160351,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIGzU0E63eDi6Ce3KjzuoWz3/8XMiSQyRGiu7Rr+0gH7mAiEAnBWeKRTH6DkbK3y2I5g0C0cOYbPNefqH48OWJO5KpO0="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.c55265e.20240124212359_1706131508935_0.4126096812341664"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.c5fe2a7.20240124214029":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.c5fe2a7.20240124214029","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.c5fe2a7.20240124214029","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"c5fe2a70768f3c9e86ac803f2648efee9b9b86fe","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-7sgJD9xIkYmuTE27GA5oYAE73UUzWBDuyfid9Upv9NfMJGWgCoYis9LZK4TA0vkQIchY0ZkHXh77/6sTI1gyfg==","shasum":"c4e6454fd02201f467bdb3aa87efae18d7bee1b3","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.c5fe2a7.20240124214029.tgz","fileCount":56,"unpackedSize":160722,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHNSe3aVC7zkt99CUlQeS8QTDlNXZ+/Bj5/ep46qrkwkAiBcQFqk3H4FnAatzCJQ7QiGl2k5D31Dpxj9J2M2okRC7g=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.c5fe2a7.20240124214029_1706132500173_0.46713856210513893"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.d4b66cb.20240124215552":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.d4b66cb.20240124215552","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.d4b66cb.20240124215552","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"d4b66cbf21c54454bf358d4e9fd6288e58adceaa","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-/ihxqq4GKSaU62lF2Qan9+3ZbTNSZqrDK02CAg7AyTYNTOKjPzdMudIDyB5QXRmF9pW5S6B3i0LzKC0B81WI6w==","shasum":"eef1261fd52945265032bfe4a81f7ec280654edf","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.d4b66cb.20240124215552.tgz","fileCount":56,"unpackedSize":160831,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDgit805vxgQ63bkCx6E+tt83AUmqhjrAhnBUKZ+A5BrQIhANA6Ua70kHc7gmKqkgJr9rKrh3Marlb96VyKGkVF4uTd"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.d4b66cb.20240124215552_1706133423516_0.7396540153338123"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.9068f70.20240124221120":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.9068f70.20240124221120","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.9068f70.20240124221120","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"9068f70f5e3564a0cf3f15769f712a713f9ed06f","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-Dg4WeOzvQ2KOeSyVV5eJHh/WT3DrzWL5gPCUTpicWwXpsILp6xGlUiN+1oTtGO+IJ2aJ1QPG47MFBKn2ku08og==","shasum":"426129f2764ca292b3a5e81d25b4d2c0c3a7b1d1","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.9068f70.20240124221120.tgz","fileCount":56,"unpackedSize":161903,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIA+/yMSOmQ9X5hlRm60a7UB8GBxvP1HRrOvfiHTsr2BXAiBnOfjWP0iyc2UVgDWD8IntK4jeDSitDJS8K/dXTRZ2mA=="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.9068f70.20240124221120_1706134349150_0.8313312404394193"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.3e934f4.20240124222949":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.3e934f4.20240124222949","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.3e934f4.20240124222949","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"3e934f461eb7f706f24dfdfc0f7ff683c78f8dba","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-vUGIYKEinc2kyq/sN2metAAEyrYqxHd5FPMnqL4olN6in5Zg7E8XV/E75hDNcjgsD9c0B1aOGoE8EGd7W8XGkg==","shasum":"8c31cfa7f59299f200aae5bea394cdad724f58e8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.3e934f4.20240124222949.tgz","fileCount":56,"unpackedSize":162481,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDy2y1Maz4HOyI5ZxOeQwkg10ZL+MHzMjfwaEOyHS8LlQIhAM9y6vFKX0w+Oj3ojf4ZaO92+BpTlZnvopR3SD0Kog1Q"}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.3e934f4.20240124222949_1706135459058_0.5460122176740314"},"_hasShrinkwrap":false},"0.0.11-dev-snapshot.db42b3f.20240124224534":{"name":"@opentr/cuttlecat","version":"0.0.11-dev-snapshot.db42b3f.20240124224534","description":"cuttlecat is a tool for collecting data from GitHub.","keywords":["github","graphql","octokit"],"author":{"name":"Ali Ok"},"license":"Apache-2.0","homepage":"https://opentr.foundation/","repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"type":"module","engines":{"node":">=18.0.0"},"bin":{"cuttlecat":"dist/index.js"},"scripts":{"build":"npm run clean && npm run compile && npm run lint","start":"npm run build && node dist/index.js","test":"npm run build && mocha './src/**/*.test.ts' --recursive","clean":"rimraf ./dist","compile":"tsc","lint":"eslint ."},"devDependencies":{"@tsconfig/node16":"^16.1.1","@types/chai":"^4.3.11","@types/chai-as-promised":"^7.1.8","@types/lodash":"^4.14.202","@types/mocha":"^10.0.6","@types/mock-fs":"^4.13.4","@types/node":"^20.6.3","@types/node-fetch":"^2.6.9","@types/seedrandom":"^3.0.8","@types/uuid":"^9.0.7","@types/yargs":"^17.0.29","@typescript-eslint/eslint-plugin":"^6.9.0","@typescript-eslint/parser":"^6.9.0","chai":"^4.3.10","chai-as-promised":"^7.1.1","mocha":"^10.2.0","mock-fs":"^5.2.0","rimraf":"^5.0.5","seedrandom":"^3.0.5","ts-node":"^10.9.1","typescript":"^5.2.2","eslint-import-resolver-typescript":"^3.6.1","eslint-plugin-import":"^2.29.0","eslint":"^8.52.0"},"dependencies":{"date-fns":"^2.30.0","date-fns-tz":"^2.0.0","graphql":"^16.8.1","lodash":"^4.17.21","nock":"^13.4.0","node-fetch":"^3.3.2","p-queue":"^7.4.1","uuid":"^9.0.1","winston":"^3.11.0","yargs":"^17.7.2","@octokit/graphql":"^7.0.2"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"_id":"@opentr/cuttlecat@0.0.11-dev-snapshot.db42b3f.20240124224534","readme":"# cuttlecat - Collect information from GitHub\n\ncuttlecat is a tool to collect information from GitHub. It is designed to be used in GitHub Actions, but can be used in any environment.\n\nFeatures:\n- [x] Feed your own search query and processing logic\n- [x] Fully extensible (contract based)\n- [x] Resumable search (store the state of the process and resume later)\n- [x] Rate limit aware (stop the process when the rate limit is low)\n- [x] Stops the process if data is already fresh enough\n- [x] Retry failed tasks\n- [x] Narrow down the scope of failed tasks (e.g. if a search for a date range fails, create tasks for shorter date ranges)\n- [x] Concurrent tasks (to avoid hitting GitHub secondary rate limits)\n- [x] Timeout for each task (to avoid spending too much GitHub action minutes)\n- [x] Max process run time (to avoid GitHub action cancellation of tasks that run too long)\n- [x] Report progress to stdout\n- [x] Record HTTP calls to disk for debugging purposes\n\n## How it works\n\ncuttlecat is a task runner. It takes a command file and a data directory as input. The command file contains the search query and the processing logic. The data directory is where the state of the process and the output will be stored. The process can be resumed later by using the same data directory.\n\nThis design allows the process to be fully extensible. You can write your own command file to search for anything you want and process the results in any way you want.\n\n## Installing cuttlecat\n\n```shell\nnpm install -g @opentr/cuttlecat\n```\n\n## Usage\n\n\n```shell\nUsage: cuttlecat [options]\n\nCommands:\n cuttlecat execute Execute the command within the given file and store the result.\n cuttlecat latest-process-complete Checks if the latest queue is marked as complete and prints the result in the stdout.\n cuttlecat requeue-tasks Manually requeue tasks for trying them again.\n\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n```\n\n## Command `execute`\n\n\n```shell\nUsage: cuttlecat execute [options]\nRun cuttlecat --help for help on common options.\nExecute the command within the given file and store the result.\n\nRequired options\n --command-file Command file to load. [string] [required]\n --data-directory Data directory to read and store the output. [string] [required]\n --github-token GitHub API token. Token might need permissions based on your task. [string] [required]\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --renew-period-in-days Number of days to wait until creating a new queue after the latest one is completed. [number] [default: 7]\n --concurrency Number of concurrent tasks to process the queue. As this search is IO bound and CPU bound, there can be many concurrent tasks (more than the number of cores). However, because of the rate limiting, there will be a lot of idle tasks. So, it is recommended to keep concurrency low. [number] [default: 6]\n --per-task-timeout-in-ms Timeout in milliseconds for each task in the queue.Keeping the timeout too long will end up using too many GitHub actions minutes.Keeping the timeout too short will result in too many errored items. [number] [default: 30000]\n --rate-limit-stop-percent Under this rate limit remaining percent, stop the queue. [number] [default: 10]\n --interval-cap Max number of tasks to execute in the given interval by interval-in-ms. [number] [default: 4]\n --interval-in-ms Interval for the cap in milliseconds. [number] [default: 20000]\n --retry-count Number of retries for each task before giving up of creating narrower scoped tasks. [number] [default: 3]\n --record-http-calls Record HTTP calls to disk for debugging purposes. \"Nock back\" will be used in `record` mode where the new records will be created. The calls will be stored in the `./nock-records/` directory, relative to the command path. [boolean] [default: false]\n --log-level Log level to use. [string] [default: \"info\"]\n --max-run-time-in-minutes When to stop the command gracefully. For example GitHub Actions has a 3 hour limit and when it cancels, nothing is saved. However, GitHub sometimes cancels before the limit to possibly make rooms for other systems/actions, so set it a bit lower than the limit. [number] [default: 60]\n --report-period-in-ms Period in milliseconds to print the queue state to stdout (0 for disabled) [number] [default: 5000]\n\nExamples:\n --data-directory=/path/to/directory Store the state of the process and the output in /path/to/directory, so that subsequent executions of the same command can be resumed.\n --renew-period-in-days=7 If the process is complete (all search periods are processed), don't start a new search until 7 days has passed after the latest completion.\n --concurrency=6 --interval-cap=4 --interval-in-ms=20000 Start 6 concurrent tasks each time, and execute 4 tasks in every 20 seconds. (change these to avoid hitting GitHub secondary rate limits)\n --retry-count=3 When a task fails, retry 3 times (in total, 4 times). If it still fails, process will create tasks that have narrower scopes. If the task's scope can be narrowed down, then the task will be archived. If not, it will stay in the errored list. This narrowing down will also happen for any narrowed-down tasks that fail (tried 4 times in total), until they cannot be narrowed down anymore. For the commands that use a date range to search for, tasks for shorter search ranges will be created that in total wrap the failing task's search range.\n --per-task-timeout-in-ms=30000 For each task, wait for 30 seconds before timing out. You change this to avoid spending too much GitHub action minutes. If the timeoutis too short, there will be too many errored items. However, the process will retry and create narrower scoped tasks for errored items, so, having a very long timeout is not very useful.\n --report-period-in-ms=5000 Print the queue state to stdout every 5 seconds. This is useful to see how many tasks are in the queue, how many are completed, how many are errored, etc.\n```\n\n### Running the sample search command\n\nTo run the sample command:\n```shell\nrm -rf /tmp/foo/bar\nmkdir -p /tmp/foo/bar\n\ncuttlecat execute --command-file=\"../test/test_tasks/basicUserSearch.js\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --github-token=\"$(gh auth token)\"\n```\n\nThe sample task will search for users who have location set to \"Istanbul\" and signed up in January 2020.\nThe output will be stored in `/tmp/foo/bar` directory.\n\nSee [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts) for the implementation of the sample command.\n\n## Command `latest-queue-complete`\n\n\n```shell\nUsage: cuttlecat latest-queue-complete [options]\nRun cuttlecat --help for help on common options.\nChecks if the latest queue is marked as complete and prints the result in the stdout.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --data-directory Data directory to check the process files. [string] [required]\n\nExamples:\n NOTE: Examples below are not executable commands, they are just examples of how to use the command.\n --data-directory=/path/to/data/directory Check if the latest state file in the given directory was complete. After you start another queue that produces a state file, you can run this command to check if it is complete. This command writes true or false to stdout, which can beused in a script to determine if the previous queue was done.\n```\n\nExample execution:\n```shell\n$ cuttlecat latest-queue-complete --data-directory=\"/tmp/foo/bar\"\ntrue\n```\n\n## Command `requeue-tasks`\n\n\n```shell\nUsage: cuttlecat requeue-tasks [options]\nRun cuttlecat --help for help on common options.\nManually requeue tasks for trying them again.\n\nOptions:\n --help Show help [boolean]\n --version Show version number [boolean]\n --requeue-type Type of tasks to requeue. 'errored' will requeue all errored tasks. 'non-critical-errored' will requeue tasks that are not in the `errored` bucket, but resolved with non-critical errors. [required] [choices: \"errored\", \"non-critical-errored\"]\n --data-directory Data directory to for the task states and outputs. [string] [required]\n --timestamp Directory name under data-directory. [string] [required]\n```\n\nExample execution:\n```shell\nnode dist/index.js requeue-tasks \\\n --requeue-type=\"non-critical-errored\" \\\n --data-directory=\"/tmp/foo/bar\" \\\n --timestamp=\"1234\"\n```\n\n## Implement your own search command\n\nTo implement your own search command, you need to create a command file. The command file is a JavaScript file that exports a class that implements the [`Command` interface](src/graphql/command.ts).\n\nWhen you are implementing your own command, you will need to return some objects. These objects will be of types that you also need to implement. These types are:\n- [`Task`](src/graphql/task.ts): A task is a search query and the processing logic. The processing logic is a function that takes the search result, extracts the output, decides if there's an error, etc. The search result is a list of items returned by the search query. The search query is a GraphQL query that is executed by the GitHub GraphQL API.\n- [`TaskSpec`](src/graphql/taskSpec.ts): This is the input to create a task. This is serialized and stored in the data directory. When the process is resumed, this is deserialized and used to create the task.\n- [`TaskResult`](src/graphql/taskResult.ts): This is the output of a task. This is serialized and stored in the data directory.\n\nFor an example command, please see [`src/test/test_tasks/basicUserSearch.ts`](src/test/test_tasks/basicUserSearch.ts).\n\nOpenTR has built a report using cuttlecat: [State of Open Source Contribution in Turkey](https://state.opentr.foundation/).\n\nThis report, uses cuttlecat to collect data from GitHub. Then, the collected data is processed and a report is built.\n\nYou may find the data collection part in [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) and the report building part in [`state-of-oss-contribution-report`](https://github.com/OpenTRFoundation/state-of-oss-contribution-report).\n\nYou may see the [`state-of-oss-contribution`](https://github.com/OpenTRFoundation/state-of-oss-contribution) repository for more advanced command examples.\n\nTODO: links to more complicated example commands.\n\n## Using cuttlecat as a library\n\nInstead of creating a command file and feeding it to cuttlecat, you can use cuttlecat as a library. This is useful if you want to build your own tooling on top of cuttlecat.\n\nTODO: example\n\n## Building cuttlecat from source\n\n```shell\n# switch to the correct node version in .nvrmc\nnvm use\n# install dependencies\nnpm install\n# build the project\nnpm run build\n```\n\n## Running the tests\n\n```shell\nnpm run test\n```\n\n## Creating a new release\n\n```shell\n# update the version in package.json to something like \"0.0.6\"\nnpm install\ngit add .\ngit commit -m \"Release 0.0.6\"\ngit tag -a \"0.0.6\" -m \"Release 0.0.6\"\ngit push --follow-tags\n\n# create a new release on GitHub\ngh release create\n\n# update the version in package.json to something like \"0.0.7-dev\"\nnpm install\ngit add .\ngit commit -m \"Start 0.0.7-dev\"\ngit push\n```\n","readmeFilename":"README.md","gitHead":"db42b3f93214a0b2a25636f89e20d0b7f75c5b83","_nodeVersion":"18.19.0","_npmVersion":"10.2.3","dist":{"integrity":"sha512-opg+obhVvHCueG2iVV+VZ9G3o0xaVAiPXDZp7AhrdKr5dP6E7t5vhzjrOEjz3WLiA9tO6KdHEdLA90aVqYzO4A==","shasum":"3d5a6a79529574823aec462349592599aec99cb8","tarball":"https://registry.npmjs.org/@opentr/cuttlecat/-/cuttlecat-0.0.11-dev-snapshot.db42b3f.20240124224534.tgz","fileCount":56,"unpackedSize":162363,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIBc0M/FoTH5RV38R1QlfW8NAzSMITOTVSN+QQ51V3YAXAiEArkC/c0uVPt80PQjfAaE9bJT0goV3Q9EFio/+2XzzFHA="}]},"_npmUser":{"name":"aliok","email":"aliok@aliok.com.tr"},"directories":{},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/cuttlecat_0.0.11-dev-snapshot.db42b3f.20240124224534_1706136404348_0.29453834160018144"},"_hasShrinkwrap":false}},"time":{"created":"2023-10-15T22:27:32.050Z","0.0.1":"2023-10-15T22:27:32.418Z","modified":"2024-01-24T22:46:44.816Z","0.0.3-snapshot.8cbfc23.20231016.0":"2023-10-16T08:34:37.977Z","0.0.2-snapshot.deeab89.20231016085517":"2023-10-16T08:56:16.277Z","0.0.2":"2023-10-16T09:00:03.022Z","0.0.2-snapshot.3197c7b.20231016201323":"2023-10-16T20:14:24.828Z","0.0.2-snapshot.20889d0.20231016201809":"2023-10-16T20:18:56.451Z","0.0.3-snapshot.d815331.20231016211437":"2023-10-16T21:15:25.149Z","0.0.3":"2023-10-16T21:15:30.867Z","0.0.4-dev-snapshot.a94c76f.20231016212717":"2023-10-16T21:28:11.336Z","0.0.4-dev-snapshot.997eb38.20231017150700":"2023-10-17T15:07:44.330Z","0.0.4-dev-snapshot.dc46962.20231017154942":"2023-10-17T15:50:45.570Z","0.0.4-dev-snapshot.774bf0b.20231018193446":"2023-10-18T19:35:45.692Z","0.0.4-dev-snapshot.47a82d2.20231018193701":"2023-10-18T19:37:43.487Z","0.0.4-dev-snapshot.c0e0969.20231018193825":"2023-10-18T19:39:15.786Z","0.0.4-dev-snapshot.b8ffb37.20231019165513":"2023-10-19T16:55:59.973Z","0.0.4-dev-snapshot.41dd1dc.20231019165638":"2023-10-19T16:57:17.935Z","0.0.4-dev-snapshot.7140b6f.20231019170040":"2023-10-19T17:01:22.161Z","0.0.4-dev-snapshot.667fc8c.20231019170119":"2023-10-19T17:02:41.738Z","0.0.4-dev-snapshot.79d3a63.20231019170437":"2023-10-19T17:05:56.189Z","0.0.4-dev-snapshot.94589a6.20231019171914":"2023-10-19T17:20:44.795Z","0.0.4-dev-snapshot.91a1a23.20231019172909":"2023-10-19T17:30:46.210Z","0.0.4-dev-snapshot.a9fbb71.20231019180105":"2023-10-19T18:02:38.369Z","0.0.5-dev-snapshot.ba02839.20231019180528":"2023-10-19T18:07:08.172Z","0.0.4":"2023-10-19T18:07:22.783Z","0.0.5-dev-snapshot.090e653.20231019220422":"2023-10-19T22:05:46.031Z","0.0.5-dev-snapshot.e84d36d.20231021215056":"2023-10-21T21:52:27.046Z","0.0.5-dev-snapshot.cccd644.20231021225559":"2023-10-21T22:57:11.652Z","0.0.5-dev-snapshot.c0c51c7.20231022203840":"2023-10-22T20:40:00.860Z","0.0.5-dev-snapshot.ec4ecab.20231022205323":"2023-10-22T20:55:03.725Z","0.0.5-dev-snapshot.bfd164a.20231022214929":"2023-10-22T21:50:50.968Z","0.0.5-dev-snapshot.2860701.20231022215348":"2023-10-22T21:55:18.256Z","0.0.5-dev-snapshot.541d5bb.20231022221405":"2023-10-22T22:15:26.051Z","0.0.5-dev-snapshot.9369c32.20231023185022":"2023-10-23T18:52:16.536Z","0.0.5-snapshot.a62d082.20231023185209":"2023-10-23T18:53:49.775Z","0.0.5":"2023-10-23T18:54:31.237Z","0.0.6-dev-snapshot.7dda9b8.20231023191844":"2023-10-23T19:20:45.191Z","0.0.6-dev-snapshot.d3d7d39.20231023211835":"2023-10-23T21:20:28.064Z","0.0.6-dev-snapshot.27534b7.20231023221314":"2023-10-23T22:14:41.059Z","0.0.6-dev-snapshot.c2d41db.20231025192157":"2023-10-25T19:24:03.189Z","0.0.6-dev-snapshot.270c625.20231025211804":"2023-10-25T21:20:08.809Z","0.0.6-dev-snapshot.d6f808c.20231028105414":"2023-10-28T10:55:56.950Z","0.0.6-dev-snapshot.b59de4b.20231028161715":"2023-10-28T16:18:47.474Z","0.0.6-dev-snapshot.f8daaa2.20231028193624":"2023-10-28T19:38:13.690Z","0.0.6-dev-snapshot.edd45e8.20231028205944":"2023-10-28T21:01:16.777Z","0.0.6-dev-snapshot.b12e77e.20231028214151":"2023-10-28T21:43:36.186Z","0.0.6-dev-snapshot.bb1b2be.20231028214304":"2023-10-28T21:44:48.194Z","0.0.6-dev-snapshot.06b31af.20231031172229":"2023-10-31T17:23:57.241Z","0.0.6-dev-snapshot.74a86a1.20231102211924":"2023-11-02T21:21:10.952Z","0.0.6-dev-snapshot.34410ae.20231103083416":"2023-11-03T08:35:29.124Z","0.0.1-dev-snapshot.58de206.20231226161434":"2023-12-26T16:15:44.360Z","0.0.6-dev-snapshot.dc0c3e1.20231226171638":"2023-12-26T17:17:47.883Z","0.0.6-snapshot.fd78434.20231226172447":"2023-12-26T17:25:55.864Z","0.0.7-dev-snapshot.3d73036.20231226172527":"2023-12-26T17:26:36.306Z","0.0.7-dev-snapshot.7c01c3f.20231226172540":"2023-12-26T17:26:49.679Z","0.0.6":"2023-12-26T17:31:27.283Z","0.0.7-dev-snapshot.bbe8930.20231226173134":"2023-12-26T17:32:42.873Z","0.0.7-dev-snapshot.020b413.20231226195109":"2023-12-26T19:52:19.491Z","0.0.7-dev-snapshot.1a23a66.20231226202132":"2023-12-26T20:22:40.963Z","0.0.7-dev-snapshot.75da91f.20231226214747":"2023-12-26T21:48:56.929Z","0.0.7-dev-snapshot.5a8606e.20231226215058":"2023-12-26T21:52:06.550Z","0.0.7-dev-snapshot.f7f92ce.20231226220816":"2023-12-26T22:09:25.388Z","0.0.7-snapshot.9fc1f0b.20231226220857":"2023-12-26T22:10:06.136Z","0.0.7":"2023-12-26T22:10:48.415Z","0.0.8-dev-snapshot.87273cc.20231226221001":"2023-12-26T22:11:11.307Z","0.0.8-dev-snapshot.2eb6052.20231228193836":"2023-12-28T19:39:47.162Z","0.0.8-snapshot.6101b0f.20231228203843":"2023-12-28T20:39:53.682Z","0.0.8":"2023-12-28T20:40:19.433Z","0.0.9-dev-snapshot.f07e3aa.20231228203921":"2023-12-28T20:40:30.754Z","0.0.9-dev-snapshot.43f7434.20231230195745":"2023-12-30T19:58:55.848Z","0.0.9-snapshot.fadb1c6.20231230201106":"2023-12-30T20:12:15.494Z","0.0.9":"2023-12-30T20:12:27.068Z","0.0.10-dev-snapshot.52b0bd0.20231230201136":"2023-12-30T20:12:45.338Z","0.0.10-dev-snapshot.bf9b48f.20240107151838":"2024-01-07T15:19:49.664Z","0.0.10-dev-snapshot.b377afd.20240107153605":"2024-01-07T15:37:15.833Z","0.0.10-dev-snapshot.2e36ffd.20240114104522":"2024-01-14T10:46:32.554Z","0.0.10-snapshot.49a80b8.20240114104638":"2024-01-14T10:47:47.297Z","0.0.10":"2024-01-14T10:47:56.953Z","0.0.11-dev-snapshot.a2b7a07.20240114104704":"2024-01-14T10:48:14.179Z","0.0.11-dev-snapshot.c112b3e.20240124202555":"2024-01-24T20:27:05.328Z","0.0.11-dev-snapshot.09c38ea.20240124204513":"2024-01-24T20:46:24.362Z","0.0.11-dev-snapshot.c55265e.20240124212359":"2024-01-24T21:25:09.118Z","0.0.11-dev-snapshot.c5fe2a7.20240124214029":"2024-01-24T21:41:40.351Z","0.0.11-dev-snapshot.d4b66cb.20240124215552":"2024-01-24T21:57:03.683Z","0.0.11-dev-snapshot.9068f70.20240124221120":"2024-01-24T22:12:29.378Z","0.0.11-dev-snapshot.3e934f4.20240124222949":"2024-01-24T22:30:59.296Z","0.0.11-dev-snapshot.db42b3f.20240124224534":"2024-01-24T22:46:44.596Z"},"maintainers":[{"name":"aliok","email":"aliok@aliok.com.tr"}],"description":"cuttlecat is a tool for collecting data from GitHub.","homepage":"https://opentr.foundation/","keywords":["github","graphql","octokit"],"repository":{"type":"git","url":"git+https://github.com/OpenTRFoundation/cuttlecat.git"},"author":{"name":"Ali Ok"},"bugs":{"url":"https://github.com/OpenTRFoundation/cuttlecat/issues"},"license":"Apache-2.0","readme":"","readmeFilename":""}