{"_id":"pino-opensearch","_rev":"3-16a076e5adfd49ea660462befa22211f","name":"pino-opensearch","dist-tags":{"latest":"1.1.0"},"versions":{"1.0.0":{"name":"pino-opensearch","version":"1.0.0","description":"Load pino logs into OpenSearch","main":"./lib.js","bin":{"pino-opensearch":"cli.js"},"scripts":{"test":"standard && tap test/*.test.js -j 1 --no-coverage --timeout 120"},"pre-commit":"test","keywords":["pino","log","fast","opensearch","elastic","search","kibana","elasticsearch","aws","aws-opensearch","pino aws elasticsearch"],"author":{"name":"Hemant Kumar","email":"hemant6488@gmail.com"},"license":"MIT","devDependencies":{"@elastic/ecs-pino-format":"^1.1.1","is-elasticsearch-running":"^0.1.0","pino":"^6.2.1","pre-commit":"^1.2.2","proxyquire":"^2.1.3","standard":"^16.0.3","tap":"^16.0.0"},"dependencies":{"@elastic/elasticsearch":"7.13.0","minimist":"^1.2.5","pump":"^3.0.0","readable-stream":"^3.6.0","split2":"^4.0.0"},"repository":{"type":"git","url":"git+https://github.com/hemant6488/pino-opensearch.git"},"bugs":{"url":"https://github.com/hemant6488/pino-opensearch/issues"},"homepage":"https://github.com/hemant6488/pino-opensearch#readme","directories":{"test":"test"},"engines":{"node":">=10"},"_id":"pino-opensearch@1.0.0","_nodeVersion":"16.13.0","_npmVersion":"8.1.0","dist":{"integrity":"sha512-UmryrqF5622byTb2SUedSwKBDeCf3S9n2bbQ89ZkJ34wH8slM2PJb5NmFXlm6tkeL8kBhVgeku+SmzCzxgmOtg==","shasum":"d5aae076d3c4fd1467dc377e7a8dd244caec6549","tarball":"https://registry.npmjs.org/pino-opensearch/-/pino-opensearch-1.0.0.tgz","fileCount":19,"unpackedSize":49569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJiNV0CACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2VmrvDA//WwhSd2RDeU7E+1Ovfjmj4svRXnaTkaTOAUzmsWP6aSXz1wq7\r\npvMgJW+Ml5qVYI+l5Ecnex91A9rHZ+46RvKFa3gYAb9s+4qi3f92SVB/633y\r\novNp/Z6E3DOP5WG2MywXjDfae2Huu8K5MPS8+ZIcICHg8eYD6in/jeKRDQBg\r\n2EM3D+Wk2skxhZwaF5Cci95rw1lqZxFz0Dh9/zy9SFuqAsKEi4zRzQZkR3kr\r\nLjQU7JHjx9oFCtXubGNsMt+d82cWG0aLRlg2+q8Vr4dPqzXsn0KbqMxlt6X+\r\nY3SEBgQHug+5iJ6bXipCGPcXEpiW92eH4R9RyU6/V1AMlCzmO5JgxNnBT/sY\r\nL3xYNiQOReJ2g5ZaBiacIhkUEgc9JxXyzuYoCQtjCjOMyKLPU2nVRje+UWBW\r\nmKy0HC1l/yZkaHqum4eMPjBRTufNrYAXRRkPkn5z2ZogYOD5ebX/qOLU8a5l\r\nOcwI9PlaawAMU7sEqK46KgH6PwVnoDQcZrgvoGdZk8lcuAmsSnf1ET+PnoMq\r\nSuiw5sidjf4S7mmv4JlAYPP7XemS67mCYXCQrEQYaONuafZftLu56K6LIiOt\r\n2Pc+shDY0pTOUMBNRyUHVuW8ziQs7+AcMYsjuhH+/kHmmw0FGBvVfZ6D9ODX\r\ne7W3el8p4ivJZ6fyjAH7GlM/BOO8s2oJhRg=\r\n=YUXP\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD5kRI5RJryOcFVKMSKpPfmHFox9lK2kioPiSOlb2AfqAIgXCtYFC6nc1Cmj3mVsAJA5A4He4ozzNuG2/uSM7ugVr4="}]},"_npmUser":{"name":"hemant_autocorp","email":"hemant@autocorp.ai"},"maintainers":[{"name":"hemant_autocorp","email":"hemant@autocorp.ai"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/pino-opensearch_1.0.0_1647664386831_0.8135898066198168"},"_hasShrinkwrap":false},"1.1.0":{"name":"pino-opensearch","version":"1.1.0","description":"Load pino logs into OpenSearch","main":"./lib.js","bin":{"pino-opensearch":"cli.js"},"scripts":{"test":"standard && tap test/*.test.js -j 1 --no-coverage --timeout 120"},"pre-commit":"test","keywords":["pino","log","fast","opensearch","elastic","search","kibana","elasticsearch","aws","aws-opensearch","pino aws elasticsearch"],"author":{"name":"Hemant Kumar","email":"hemant6488@gmail.com"},"license":"MIT","devDependencies":{"@elastic/ecs-pino-format":"^1.1.1","is-elasticsearch-running":"^0.1.0","pino":"^6.2.1","pre-commit":"^1.2.2","proxyquire":"^2.1.3","standard":"^16.0.3","tap":"^16.0.0"},"dependencies":{"@elastic/elasticsearch":"7.13.0","minimist":"^1.2.5","pump":"^3.0.0","readable-stream":"^3.6.0","split2":"^4.0.0"},"repository":{"type":"git","url":"git+https://github.com/hemant6488/pino-opensearch.git"},"bugs":{"url":"https://github.com/hemant6488/pino-opensearch/issues"},"homepage":"https://github.com/hemant6488/pino-opensearch#readme","directories":{"test":"test"},"engines":{"node":">=10"},"gitHead":"cf2336cdc42dc5cbdb4cd78d6bc750f093f7674e","_id":"pino-opensearch@1.1.0","_nodeVersion":"16.13.0","_npmVersion":"8.1.0","dist":{"integrity":"sha512-LKIj+qis84k3GA2bOWkke5f78kINKmUfRn9QGZGi29O7nKCNBy4MRWxEHreuHkWEL17unqoLyckHT42+tDS9JA==","shasum":"3900c98b5a9bf7dfb571c30b11c9f65b0decc82f","tarball":"https://registry.npmjs.org/pino-opensearch/-/pino-opensearch-1.1.0.tgz","fileCount":19,"unpackedSize":49569,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJiNWCOACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2Vmq74g//VPn18LsIxrgPRuX1Mo5LHhRcXWMa7h1xt+Y8mBRKMGMNHieE\r\ndmDs0Gbhs9R1dWsWdNce3H4WHQ7wArOEtejBl4AwR24Mo74yKWzo6kX5TU8i\r\nxSBhBN2jcbnDrRravGj0YxwCCx+S2pVmuevG9tfQIX8cxisxoloM2I/ZNDVs\r\n6NxM0xM7BW0gq2xUT/OGbMsD1OOBfHmxhGmSdOeI+BQ+R0fYRX7pJMQnumng\r\nzSemogTG43iy2iworRf2bInBV4b0XyVI8GqgHwOPCpeTx8MXNmq/a3/4Gw0E\r\nwY683MskxujoZset4+rfTHoLSgtnOak1iV13tnR+/VSINCHsU7r01zWSE4wc\r\n4zPqRfruP+iIcNtOAGI2G/kBPXH+uSmsal13R7Lp9uakXCu0Glm0pvxATN8p\r\na6JU6RCfVh5FE2CzvcLG4gBx93UokTjFG1/wm2tRNATklcYGkyyxS3WAmRgK\r\nBOgAVjhKh0WZGBDR/ybuXQvrlzdB/ozZSAyjQQoHLp1RVjusZMQCIvcSK3+Q\r\ni5nQ/sc8vf4bcBA5SB5mcYoT8UhCZFRY5szpdx0ZdwJiKp9pyA7Y8jcBnAuA\r\n3HnoDVc3cSBOYTdPtZE6iMRlCHaY+a/ShX8DtowdXsaSodmvr2NsDq9282Q+\r\nqYyi8GwuHRGd9/XZdkAso9oqr8Eb0ko3ZBU=\r\n=bNzx\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIE3bnqb8dDJj8W5Mp4G+QkoR0eyVbaaguXiZoJ1eOhpPAiATB8O6sVulhhu8uoGX9dsLflfmKOvWOJX3k/tf61kHsw=="}]},"_npmUser":{"name":"hemant_autocorp","email":"hemant@autocorp.ai"},"maintainers":[{"name":"hemant_autocorp","email":"hemant@autocorp.ai"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/pino-opensearch_1.1.0_1647665294133_0.1228582501718618"},"_hasShrinkwrap":false}},"time":{"created":"2022-03-19T04:33:06.830Z","1.0.0":"2022-03-19T04:33:06.984Z","modified":"2022-05-12T23:21:16.368Z","1.1.0":"2022-03-19T04:48:14.335Z"},"maintainers":[{"email":"hemant6488@gmail.com","name":"hemant6488"},{"email":"hemant@autocorp.ai","name":"hemant_autocorp"}],"description":"Load pino logs into OpenSearch","homepage":"https://github.com/hemant6488/pino-opensearch#readme","keywords":["pino","log","fast","opensearch","elastic","search","kibana","elasticsearch","aws","aws-opensearch","pino aws elasticsearch"],"repository":{"type":"git","url":"git+https://github.com/hemant6488/pino-opensearch.git"},"author":{"name":"Hemant Kumar","email":"hemant6488@gmail.com"},"bugs":{"url":"https://github.com/hemant6488/pino-opensearch/issues"},"license":"MIT","readme":"# pino-opensearch&nbsp;&nbsp;[![Build Status](https://github.com/pinojs/pino-elasticsearch/workflows/CI/badge.svg)](https://github.com/pinojs/pino-elasticsearch/actions)&nbsp;[![Coverage Status](https://coveralls.io/repos/github/pinojs/pino-elasticsearch/badge.svg?branch=master)](https://coveralls.io/github/pinojs/pino-elasticsearch?branch=master)\n\nStream [pino](https://github.com/pinojs/pino) logs into [OpenSearch](https://opensearch.org/docs/latest/opensearch/index/).\n\n## Install\n\n```\nnpm install -g pino-opensearch\n```\n\n## Usage\n\n```\n  pino-opensearch\n\n  To send pino logs to AWS OpenSearch:\n\n     cat log | pino-opensearch --node http://localhost:9200\n\n  Flags\n  -h  | --help              Display Help\n  -v  | --version           display Version\n  -n  | --node              the URL where OpenSearch is running\n  -i  | --index             the name of the index to use; default: pino\n                            will replace %{DATE} with the YYYY-MM-DD date\n  -t  | --type              the name of the type to use; default: log\n  -f  | --flush-bytes       the number of bytes for each bulk insert; default: 1000\n  -t  | --flush-interval    time that the helper will wait before flushing; default: 30000\n  -b  | --bulk-size         the number of documents for each bulk insert [DEPRECATED]\n  -l  | --trace-level       trace level for the OpenSearch client, default 'error' (info, debug, trace).\n      | --es-version        specify the major version number of OpenSearch (eg: 5, 6, 7)\n                            (this is needed only if you are using OpenSearch <= 7)\n  -u  | --username          Username to specify with authentication method\n                            (can only be used in tandem with the 'password' flag)\n  -p  | --password          Password to specify with authentication method\n                            (can only be used in tandem with the 'username' flag)\n  -k  | --api-key           Api key for authentication instead of username/password combination\n  -c  | --cloud             Id of the elastic cloud node to connect to\n  -r  | --read-config       the name of config file\n  --rejectUnauthorized      Reject any connection which is not authorized with the list of supplied CAs; default: true\n\n```\n\n### Usage as module\n\n```js\nconst pino = require('pino')\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n\nconst logger = pino({ level: 'info' }, streamToOpenSearch)\n\nlogger.info('hello world')\n// ...\n```\n\n### Using multiple streams (output to Console and OpenSearch)\n\nIf you want to output to multiple streams (transports and console)), the simplest way is to use the `pino-multi-stream` library and register a stream per output.\n\n```js\nconst pinoOpenSearch = require('pino-opensearch');\nconst pinoMultiStream = require('pino-multi-stream').multistream;\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n});\n\nconst pinoOptions = {};\n\nreturn pino(pinoOptions, pinoMultiStream([\n  { stream: process.stdout },\n  { stream: streamToOpenSearch },\n]));\n```\n\nCheckout [pino-multi-stream](https://github.com/pinojs/pino-multi-stream)\n\n### Custom Connection\n\nIf you want to use a custom Connection class for the OpenSearch client, you can pass it as an option when using as a module. See the OpenSearch client docs for [Connection](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/master/advanced-config.html#_connection).\n\n```js\nconst pino = require('pino')\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst Connection = <custom Connection>\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000,\n  Connection\n})\n\nconst logger = pino({ level: 'info' }, streamToOpenSearch)\n\nlogger.info('hello world')\n// ...\n```\n\n### Troubleshooting\n\nAssuming your OpenSearch instance is running and accessible, there are a couple of common problems that will cause indices or events (log entries) to not be created in OpenSearch when using this library.  Developers can get feedback on these problems by listening for events returned by the stream handler.\n\nThe stream handler returned from the default factory function is an [`EventEmitter`](https://nodejs.org/api/events.html#events_class_eventemitter).  Developers can use this interface to debug issues encountered by the `pino-opensearch` library.\n\n```js\nconst pinoOpenSearch = require('pino-opensearch');\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n\nstreamToOpenSearch.on('<event>', (error) => console.log(event));\n```\n\nThe following table lists the events emitted by the stream handler:\n\n| Event | Callback Signature | Description |\n| ----- | ------------------ | ----------- |\n| `unknown` | `(line: string, error: string) => void` | Event received by `pino-opensearch` is unparseable (via `JSON.parse`) |\n| `insertError` | `(error: Error & { document: Record<string, any> }) => void` | The bulk insert request to OpenSearch failed (records dropped). |\n| `insert` | `(stats: Record<string, any>) => void` | Called when an insert was successfully performed |\n| `error` | `(error: Error) => void` | Called when the OpenSearch client fails for some other reason |\n\nThere are a few common problems developers will encounter when initially using this library.  The next section discusses these issues.\n\n**Pino output is not JSON**\n\nAny transform of the stream (like `pino-pretty`) that results in an non-JSON stream output will be ignored (the `unknown` event will be emitted).\n\n```js\nconst pino = require('pino');\nconst pinoOpenSearch = require('pino-opensearch');\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n\nstreamToOpenSearch.on(\n  'unknown',\n  (line, error) =>\n    console.log('Expect to see a lot of these with Pino Pretty turned on.')\n);\n\nconst logger = pino({\n  prettyPrint: true,\n}, streamToOpenSearch)\n```\n\n**Events do not match index schema/mappings**\n\nElasticsearch schema mappings are strict and if events do not match their format, the events will be dropped.  A typical example is if you use the default `pino` format with the `logs-` index in OpenSearch.  The default installation of OpenSearch includes a data pipeline mapped to the `logs-` index prefix.  This is intended to be used by the Beats and Logstash aggregators and requires `@timestamp` and `@message` fields.  The default `pino` setup uses `time` and `msg`.  Attempting to write events to the `logs-` index without mapping/transforming the `pino` schema will result in events being dropped.\n\nDevelopers can troubleshoot insert errors by watching for the `insertError` event.\n\n```js\nstreamToOpenSearch.on(\n  'insertError',\n  (error) => {\n    const documentThatFailed = error.document;\n    console.log(`An error occurred insert document:`, documentThatFailed);\n  }\n);\n```\n\n### ECS support\n\nIf you want to use [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/index.html), you should install [`@elastic/ecs-pino-format`](https://github.com/elastic/ecs-logging-js/tree/master/loggers/pino), as the `ecs` option of this module has been removed.\n\n```js\nconst pino = require('pino')\nconst ecsFormat = require('@elastic/ecs-pino-format')()\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n\nconst logger = pino({ level: 'info',  ...ecsFormat  }, streamToOpenSearch)\n\nlogger.info('hello world')\n// ...\n```\n\nYou can then use [Kibana](https://www.elastic.co/products/kibana) to\nbrowse and visualize your logs.  \n**Note:** This transport works only with OpenSearch version ≥ 5.\n\n#### Dynamic index\n\nIt is possible to customize the index name for every log line just providing a function to the `index` option:\n\n```js\nconst pino = require('pino')\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: function (logTime) {\n    // the logTime is a ISO 8601 formatted string of the log line\n    return `awesome-app-${logTime.substring(5, 10)}`\n  },\n  consistency: 'one',\n  node: 'http://localhost:9200'\n})\n// ...\n```\n\nThe function **must** be sync, doesn't throw and return a string.\n\n#### Datastreams\n\nIndexing to datastreams requires the `op_type` to be set to `create`:\n```js\nconst pino = require('pino')\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: \"type-dataset-namespace\",\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  op_type: 'create'\n})\n// ...\n```\n\n#### Error handling\n```js\nconst pino = require('pino')\nconst ecsFormat = require('@elastic/ecs-pino-format')()\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n\n// Capture errors like unable to connect OpenSearch instance.\nsteamToElastic.on('error', (error) => {\n  console.error('Elasticsearch client error:', error);\n})\n// Capture errors returned from OpenSearch, \"it will be called for everytime a document can't be indexed\".\nstreamToOpenSearch.on('insertError', (error) => {\n  console.error('Elasticsearch server error:', error);\n})\n\nconst logger = pino({ level: 'info',  ...ecsFormat  }, streamToOpenSearch)\n\nlogger.info('hello world')\n```\n\n### Authentication\nIf you need to use basic authentication to connect with the OpenSearch cluster, pass the credentials in the URL:\n```\ncat log | pino-opensearch --node https://user:pwd@localhost:9200\n```\n\nAlternatively you can supply a combination of `username` and `password` OR `api-key`:\n```\ncat log | pino-opensearch --node https://localhost:9200 -u user -p pwd\n```\n```\ncat log | pino-opensearch --node https://localhost:9200 --api-key=base64EncodedKey\n```\n\nElastic cloud option `cloud` is also supported:\n```sh\ncat log | pino-opensearch --cloud=name:bG9jYWxob3N0JGFiY2QkZWZnaA== --api-key=base64EncodedKey\n```\n\nNote: When using the cli, if you pass username/password AND an apiKey the apiKey will take precedence over the username/password combination.\n\nYou can also include the `auth` field in your configuration like so:\n```js\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  auth: {\n    username: 'user',\n    password: 'pwd'\n  },\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n```\n\nAlternatively you can pass an `apiKey` instead:\n```js\nconst pinoOpenSearch = require('pino-opensearch')\n\nconst streamToOpenSearch = pinoOpenSearch({\n  index: 'an-index',\n  consistency: 'one',\n  node: 'http://localhost:9200',\n  cloud: {\n    id: 'name:bG9jYWxob3N0JGFiY2QkZWZnaA=='\n  },\n  auth: {\n    apiKey: 'apikey123'\n  },\n  'es-version': 7,\n  'flush-bytes': 1000\n})\n```\n\nFor a full list of authentication options when using elastic, check out the [authentication configuration docs](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/auth-reference.html)\n\n## Use as a module\n\nuse pino-opensearch as a module is simple, use [pino-multi-stream](https://www.npmjs.com/package/pino-multi-stream) to send log to multi transport, for example:\n\n```js\nconst pinoms = require('pino-multi-stream')\nconst pinoEs = require('pino-opensearch')({\n    host: '192.168.1.220',\n    index: 'zb',\n    port: '9200'\n})\n\nconst logger = pinoms({\n    streams: [\n      {level: 'error', stream: process.stderr}, // an \"error\" level destination stream\n      {level: 'info', stream: process.stdout}, // an \"info\" level destination stream\n      {stream: pinoEs}\n    ]\n  })\n\n\nlogger.info({'msg': {'info': 'info'}})\nlogger.debug('debug')\nlogger.warn('warn')\nlogger.error('error')\n\n```\n\n*** Notice, the `host` and `port` parameters of `pino-opensearch` are required ***\n\n## Setup and Testing\n\nSetting up pino-opensearch is easy, and you can use the bundled\n`docker-compose.yml` file to bring up both\n[Elasticsearch](https://www.elastic.co/products/elasticsearch) and\n[Kibana](https://www.elastic.co/products/kibana).\n\nYou will need [docker](https://www.docker.com/) and\n[docker-compose](https://docs.docker.com/compose/), then in this project\nfolder, launch `docker-compose -f docker-compose-v7.yml up`.\n\nYou can test it by launching `node example | pino-opensearch`, in\nthis project folder. You will need to have `pino-opensearch`\ninstalled globally.\n\n## Acknowledgements\n\nThis project if forked from [pino-elasticsearch](https://github.com/pinojs/pino-elasticsearch).\n\n## License\n\nLicensed under [MIT](./LICENSE).\n","readmeFilename":"README.md"}