{"_id":"@aciesai/azure-kusto-ingest","_rev":"5-d182a1780d27a783cb867d50ca596360","name":"@aciesai/azure-kusto-ingest","dist-tags":{"latest":"2.1.0"},"versions":{"2.1.0":{"name":"@aciesai/azure-kusto-ingest","version":"2.1.0","description":"Acies AI Ltd. variant of the Azure Data Explorer Ingestion SDK","main":"index.js","engines":{"node":">= 8.0.0"},"tags":["azure","kusto","sdk"],"keywords":["node","azure","kusto"],"scripts":{"build":"npm link ../azure-kusto-data && tsc -b","prepublish":"npm run build","example":"npm run build && node example.js","lint":"npm run build && tslint --project tsconfig.json --quiet","test":"npm run build && mocha --require ts-node/register","e2e":"npm run build && mocha --require ts-node/register test/e2eTests/e2eTest.ts","allTests":"npm run build && mocha --timeout 240000 --recursive --require ts-node/register"},"author":"","license":"ISC","dependencies":{"@azure/storage-blob":"12.1.2","@azure/storage-queue":"12.0.5","adal-node":"^0.1.28","azure-kusto-data":"latest","moment":"^2.22.2","request":"^2.88.0","uuid":"^3.3.2","uuid-validate":"0.0.3"},"devDependencies":{"@types/mocha":"^8.2.0","@types/node":"^14.14.13","@types/sinon":"^9.0.9","@types/uuid":"^8.3.0","@types/uuid-validate":"0.0.1","mocha":"^7.2.0","sinon":"^7.2.3","ts-node":"^9.1.1","tslint":"^6.1.3","typescript":"^4.1.3"},"_id":"@aciesai/azure-kusto-ingest@2.1.0","_nodeVersion":"12.14.1","_npmVersion":"6.14.9","dist":{"integrity":"sha512-hlrka5iiwMzQZkkru//ylIPsK98FZ1qI9h7mavTtBw/8lei1IalmnrDdyr28A6LopQtoQKeAF7BIpAtzP9oKuQ==","shasum":"871ee934c5e61d471ac22cff6343a8d4c294a1b8","tarball":"https://registry.npmjs.org/@aciesai/azure-kusto-ingest/-/azure-kusto-ingest-2.1.0.tgz","fileCount":47,"unpackedSize":433528,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf/pXLCRA9TVsSAnZWagAA/qcP/R/r59wmXFjDaOa/wAnj\npnysoOs89qhIpdo8N+A7Vj69ieV9jaEUkpe+KOayEeZeZV3aof1LV9QBYl8I\neOuloVnMxRrNZ6TifzC4oapq2a2Y2DynM2GOlr835PZIRMT1c7bZo4e8LsGc\ndjKmPl6YleRWqUon98pK2yrPSUjoVb1Q+JqD1cY0WeJeuZooq7Fb5ojskvfE\ncvxt5P7zvw5LG7figcCpAhUf1wkgbCDllrGgenY806JisOl2TnIMHxYJfoIr\nzfcsruwgM+I9d9FqeQDmr2CqbIemJg50ALVrEoMGR9Clrz4ejJCyYSV0j2xP\nltLc4LMqKBJW6wgK21PW3N0k/6r2Yl9T3X0H1vx2yOIfUr51bM8vscR7dWZU\nnJFVqC44zjLwF0wWxJYnAHac4l4oUylZXJRdqKtLx2scNGbomCf73NY67Oe4\n0pz39NTllE5bSjZT0CgkNmGknsjjlvKkan+qD+SNa+WBP6lBEo0nzM7uUjC5\n3ZE8YzZCl2xbc6gYZ/me+Y3YXCJJzUeQ4yteP9bpFFegP0Q3iXoYjqEe2dEo\nmTLa3eZxtv7vC8x78bq6pKYpLQ+e1lLPmx0lHE0Jk3NPmVrU/oYfsw7TuBYj\nSnXtQbguNwr7BCjAAcKfmoLHSfyD/qTZnGRD/5Fql86uPChBWW0sTUKGuwJy\ndTYf\r\n=C7TQ\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDTSLrLXbFJ8p7nhqLrpucPgo7szTm6OjF1Y3I6PgjncAiEAt1xP1hU4Htdic+k3Re05/NTr7K5wfCbyCrRczUIwnGY="}]},"_npmUser":{"name":"hyperyn","email":"borjan@acies.ai"},"directories":{},"maintainers":[{"name":"keebler","email":"jonathan@keebler.net"},{"name":"hardik7282","email":"hardik@acies.ai"},{"name":"hyperyn","email":"borjan@acies.ai"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/azure-kusto-ingest_2.1.0_1610520011236_0.07467536427643418"},"_hasShrinkwrap":false}},"time":{"created":"2021-01-13T06:40:11.236Z","2.1.0":"2021-01-13T06:40:11.439Z","modified":"2024-03-06T02:47:50.668Z"},"maintainers":[{"email":"jonathan@keebler.net","name":"keebler"}],"description":"Acies AI Ltd. variant of the Azure Data Explorer Ingestion SDK","keywords":["node","azure","kusto"],"license":"ISC","readme":"# Microsoft Azure Kusto Ingest Library for Node\n\n## Installation\n\n`npm install azure-kusto-ingest`\n\n## Quick Start\n\n```javascript \nconst IngestClient = require(\"azure-kusto-ingest\").IngestClient;\nconst IngestionProps = require(\"azure-kusto-ingest\").IngestionProperties;\nconst KustoConnectionStringBuilder = require(\"azure-kusto-data\").KustoConnectionStringBuilder;\nconst { DataFormat, JsonColumnMapping } = require(\"azure-kusto-ingest\").IngestionPropertiesEnums;\n\nconst kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${cluster}.kusto.windows.net`, appId, appKey, authorityId);\n\nconst ingestionProps = new IngestionProps(\n        { \n            database: \"Database\",\n            table: \"Table\",\n            format: DataFormat.JSON,\n            ingestionMapping: [\n                new JsonColumnMapping(\"TargetColumn1\", \"$.sourceProp1\"),\n                new JsonColumnMapping(\"TargetColumn2\", \"$.sourceProp2\"),\n                new JsonColumnMapping(\"TargetColumn3\", \"$.sourceProp3\")\n            ]\n        }\n);\n\nconst ingestClient = new IngestClient(\n    kcsb,\n    ingestionProps\n);\n    \nconsole.log(\"Ingest from file\");\n\nIngest();\n\nasync function Ingest() {\n    try{\n        await ingestClient.ingestFromFile(\"file.json\", null);\n    }\n    catch(err){\n        console.log(err);\n    }\n    console.log(\"Wait for ingestion status...\");\n    await waitForStatus();\n}\n```\n\n## Authentication\nThere are several authentication methods\n\n### AAD App\nThe are two ways to authenticate is to use app id and key\n\n1. Using app key\n```javascript\nconst kcsb = KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'appid','appkey','authorityId');\n```\n\n1. Using a certificate:\n\n```javascript\nconst kcsb = KustoConnectionStringBuilder.withAadApplicationCertificateAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, 'appid', 'certificate', 'thumbprint', 'authorityId');\n```\n\n\n### Username/Password\n```javascript\nKustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://${clusterName}.kusto.windows.net`,'username','password');\n```\n\nAuthority is optional *when it can be inferred from the domain* ('user@microsoft.com' would make the authority 'microsoft.com'). \nIn any case it is possible to pass the authority id\n```javascript\nKustoConnectionStringBuilder.withAadUserPasswordAuthentication(`https://ingest-${clusterName}.kusto.windows.net`,'username','password','authorityId');\n```\n\n### Device\nUsing this method will write a token to the console, which can be used to authenticate at https://login.microsoftonline.com/common/oauth2/deviceauth and will allow temporary access. \n\n**<!>It is not ment for production purposes<!>**\n\n```javascript\n// will log the DEVICE token and url to use\nKustoConnectionStringBuilder.withAadDeviceAuthentication(`https://${clusterName}.kusto.windows.net`, authId);\n\n// in case you want to do your own thing with the response, you can pass a callback\n// NOTICE: code will still block until device is authenticated\nKustoConnectionStringBuilder.withAadDeviceAuthentication(`https://${clusterName}.kusto.windows.net`, authId, (tokenResponse) => {\n    // your code, for example copy to clipboard or open url in browser\n    console.log(\"Open \" + tokenResponse.verificationUrl + \" and use \" + tokenResponse.userCode + \" code to authorize.\");\n});\n```\n\n## Usage\n\nA Quick Overview is available at https://docs.microsoft.com/en-us/azure/data-explorer/ingest-data-overview\n\nNotice ingestion is done against the ingestion endpoint, which usually include `ingest-` prefix on the cluster name.\n\n### Ingestion Properties\nIngestion Props are instructions for Kusto on how to process the data.\n\nThe easiest way to provide ingestion properties is to set them on the ingestion client like in the sample above.\nIt is also possible to pass them on each ingestion (will merge them with default props).\n\nExample props:\n\n```javascript\nconst ingestionProps = new IngestionProps(\n        \"Database\",\n        \"Table\",\n        DataFormat.JSON,\n        [\n            new JsonColumnMapping(\"TargetColumn1\", \"$.sourceProp1\"),\n            new JsonColumnMapping(\"TargetColumn2\", \"$.sourceProp2\"),\n            new JsonColumnMapping(\"TargetColumn3\", \"$.sourceProp3\")\n        ]\n);\n```\n\n### Ingestion Sources\nThere are several methods of ingesting data into Kusto (Azure Data Explorer) using this library\n\n#### From Stream\n\nThis is useful for cases you already have streams available (http respinse, file stream, etc..)\n\n```javascript\ntry{\n    await ingestClient.ingestFromStream(readable, null);\n}\ncatch(err){\n    console.log(err);\n}\nconsole.log(\"Ingestion from stream DONE\");\n\n\n#### From File\n\nIngesting a file first makes sure it's zipped (if not, it zips it locally) and then send it for ingestion\n\n```javascript\nlet blob = new BlobDescriptor(blobUri, size);\ntry{\n    await ingestClient.ingestFromFile(\"file.json\", null);\n}\ncatch(err){\n    console.log(err);\n}\n```\n\n#### From Azure Storage Blob\n\nProbably the easiest way would be to provide a uri (with [SAS](https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1)).\n\n```javascript\n\nlet blob = new BlobDescriptor(blobUri, size);\ntry{\n    await ingestClient.ingestFromBlob(blob, null);\n}\ncatch(err){\n    console.log(err);\n}\n```\n\n### Ingestion Status\n\nIt is possible to track the status of the ingestion using status queues.\n\nEnabling is done simply but setting the `reportLevel` Ingestion Property to `ReportLevel.FailuresAndSuccesses`\n\nFor Example:\n\n```javascript\nconst IngestClient = require(\"azure-kusto-ingest\").IngestClient;\nconst IngestStatusQueues = require(\"azure-kusto-ingest\").IngestStatusQueues;\nconst IngestionProps = require(\"azure-kusto-ingest\").IngestionProperties;\nconst { ReportLevel, ReportMethod } = require(\"azure-kusto-ingest\").IngestionPropertiesEnums;\nconst KustoConnectionStringBuilder = require(\"azure-kusto-data\").KustoConnectionStringBuilder;\nconst { DataFormat, JsonColumnMapping } = require(\"azure-kusto-ingest\").IngestionPropertiesEnums;\nconst fs = require(\"fs\");\n\n\nconst ingestClient = new IngestClient(\n    KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://ingest-${clusterName}.kusto.windows.net`, appId, appKey, authorityId),\n    new IngestionProps(\n        \"db\",\n        \"table\",\n        DataFormat.JSON,\n        [\n            new JsonColumnMapping(\"Id\", \"$.id\"),\n            new JsonColumnMapping(\"Type\", \"$.type\"),\n            new JsonColumnMapping(\"Value\", \"$.type\"),\n        ],\n        null,\n        null,\n        null,\n        null,\n        null,\n        null,      \n        ReportLevel.FailuresAndSuccesses,\n        ReportMethod.Queue)\n);\n\nconst statusQueues = new IngestStatusQueues(ingestClient);\n\nasync function waitForStatus() {\n    while (await statusQueues.failure.isEmpty() && await statusQueues.success.isEmpty()) {\n        await new Promise((resolve) => { setTimeout(resolve, 1000); });\n    }\n\n    const successes = statusQueues.success.pop();\n    for (let success of successes) {\n        console.log(JSON.stringify(success));\n    }\n\n    const failures = statusQueues.failure.pop()\n    for (let failure of failures) {\n        console.log(JSON.stringify(failure));\n    }\n}\n\nasync function ingestFromFile() {\n    try{\n        await ingestClient.ingestFromFile(\"file.json\", null);\n    }\n    catch(err){\n        console.log(err);\n    }\n    console.log(\"Wait for ingestion status...\");\n    await waitForStatus();\n}\n\n","readmeFilename":"README.md"}