{"_id":"ul-imports","_rev":"53-5d0ea18421f998418b8be999f29b80da","name":"ul-imports","description":"The scripts used to import data into the Unified Listing.","dist-tags":{"latest":"1.0.0-dev.20170123T141639Z.9e14821","dev":"1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin"},"versions":{"1.0.0-dev.20170123T141639Z.9e14821":{"name":"ul-imports","version":"1.0.0-dev.20170123T141639Z.9e14821","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js"},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"deep-equal":"1.0.1","gpii-express":"1.0.1-dev.20161219T161942Z.50ed093","gpii-express-user":"1.0.0-dev.20161219T192728Z.df3de96","gpii-launcher":"1.0.0-dev.20170113T104654Z.903ad25","gpii-pouchdb-lucene":"0.1.0-dev.20161219T181232Z.aab549f","gpii-sort":"1.0.0-dev.20160630T152357Z.3508dc6","infusion":"2.0.0-dev.20161219T170555Z.5778f7e","mkdirp":"0.5.1","universal":"github:gpii/universal#88aec55cd2cafb2cab914978b11c5e1bc7926260"},"devDependencies":{"eslint":"3.10.2","eslint-config-fluid":"1.0.0","fluid-grunt-eslint":"18.1.2","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.3.0","node-jqunit":"1.1.4","request":"2.79.0","ul-api":"1.0.0-dev.20170123T141254Z.2801782"},"gitHead":"9e14821a6b2c8b8118cdfde7e4a1629618bce713","_id":"ul-imports@1.0.0-dev.20170123T141639Z.9e14821","_shasum":"d6d9725c00e440611bfad74924ea76d092191349","_from":".","_npmVersion":"3.10.10","_nodeVersion":"6.9.1","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"shasum":"d6d9725c00e440611bfad74924ea76d092191349","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20170123T141639Z.9e14821.tgz","npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJh2+3SCRA9TVsSAnZWagAApaoP/3Rm6JM9UWeRHKMvjaTs\n/RPUSpPaLjKWF5f9UBCsu/4li45+FvRSVuD5ljhJESaWr9LGtPJUD96iEAsc\nOjWvponBXdvOQJat5o0BmMY6TtzMvFkAWcQoJDpSzZ5VlbAmZi4FsXQOsc4w\nr9PuO9D8C0VXjDlvWu7A4/9BL5Jta/R3xB4dbjGjFA2KM4/xYaj2wT2lN9ZB\nMQ4qLT0SBjP4rDSDdylvMNLvTxt6bRmKQdnomoFsws7DXcOUPDH2OGbgLxbP\noZTLeovexQXPInzIE9b1tN1WRrQzsYaZOTRyvSinifKkCXGqRPa2pWrisBKt\nn160wc/ORWyAmXp4U1E4zgPU+t1Vc+FjRDxO/TGzgkth8C5Mu2UEtdMnYqiX\nkK/ezyaRh5pqSnt3Ox0grrXK7cgThnUjfkIu+5qD3PJfS30RfP9ViqLMBbSq\ni+wLBwcnaa1RdkACqsWC8fvj4Yp/uXusVUkYe2a3+9YU4XFkqSv0kOK3x12V\nJ2wfzBPu7KFEi3nXYLRMaPfsFDEDjDnqpot2usD3R+BiSQ7GzhllJmzRW59k\n70MzkDhE6bMzAOKiSAXmhuWbpTyems40QQs1kvCw25GsBHgFhavg7Psiug/n\n+Gq/y5oNo07fBrYUS3zqwPO6opoR0+Zl7OstMUTGfqdFzOpNupCtUwDB126+\n/KDy\r\n=8PDY\r\n-----END PGP SIGNATURE-----\r\n","integrity":"sha512-aV93UYCFI7m2R0nQFdrT0c9xQlsR9evtU+8q3O2IXErrFOAm4RZ39vyBEHXVqI1lt2B9E8trt2p1V0EuNgwuaQ==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDFjlAAdpkewc2UnnSbFIDxxV7Qlz54Bo0IQzc2JDEQ1wIgZOUfpnZJIk47Vo4W+ncgVemxOpJjDBlnBIBXuJ34yjo="}]},"maintainers":[{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/ul-imports-1.0.0-dev.20170123T141639Z.9e14821.tgz_1485181013970_0.5647641094401479"}},"1.0.0-dev.20181025T111840Z.7b93e2f":{"name":"ul-imports","version":"1.0.0-dev.20181025T111840Z.7b93e2f","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: `docker-compose run ul-imports <command>`."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","request":"2.81.0","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"7b93e2f9bc0b383d9a1aba16d00eb8a4b177301c","_id":"ul-imports@1.0.0-dev.20181025T111840Z.7b93e2f","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-8ME+Bal3lfZoUvkTNKebtXFuySqWlTNR0lDoyk+057mVYOZODh/1Q8Y+DolQL1EBZ8Jnk/L6ykM1lpM5Jwy8FQ==","shasum":"c664e77fa08282637169b807db9eafbb54a36cfc","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181025T111840Z.7b93e2f.tgz","fileCount":215,"unpackedSize":4885788,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb2aJrCRA9TVsSAnZWagAAhGoQAJJnw6d80rDMurU8uaGd\nllSzoxexFxYCbkcns6L8P9oY7VTLu0Nmv/odPc+z9QOBZuKu6tdWfJ/BZZta\nosZGcswyjgj2XH19VFne91NeOVXADYkk47CpPO/50XrXemd3lDiPo4UCSWzh\nP61bU7sh+bSzpNYPyfuUb5vtyg8BTDLSS/KzZYJiH48igIrL7UYimM8ZLX63\n7eoeFGemPm8SvR2yFWn9RAPGk17W9PWEGnyrJEnByD6C4/cEKF/EqbX5ue1d\nxHa0TMShlbxEevJYtjzkWTSE/H8LKu/g7cZtrVGSyKB9ZGgu5ZqTqC0pJC89\nRqBqkKo8ZEKc7R72vgnlalsoI7BAp89GKdrEYm69t538dH60j9W8nhcyvR/q\nVhO0u89t0uxGDmLh0b41MvyIL8Gfa4ksc7Wvx/sN6gd20h7iA529n/E/iZKf\nQJbihwoAG2wt+jVMBPPB+BQwcrdLh5OB75/yr4lbKMW/jvuk4yK7K/0sr+kB\nKJitTlxhwO7KOp7S097NXTQwzvylUvTCpEVF93YaILdUnn/7eF8W9m8q9dxQ\n4BeWzaDwzUR82Tthmem8aEhqmkDcrXjo+pXFzLsIwH+sjBPf3E2hBKtu+DXE\nWMwgmpgqlOIhROnd3Cw4q4pItWVBXvkcWX60A9RdrL7Z7Qac2RHE8BeZj2gC\n85tZ\r\n=on97\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHXuSt+yiLLaFVuR79uDekboW3vOVYYBXThVC02NgFP0AiA6mpZDvlgs4rAPSZ5PosuWV4rsFMM0zkyYCnaY3EXW+g=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181025T111840Z.7b93e2f_1540989546935_0.06116958702624231"},"_hasShrinkwrap":false},"1.0.0-dev.20181101T100429Z.bd8ed6b":{"name":"ul-imports","version":"1.0.0-dev.20181101T100429Z.bd8ed6b","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: `docker-compose run ul-imports <command>`."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"bd8ed6bb434b67574d6500b7b54138f64c1e2896","_id":"ul-imports@1.0.0-dev.20181101T100429Z.bd8ed6b","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-fIY6z0X00FBHfcRyTCm+OJOOMz7L7mB3xG0+JzQ6o16r4Zov8KPvyUgYrUfRLEJY1zeNabmfc7BqPihMZ5trLw==","shasum":"b8c736bb74ba1c93f558de15ed6c1ff9c0e9b771","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181101T100429Z.bd8ed6b.tgz","fileCount":215,"unpackedSize":4885960,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb2s/FCRA9TVsSAnZWagAAEJAQAItMjrRog/gbq8iNS5hV\nsvV2D5kn3beDkyFQLy/pGyM94qjjhNIOgcm5adMejHaH7myqyuZbh5a29MmT\nU4ctXj+xw2H/eSRufReNG9De0lbuoQUMUosppzGUBWXTfeLn/GSkPFWLp4hs\nlDGOW+SOVYstMnUc57ZefHxS2KZCTafht1jtpfYE+Ml5Zb5YZPuxifA10esd\nK9jonT4X5OOWzNAhw3abd7z0ubJI1EWH0cDOJCkabQdpwjUQ4/JkqaJOVCey\nf31DeHAET0RU4bh3AQXBnB+lvmd+mgVTfVxycMoj95KtM3yH2yN2Gh15MvoF\nS0qsYW+WwrYyiv4z/iTePKJu680Drr94F4VHH3ulxhTXla4vqLtU5PqO/3uQ\nAz5U2fmtHVQCcb3O0S/ByGMnAx5zTSekyTXCFdRbnseKjCtJkMF4WNUlF5bs\n7zGx73rUec3pKYDnjE8Z//DWFdYM/yAISs2V/aDPwnBkdEt9X+R53YA0IpZn\nFp/YxXxZXAuDdsxEv4Sg4d5dbHNtUkquxQy15cFDkYV5oABcXE2mBotvaV76\nBMO9CVRhZFHVgYPJv6YmrHlTwMmgthvqgykwZo05EtIv+yzMyl3XTYeA5qit\nERxwHhnu8sL8yC2LYcMNND1pOv9am4ivY0dSZoBlVz/nS3nXN55QRDjW4ztJ\nUsOp\r\n=a9F5\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIBYRMPH+JHQH46RP8Epvwt3zjW8W0fSqp4N3HR3TYDOMAiBVS901HDH+TGABEqAKyc/UU5MDiHwoY7an9ffEeizQ/A=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181101T100429Z.bd8ed6b_1541066693009_0.38600707717159977"},"_hasShrinkwrap":false},"1.0.0-dev.20181107T083400Z.2ed57ad":{"name":"ul-imports","version":"1.0.0-dev.20181107T083400Z.2ed57ad","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"2ed57ada2b942a4dedf656938a7df97af8aa4f05","_id":"ul-imports@1.0.0-dev.20181107T083400Z.2ed57ad","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-zGMGMaWT9W6HbuXL1nhKvb+UbRaHiofPkylLYoqJEfAHC5mG94iL/y5KmEmJXsL2brOvBQXWyzoXkozAJAEuCA==","shasum":"981e23d5597c5edff3b474c77747c9d89ae0aef6","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181107T083400Z.2ed57ad.tgz","fileCount":215,"unpackedSize":4887315,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb4qONCRA9TVsSAnZWagAAkekP/RfZI34acxN6OoWyQjxK\nFzb9xAamyEW2Wt1JjM2i5X4a5tv+HWMehb6RCvfZas/ZwFTM7Cq4vdQ+btXq\nnxKaBadE+ky9XKIOkBpvL37Nl2nvlU/tSGiJcPotnFuZGMj4cc8a2jXQKKY3\ngndcFuOovzXBMA9gGR67C4762Ckh5lPiLLYaSJScTAvdJZgbfKyJtIX/5QQh\ncg0FfGSzOJ6ugaVSsPdT7tf+x3mrurgfFUUYUKNEotPWvIwu5LgOB/C5hBdP\nj3MwbgNQlzgzJAek3KhQIMKQLaDtKKVZBQC99xjzv3fMFXdKWrWq8H2KETcm\n3ZmeX78Isefw6+paJNtDsdwgzugmP/FJjcpGiteE0H0CdiNDbhuRYYy4/wsT\nhepOWJtBq9gVwoFP7fcZre6Se/H/dDcKmHRUvQPd6PmfdyA1vsse4Mw62QLW\nZ3QnwF6bx2bnpmPsmTAPKb5PzuB1VJTGo3bkDFv+B1hwJ+AiFUBGVG+Hc9/h\noVMhiOIqCtSt4P1LQ8JQ2jDO4tcfpvQpmyEFYyX1xcyic6hOxzQdBHtjqx9y\n1rw+wthnR98SW9AL8Qnvm+OfgoslFZj+vAiWGzU4um6+de5OCINDtNf9rCG+\nXVAQBfQ/eXYBam263MQ+YjfZvgMPPKY9BNbWu0x1abbhJ4aXDhfRpUCruwu0\nBces\r\n=jN1/\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIDo1Ar5gai2jU7WL7RBNQPzJlDezGWp4RTIqnBupHpXwAiB16NvgZI058XN/m627P6Rz47HDRpo7HWitNPO28A3KUg=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181107T083400Z.2ed57ad_1541579660731_0.8896983533228326"},"_hasShrinkwrap":false},"1.0.0-dev.20181112T151503Z.1e997a2":{"name":"ul-imports","version":"1.0.0-dev.20181112T151503Z.1e997a2","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"1e997a233ce985e82f7fc012ca666611da1ccd69","_id":"ul-imports@1.0.0-dev.20181112T151503Z.1e997a2","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-/DUdFg2JAYK6FvXIWaCUK00yXJrsMGOEZm2QxJTejFEVWYZF/qfBpHiLM1yXwNbvVta4Zb5P6GUHM7iN3U/yNA==","shasum":"31c9db8d6a0a55af20aec39958bfdbd8c6773d1f","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181112T151503Z.1e997a2.tgz","fileCount":215,"unpackedSize":4888641,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb6ZkQCRA9TVsSAnZWagAATAoP/2849dysTPQAB5OuAaAJ\n85pGPnLJNR1pxhJ0vLWQrquHWahfiK1DNP0Ok7Coq3zXW3o1bvvj2smHtJ6f\ns180fKWS1su5h6NXv28QVG6O8LMmvs6+YNFIN0pNrTYEHDGyBDHoIgMzwrQB\nbcE2q41H2L5SR+01JVKbKbjHsl+iHtpe9QEy4moB6m+scS2+4QE7wJwc+uXg\nJwbYPsG072gnG8p1gyfMNcMHp91D70wlfsw/73e4mkuOGPcXTNceTyY+rXv0\nmvmgwSKKr2SijZ+VLp3mq7UdIP/Y+USyix2QcxIhn8eWcLUQSaFne7nfn4+m\n5oDFnEBSpwYZVlRSBYSTol6uDR4yVICnnOcqIpVrKuR46FXUy+YuQYctgIsj\nMHaOOs3lSULCBsaD+bt3VkIdIk+DWz2mtJ57lWzVfwTDss1uMnLdue1WBWqz\ne4LcE7xM72lQPFW1bOLq2YKxAZpg9b3/l3LVWz1bX7miTgVEE/jWW/65dj6X\nEHQzchUy6cD+KZpLo4DUqR1lH2Xu6Hkq/ob4f8Qf2foRz68PW/WkWgiUjAgy\nT0ZiOXSUMrmEyPjNlEoggHre/RSoZ9l/EIez50xn1gWJayG54Bw/Q3tpJwCj\nUv12SAEAjuBDGQFlg5rbS8rRWykwQprjRDlwTeeT3X3Yc0K7asEJu3NUHuPO\n6820\r\n=1ZQF\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCkmjSyjgfSYAocauWun/k17Y3P4hLfHMdO7zxUWg1D1AIgdZ9550ygbhPIQIrVcpJnqdqUPLjmyeiMwBsoRJA6tSQ="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181112T151503Z.1e997a2_1542035727810_0.22503925335027386"},"_hasShrinkwrap":false},"1.0.0-dev.20181114T133929Z.c780f41":{"name":"ul-imports","version":"1.0.0-dev.20181114T133929Z.c780f41","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"c780f417e029a413331a3f107644f435936a0dd9","_id":"ul-imports@1.0.0-dev.20181114T133929Z.c780f41","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-2UMCLsj9nKDCMt1nF2FtTfwWX+3GqkGtdPpe0iZ+YvzAPBLylkyUq1wTLD9NLXwZrYTV7HZRx51053PFBa/45Q==","shasum":"9b274e0e9b423f708b9c959749d58f9f258aa957","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181114T133929Z.c780f41.tgz","fileCount":215,"unpackedSize":4888256,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7CWlCRA9TVsSAnZWagAA/x0P/Ro2cV6YjYBTGVIK5d4d\njFLGr/mkknZC7FUwEolb8YnepF6Rpc7LAfbo9sPM7Bvnd9Kg7ozTduxYWews\nyvryAwYv1Xxy9KVl2Dd0SU+E/aBaMVCwoamZs7uaKvSjnEb8fdOOi5lrU3C+\niE6Gk6N0k6h9z5TJUe/CWFKhQ27I+86CsNRtMcuenaCVCaY9EDcel9JKY99B\nfnGffv07DalENWipQnZEBJyOAh94cSdbJvE1+YCL6ebFx500pCljPhIc/0If\ngw+xGkV2AbWt97nRIqMauffraT953uiIOBLCieoucyRYhhq9qKy9xyZY2Zs6\n6ieHlpwr3DhWf+n1HxMTvJLrY+DvCbCD9JDwcTUzz5qkoe1IR6JUHEVGl6An\ns92gHdQhG40YnJcx0fhFDLn5J9kj34YIP6adl5lSYQ7g8NCIE9LsVgdxknPj\neT2qmg4Ft2MkBaSo7/2kh98x/6gowofFBulbO94KduuNSaBYhxpVLOAX4imA\nxbvrh+fs40UMsYSXtkeJOwpXg4dt9LLXLkAygu43MWqD01n6gf7ZLCfU3Aox\nGJSHGE2oESsxpWEtaBTe072ypG6/f+3za1ZVH6aAqO1axIsqQSyVsnUB5eNH\nHcP4G4HfuTqdl7ks/ih6Szf3cAfabuAPiE/DHPkuZTyK6jmvFnVY+JgAeWF0\nvVua\r\n=chIH\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCTNROSjXLKnYdQbtHdKvh2sx08+JCP0DHOii1FbqaNAgIhAJ+82kcHG54/bQJaZeE/2DFyMxAgxbQF3QMmW2kbWuXR"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181114T133929Z.c780f41_1542202788678_0.4558897507652362"},"_hasShrinkwrap":false},"1.0.0-dev.20181114T141610Z.52a303e":{"name":"ul-imports","version":"1.0.0-dev.20181114T141610Z.52a303e","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"52a303ed3754bf02bdc6d586620543a118f388b4","_id":"ul-imports@1.0.0-dev.20181114T141610Z.52a303e","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-MV6rENKdyNTsPGrv8Ph9wMuV9IV9ZLIuqIxW+4NI/UU3yJdQyqR8HEntYyY6JYBzFbkE66IaiBC03QDwy8qTiQ==","shasum":"be02d2c05759fe7ab9b16d8de174138ce534f2f7","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181114T141610Z.52a303e.tgz","fileCount":215,"unpackedSize":4889998,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7C47CRA9TVsSAnZWagAAKIMP/02KTHxaGpKWzO0svKg2\nY2sL6XASwgAKq/O047r/Hhfms4/tZaKqiASwpI9NTppAkZF279Op/DoAINib\nGCnKy89RLA3Q9lrdsvjxPt+hAzw9iw1VapbwG48CCVU0dSseGGugeisIcyzb\nAL/UqArEAG1x9TpWyu2FJ7+/SicAiSbLw87BBcerHHaldTL+s+zaYD+UiFK6\ni/HJ6O8Yq9Ek9N/jbUzc4a+xzjHC+eD1FWCCyoEKyDjof6g7z0rUkzbnoty0\nCyUXCCHbING6Rr3e1z/+wokr4h9YfYlDu/eJbfomsbPt0/b3she5e6CvRt7P\nm7ihgnRQZv2jfY4hUzNNBxufo0hJofC17XP2klAuaT/ADR3qkhWT2A1QIv6Y\nX2wN8gcget+++Fd89TqLKEQJfWVZW516DovfRxQMb7/XU5TsxT2/ercTYywR\npogBIrmKnqaCtGO6Zq+GJ7T0LL566OD4S7an7sBSRZC6HPjm83wj/FDL5s7D\n2p4HU2q/3H0PCXEAtkd8x41U6ZlnuKbX108xo/ejxDlUKedcljwV4KBKj/h3\nnSTxs4PWZ+XQcnQI6IPKBWU/Q8YnLehMq/gZONsuHlYl64oDTF3se/Sraeq6\n8Bumu5nzACstU9KTLFmGGwVFhdxHlUtmyDrvMhq7moYI4080vOsyexOrGkxf\nG4FS\r\n=EYWg\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDT9g5oCwo7hMPJps1+VRT+q7iCrA9pdUh0EZQKHcGQTAIhAON39rMFwHJtMUYkoIqsPsRbDNV3a5AAo6RFERxHgRVy"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181114T141610Z.52a303e_1542204986816_0.26294539152999086"},"_hasShrinkwrap":false},"1.0.0-dev.20181114T145001Z.b60c93a":{"name":"ul-imports","version":"1.0.0-dev.20181114T145001Z.b60c93a","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"b60c93abb2cd34e6c0ff0b0f5205ad90afff6b68","_id":"ul-imports@1.0.0-dev.20181114T145001Z.b60c93a","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-1vcRsDMcHlIFcW72MfJqyLM4C/La21JZei4EDz5qVR19tynLIg+ehr++bAGd1QXyxeKnVydOh5X2kCLvvcr1SA==","shasum":"3d0993094c2a38319a556656bf4b802c6b37f29f","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181114T145001Z.b60c93a.tgz","fileCount":215,"unpackedSize":4889756,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7DZCCRA9TVsSAnZWagAASfYP/3CoN/whr+jq4sqHYIdL\nOcRJvkBg/B/nn20GEl1IBsPeVYenSJU8SZPVloLC/Bytc/wEM3FpOOOLrkzM\nLsYzHJZQ55z4ivxlbCiPY5fO9guwhl41K0X374TdA6ahaukAH17BfLXGnba/\nSfkITSNaoYasgBH2Bgb9xI2LaF53VZKBrHVYh9ljVtV72aQbsvZndwgJzHvG\nfzHV5JZm583MT1s9VZubL/XToi37SfIQCfpg/CP1uhPqB5GAO17M5x197WdV\nrLAkxl9Pv7oJEf+qiecO+5xq9lkRj84IGIto3Q0dN6Fw+6rx/StFIEH08b9E\nRJvOck3RX6+Jc39patevkoVzNqH6Aqq9+V7re4WZzwlJ5xrvWufw+RlZ8Tot\niUqJEgOFDSJxX1rJ0wr+uDUdt3AgFkPvj/C/dkd3YElOIv836I7qvCBG2Css\n5JFuBNR+O7UliEfxQ9e+sHlp3VQTOQC/uYH1tNV7dSm9mQ7WrbO33CRcChxT\nS/yGNjxsfTvZopukAknnsPLQFVLs+eORAnm+FkRr4tQBdK5KcyxlyURvm3+T\newcJ45rns85NeoBDYvQUtMfDfYDrknZ485uaz18iw4JeqaBjG2jrTG+qabck\nJKrzcnkRk9DcHRqyShvdUqYVqdIfE3Q08zDAuww3PoxBq626jcW4ZTVHNm2B\nAsAG\r\n=RQMh\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCWkehFIo6s6qpfXpCfrVRturF8cXxojziCq8f5BA4mPgIgHIcozNFwBTzHrUuRwPKuPzLxDn8utqMKEVgAVsapDmQ="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181114T145001Z.b60c93a_1542207041978_0.7181394448097775"},"_hasShrinkwrap":false},"1.0.0-dev.20181114T145550Z.9d7148a":{"name":"ul-imports","version":"1.0.0-dev.20181114T145550Z.9d7148a","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"9d7148afc7223588c1dfa9560afb1b95aaacb958","_id":"ul-imports@1.0.0-dev.20181114T145550Z.9d7148a","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-2g+5YYlPf3qvDv0vkG3dE66Cq8afq3DNcRNRi2rU0F+s1vgJCGtpVBPCwnpiDXYILfNkadT/BOezF1TViiVwwA==","shasum":"231de11645b96da86b9951255510c1f69e128cd9","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181114T145550Z.9d7148a.tgz","fileCount":209,"unpackedSize":514163,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7DeECRA9TVsSAnZWagAAtzAP/2Ok3oqTh7lvoi/eBMt9\nQrwuswlOg7kqq482nycdvCDjUt8xTLVfQu9vkNVMWKtX+/sbJXmUbiCa06h8\n+v+WDjZ4CN6R8o1AWXoigMP/eVeQQQ42aK/Y37HXiRSz8lRt6S+gMw77rmln\nch2pBmAcVKweqB/LASLfnBNGfWAih/9mEOf8GKwQWQtNQmdEaTowfWd/PZyd\nLrxs02PjU2t33w/Dqvd8CI3uQ695K7mzCZKAN6bjUlxyehOvk5aWveOUDKj4\ntI0lTbwO8odwE/HGoETZG4V1NjrwIHK4yRNG/kWuGqH16Rt7Y2GnmaCyRvwe\nL1zbhvCf7aYi/RgabHBabSdJFYzmw5BAtmMi3AJUDvlXyc7PcDtaeCqFH686\n5cPwPy3+S8PzAuqG4j6vWQ7Of6wGtlJNnVtGr5hkuaZY4WEe8lCMz7CAoXPA\nb8XJKdm/r4p6PEHOVxxw/CkVMXgLVVqjY76hU6SrdKtkWZqmIocighUz8TfR\n5N3zfkX7mk9r7b7E3iuteLAO1iAwQt4WvUUk10ZkD6XrBnOxhhnSSum1ulZ/\nM46jRiMfKhmUXTktGQPez3G7bzbUqboCPht9IkAEnLy3KQ6sybIJ25CLFBEU\naTe49X3z19TAfEaMV8UmbVnVoes4aIUJ9FGbhQsTf3RPrAJ1QoMQld1kDF14\nVEAZ\r\n=WvQh\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHRVyemWrzFdD4RjTjXTSfWEbPCcN1w0D1NECCJJ+gw8AiEAoTkSTiBkrVzlV8JavOp6XOXMYB1rMJxbA02GOILUvfs="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181114T145550Z.9d7148a_1542207363591_0.2405972075142888"},"_hasShrinkwrap":false},"1.0.0-dev.20181114T145712Z.f60fc66":{"name":"ul-imports","version":"1.0.0-dev.20181114T145712Z.f60fc66","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"f60fc66e56ff861195c352e7ca306d767be148b0","_id":"ul-imports@1.0.0-dev.20181114T145712Z.f60fc66","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-3tL73L0+uP+bvwIJ467/zSFYwt3Aqj/Su0tQ4lmJaNi9UddTB60sHVQddkDiHz/Nsb2ijYsytbyL8GNyn1TQsw==","shasum":"9ff1e8065e787210b14c3fe9c345f5eb4f3abe16","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181114T145712Z.f60fc66.tgz","fileCount":188,"unpackedSize":351928,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7DhrCRA9TVsSAnZWagAAofgP/0bmPH1SlWdCRexlqFU7\n1sMevbHcnz8ICBIwdMtdcDfwaF4qmGTN22lrTchrseVC2lQQa3GviWgYjPwR\nEKKnphpOFr/L5ogL7y6fFENJ5WLTimeU+isJQPqThWvU8GYP0qiWC0+8DFEH\nZAk7FYvnYA+pOb8NTPrJXtz43D2ogzlNJjj+ocB9ipKPEU30ZolhIKe4Pmc3\n/m5TGz618uR6aHVaArmDzgqHacBtpr1MD97VteKSVplw9OMaZcFDUI9uBdZ7\n7rAX+WvP2PYB4uaS2VhphS4cQMj5eacJ0E/gFZv/YUMVDnvxBfP0U72pLGL7\n2fE9pWN9JCzbFbnVvN7wRtV4h76gc522xGn7tl18EW0hoK/cgO76YzVnmC8R\nMSjODNt6f+cju5fZTRaEMzAvckpDBo8Vp8a6vXh/ZeE0DZuj8yV5HUfaSw7x\nVKvtRPHgrWZXKDYQ3dANV/rMUTX/PDU6wVWtoWhuO4MdtZQoKE93HVaCe3tI\ne7Q/meOs5uhsd4ZAMnp1WHml17Pd66yDYLKHyMyL3FoBBsvxbdADJGi+Ttmf\nXoqYLlj/vgS21ygqDyLTWyfkU2uPEHKv4lxkixGzI5Sy6CUffYYxZPujlY4K\nd2YpR5V4+pE4ZxE7zWltzcye4YuMJtEyhc6knfAyGyoW5Npl6rcp3/SkPefT\nY+nb\r\n=8mdA\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQC6/ZqKpKnBUWMx7jH2NPWOikIRaxmJEsiZhgCEKkQixQIgDMN5eGqKOvbtFi6BG5ZX++Uyr2LpEAl86c7RSvFxSwg="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181114T145712Z.f60fc66_1542207594950_0.37055905116800125"},"_hasShrinkwrap":false},"1.0.0-dev.20181115T091929Z.759364e":{"name":"ul-imports","version":"1.0.0-dev.20181115T091929Z.759364e","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181024T125123Z.afea43d"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"759364eabe83d0bc455d614d61fa4619f74800c4","_id":"ul-imports@1.0.0-dev.20181115T091929Z.759364e","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-yujRnTb7T+zBgA12T+qtUesO/16c/6O0GQE9KUo8WSIjKXc1j5huMCey4S1wqoQCG3jt8Hm8bENpRbvIC87hwQ==","shasum":"254a880695f9ad16b583cc4dadfdcaf8c263bd18","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181115T091929Z.759364e.tgz","fileCount":188,"unpackedSize":352462,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJb7TqPCRA9TVsSAnZWagAAf4cP/33/EpnnpC7000aQJqUk\nYkqRVs5hT1he4+0Pl6K6Mkcl9QfqZalIIpvmUPb4HqqoiOkEsJvn90WVKZPk\nA+VIf6em48iT27ssa0TYoW09e1+8TyjPHI3pN8zmCsUfcO1ciJqXPhqBCIlP\nEnhEYq+CJC382kUOotYBK7U0WD7aQxg5usXztdAUTepj7AgwNOXMJKY8adie\n36JMyHTiZTJ+20U5R7XEDvS8tTU9WwBhGa4TOBT/gLpJpPHT6kqt2veFVXax\n1HvJu+7b8K+4tkL1/BnizAGLI9eGBJcrargPTETeXpvppBts2q9cljUDls/N\nC0Q8aGqUakuL+X5x90s0TCoQgYkkEVNOfyHgY8rGZWVKGzNK07IBlC0JRL2G\nt9I713GboTTMGUDfTpV20d6yhLwSgqBXWMc4DKQvjNkuhdc4CWbZ6EjlV3kZ\nndz1HsapaeKcGTezsb5407BY9wtrqR6PIPOjnmVf36wpZ2LE3wU2cYTNU8Le\n20jv+hcUsD3jeYmK9wgq6bFT61Fs23YzZQg/AtGFx1NmftwlrW5mT6l67xyv\nObq+3Jnr3y+Fpxt6GaJ4sFu9tG2T0OHtb6YmccbHZg9JAi2UfAOQWHoyWxgd\nI3NYSCOMMTG80HIcONLd/xtrIJ4kMswc8NiijWirQ1TsZgKyt/PohHctohGR\noiZ8\r\n=Y8Ck\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCCQiU/7fno8Gy37xV+SzbcFkWH1rxrfIlSgyEtj78xiwIhAPpksompYo83IJJ6cD7tKUEUH8jeXa2L1snohxSeQMcc"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181115T091929Z.759364e_1542273678768_0.826725329676665"},"_hasShrinkwrap":false},"1.0.0-dev.20181205T105745Z.accef71":{"name":"ul-imports","version":"1.0.0-dev.20181205T105745Z.accef71","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7","rimraf":"2.6.1"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"accef71f5c8e13505e0bd30daaa5b934c9868956","_id":"ul-imports@1.0.0-dev.20181205T105745Z.accef71","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-8X8uLLuagnhaACHIv+9CCs/LqAd391Z7iVoVbqgg/N/kez3vvBahLEXVD2CZS6PSDiIgizl472Un7wdD7hlh3g==","shasum":"33850024e06994c3720e946700a6e199e08ca105","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181205T105745Z.accef71.tgz","fileCount":188,"unpackedSize":353947,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcB68+CRA9TVsSAnZWagAAMjgP/jtx3nx5oDSVjBIm7AJ/\nVd3rxZiJecUNllrnDhP24DY1onnkzYmctEEFygLiObYB8i3PrEy7zivjdlzH\nmpm1NOCgq3csCufvpw0S5UGYY3mAnZT5qhOk9BfYlSqsBX/KgUvVuGjR7YaA\nAWlGMyHiKBsb6T8iNsmsW1I4eASjfLaUtr9BZY2E1aNFj4oEu5s1RydY+2sK\n1LpHQPM6CmPIld2BxZryLDPYXE0pMN6FGss6Qj/xANnKHBypMB+hreS+IXNE\nxaRHFCMco+ot8sVsPmD6lNwH+jMj8TgAldvPmdPlgjprVHixd35qwX4G5832\nbQLJrmUeiGGgzp/EVa7wIL1ObQfHsyn5tG+51a6ekusmQsjMliidPDFhRTaq\nAEQWr6ydhRQ3vRKYJ9NkG+sPWfYc4UKyGkvkwMQw/XQyksSg3E63W4+ALtLn\n3d1jWbhd2izCSxNOzOuVRVhlBUyUVG2hwEulGw52b6EkDXR9m8kHEvgTJb3a\nixc0HIxG9qlP7se2EpUww83KNsQr3Wecd273tn7gCn7bGdgTMb8i3C/expRw\nzXyICEgl5SWcqAIRi2cT7UPMVvAgy1NSznmyKHNII5Gg0798jBqUzhCzJaEn\n47j0MNLe+zoPWMHKTqmhobE5KKwbg3Yfqf2HpTXbonCMbseKt1ME+bDCkuMo\nxcVe\r\n=AtkL\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDKCC3nVGNvO47FAX0n0j+wARwk184kZxKvPEQ5iPlpJAIhANF7RL1W4JsLjikXf0DzKyt0k23PWurpexjGgWco741R"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181205T105745Z.accef71_1544007485336_0.0866838144902975"},"_hasShrinkwrap":false},"1.0.0-dev.20181205T115420Z.5af93eb":{"name":"ul-imports","version":"1.0.0-dev.20181205T115420Z.5af93eb","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","rimraf":"2.6.1","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"5af93eb4cb41ed1d041305148e439a0535d564c8","_id":"ul-imports@1.0.0-dev.20181205T115420Z.5af93eb","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-fnLRzvPEOB66sUfo/Z+MkRHYISvasxlspL5RkkEwkco3OLzfDi92R92rh3zQUjCcAwpm7MIkEPbDwYt9BCcCvw==","shasum":"10f49b560522e5414eea8ae2ff69bbe51a522a22","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181205T115420Z.5af93eb.tgz","fileCount":188,"unpackedSize":353947,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcB7yMCRA9TVsSAnZWagAAQSEP/1Qu9QQcTcR6WAqwlkHv\n4qMPmmtXmpxlTYWFXr4qb7nlGhQb1PSbgEPQq8JSw1YwNSLPSeWzy4R0zWi5\nZNWDy2HoElJTQnyjhrncBjkagixtRVXl73m17bhk4Ae6ObYTMBs0L97Rq4rv\nHSrbAgxoYrVjWvbLkbsvMZYetEH5smsGTIJIhpM/T/Hr2q5NfvSjZGHmSMI9\nlbB3nFGrwFvfe1knnUAdBX6jqLS3pkgwnZ/vjiFeNy+0DTzOtWpWo3zTBGlK\nC3pA1rS1MqGcc5+o9AhxHDzcDJfLYBMJOFTGumeBgqy+wxPHEZidbRFWFpb2\nrB6R92X/JcnB0h4yibNcQIXH37Wazr84bH6/yI5jcRIE+cWuGRawE75+bXo9\nbpNLDu38YylbSA2gxCW9jQzuvd05s0hrF6FdrML2whZyfBdQTSQI0i1j3rgb\nkGDCX83Z7fN3NPeN9h8waLlSj5SnVZWd+IKkooAZWcGrUrmLInNNisAgEeuW\n1+4AsbooA6miIob3yweuX41mIxWS1x7S5/t5/FtRk6QRidmqad2iGJqkI22F\n9G8id6a0uUDIyh9+r5SwEBTEiiUMOSAy/VY6a6x4LdizFlI655XjbTk8P7tW\nfsDJrsBYHjMrE++QgVYW8VVDekOqcbcFOijMsyGfAHKkrejIh5PJon/7Bi0G\nMv3a\r\n=RoXv\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC0ncJ3caH3uICmrPOGDfbK4fokDkYNymSgrU4gP9HgDgIhAM2UINSulImy8zESFsaDWGpzdWFnuEs6FhWFjQ/1MgCp"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181205T115420Z.5af93eb_1544010891394_0.173985232611215"},"_hasShrinkwrap":false},"1.0.0-dev.20181205T131852Z.396b0a9":{"name":"ul-imports","version":"1.0.0-dev.20181205T131852Z.396b0a9","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","rimraf":"2.6.1","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"396b0a977aaeab074c0e018b276243d19f24b65d","_id":"ul-imports@1.0.0-dev.20181205T131852Z.396b0a9","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-u6yWT6WRvH3YDUlZtSSPo/bJSfpUFD6L1z1ImvnoAtuz8Yex4TIgOqGbV5qk1HG1unMnchxtSgIj0qSZXanY5A==","shasum":"147eff708a885ad67b85dfd30a85fdc4e5240a62","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181205T131852Z.396b0a9.tgz","fileCount":188,"unpackedSize":354198,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcB9BWCRA9TVsSAnZWagAA1ZcP/RV3DV8SZquVO073zaX+\nbkGU51YP3EYbLOnTUl/Ne2oF/jIef1W3lF9w9Psrg8NRe3LxctpLtnW7HiOI\ncoIGaFyH8/OLvWQG+lDfJOUt/q0Txd5HSeHZRlWydspBp+9UvP1beZILJld/\ncikL41nkZ9QUtdbXXwsWUK4rYN8emubN+7553W3svd7Oaj+hb+8nE7t4YCKF\nKoArotg3EKDjIicmaNqtBc9rwbaYgJyOmOPauTC8EZnSAoGHcVI1VqrIvcPp\nszOvf92U/XaBJMzR7VLfMSibkXZYO+6SXVHvQm6SgQfXEUcWjjZOCNZ3kyXf\nSaNjwvoleaFkFcHchKaEivR0IaMk2xRGESKD4cOqy3Yzn4GQQd44PZsZpW73\nvaYRneiRfAG2mK/j73qADFsJSak2M4XO7hQfU234doo9C7ViF8iFHAusr2LB\np5DcWE26iY87ElI1igcj3Bqdiw38p/2YV8/AAkTaCegtTUoZe3DUHRXb/HAJ\nu/n56maFtuGfkT1CkIJtCs6ekKW5vqPlEeN8Y9yjIU+70MyyaUtAZ4UhL01v\nK7kwjp0FqkfOkhjTNib8klOCeLipbaYZL4lu3F2K34pdl0HkwmFBgDWN3rU3\nYZ1N4BacT+op8zKlylJjQAe8v0iO81V51E4UtLOZPUoeuTWdpmrkHvGchwN5\nKKOf\r\n=yJkA\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC7mPXFzhYGLM1macXBd8ANRJxfOg2Xa3L5uO1LLtcwaAIhAPJp+MJkTJrfBK3omM7xIPppQ1+sbE5XxDGify81X/qN"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181205T131852Z.396b0a9_1544015958260_0.825324690328683"},"_hasShrinkwrap":false},"1.0.0-dev.20181205T132454Z.ad7f2ad":{"name":"ul-imports","version":"1.0.0-dev.20181205T132454Z.ad7f2ad","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-image-import && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata && npm run sai-curation-delete","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","rimraf":"2.6.1","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"ad7f2ad7e3287bc5d7e9d7890f5d4528a278428c","_id":"ul-imports@1.0.0-dev.20181205T132454Z.ad7f2ad","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-4XJvn7kcBdVKgGXw2kX57/1Kr1leQ2lMJIqIfTfI6MEIeact1T8lgSyiAzk3tRUpQsgO71AFGYRdnL6hmqMgcQ==","shasum":"7742205b5ce8e9dbcc982e6f1f00db51d09c9e88","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20181205T132454Z.ad7f2ad.tgz","fileCount":188,"unpackedSize":354282,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcB9G3CRA9TVsSAnZWagAACbgP/2VIUGRX6IVxWVj+d7ys\n5kXujecD5CXHAj5VGrs0E5RLozN1y3pc3nFUFkVm3dw9mhDD1souuwc2EIAt\n9WNfk13cFo3ffvhJ4OA+Q9zYNGvUhJ+pVbsNJN6bb+36rkOe8cYdAF20PY7d\n3Aw9R83vUU2VuAV2Z3Ny7u9kAfzVY+HvCfbNJdLtJOYNZqdsndZDMwAAEGoj\nO9qbaFMc002+HF3Nd9yIpf4X9SOP13EQa73LWSbg8umK77jKk7cdfcwpFzPo\npgn9v/TZTtnrX/vzHfGnify1UDRnQzyl5abnQ4prZ4JjjBt6RdRCuZbW7/lE\nlZvE0FDoLYoPXvXGYmJGqbDzDwvirrcl6Hsr9fIzZQ1g5ahBvXUhyGSOPkBh\nA9hgLyVDgtc/2hOHoPlIwsRadZxnowzj8QlFt0vQ56iREPf+4K+TE0lm2mFe\nhDsXhY9tTFZt3rrI06D6qXHBBlkWGtu7MuITEj3jT/q91NbS2tYmvXsZmddI\ndnJ64GP3WttFq+m6//iovAvRIB+/slWFKGIDWNtucWJqR1nv86enG2auAiNT\neJxyKaEnemUeznrW1sQmMbwcv5do9SGekf7UXIrIUT32M61Fxtta5WqAwUrG\nulwcdOafBR+942TzZpOrmYHsMiyOqMSBUMPfVs6nKAGJiHwws5N3aEvPmh5m\nXoFC\r\n=009k\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDzSyvZQsmzl8HwWJugo56io+oNEtvVwWrr2vpBsWne7gIgEBVxdFtfMSejOfncTPSnC9LxsC0CJaUH2KetpLST+fc="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20181205T132454Z.ad7f2ad_1544016310317_0.9148859883446483"},"_hasShrinkwrap":false},"1.0.0-dev.20190227T132007Z.eda3df9.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190227T132007Z.eda3df9.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run unifier && npm run all-image-imports && npm run sai-curation","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js --commit","curation-invalid-records":"node src/js/curation/invalid-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.2.0","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.9","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20171121T212609Z.9b4fde781","json5":"1.0.1","juice":"4.2.2","markdown-it":"8.4.0","md5-file":"3.2.2","mkdirp":"0.5.1","nodemailer":"4.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.6","request":"2.81.0","rimraf":"2.6.1","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"4.6.1","eslint-config-fluid":"1.1.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.0-dev.20170420T094715Z.667e69a","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.1","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.7.0","node-jqunit":"1.1.7"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"eda3df9f9f50a6a0b555d0255a2c229727b7a60f","_id":"ul-imports@1.0.0-dev.20190227T132007Z.eda3df9.UL-67","_npmVersion":"6.4.1","_nodeVersion":"8.12.0","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-OttPXnmwLYxhRCnJvpeutcufbjSKHnP9JlKuvsWwzP4b9K2zEWV+ZYrNgGpGm12WoVmGC6ElbUwDXzJ8T7I2tQ==","shasum":"5b770193ca8394f8311bba61a0426d3c8df2b539","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190227T132007Z.eda3df9.UL-67.tgz","fileCount":193,"unpackedSize":440533,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcdo6UCRA9TVsSAnZWagAAoIEP/jFe5C2zukUDo/mRFsO4\n+vvFsQ1XDl3FxNp9aUy7sG+0E07cqx2tqFhNaANpuicNMlLF81YcKpIRpBC7\nJvU8foCX4GTn/nWsPavf41Xos34GUk9rPogA4RHQAxTuqZUmr++woSqFmkRq\nbySS6T8Cr2KLRl4nXm+BnHqOmDXk8cBHsAtjGOcdoC/ZOQAqny6I4lXvBb8e\nhLm4UbsOqBZxqSB9tI+gnvwd4oMK24FlvkJ1KOKyG1rFWtbyMEZ7ylD62qbq\nVO9z3NUFZ2frWDig2SIHChJdU+MaeToHtz9cfsNueYhVSOdNLUC65DBq5iRT\nOOtS6I1dKPB6m1RLvyA90fIdlj82O/9inqOnKtkGkkBDcRQq13hvRFEU+gw4\njGRERQ7FIcCXiUe7rVr89ehnd9zB9dcPUcBTA8cPFF7hQhL2NSzSoGLxgxyw\ni+wdGBzwumPLs+mx4fuZw5fmbuZHobvVWlPlnxNiqOWPt+b0tc7czZqAvUEi\niP5MHQUbW/td57aW2oKGEy32XtQqvoPNJ+nVIOIjV56Nol0wGXiZ2IALtuTv\nY+q6muBXCWD/qcurDbbwvnZMsS8+jJan9XGb3Xd06y/JH4vydd/zSTPioYzQ\n3sfxkBLsgRhLQA5ka0pi8KySEjefQ+G1RfYHVYUWGO2N3pjT1r/61L2B31yU\nw/FY\r\n=RCT6\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIFy3S4YZzPItI3zy9KBRQ/bWveJMpODil3yKHWszj8MWAiEAk3fgjmfu2lhp2H1bsgJeiS0PixYVR/rNHzO4Dp93Q3I="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190227T132007Z.eda3df9.UL-67_1551273619551_0.15974440790985178"},"_hasShrinkwrap":false},"1.0.0-dev.20190411T112616Z.51ab84e.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190411T112616Z.51ab84e.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","kettle":"1.10.1","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"51ab84e8d8345d4d871b2ae07c86fcef5c73a9d5","_id":"ul-imports@1.0.0-dev.20190411T112616Z.51ab84e.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-+nJWY5SzxFaeeKVJAEU5fjNnet5yqSLlITFyfQ2jiXgqzy3ot2Oqi1jO0kn02TEh1m6Mg+GuX/xX3vt8omr4sA==","shasum":"8962831a41c7a95a5d048ab8a9f2276be523ba73","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190411T112616Z.51ab84e.UL-67.tgz","fileCount":200,"unpackedSize":467019,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcryRpCRA9TVsSAnZWagAAY4IP/RWFMYa5iphkhI1YIbG2\ny7FKK5PesDC0Rzv4NVilJLfUGWVaES8Q5JPpe+gO3xJkskqpCx+BCy9EW5V1\nivQsjnm6hhCg9Y2GGUbDsjPYyAsSxTjweghXpxbiM396AuzSKecE0KMb1DZr\nksnkfeAM41u8umg/K7ur1xfkJ8dqLg22AkacgOyXz38bk00EjcKskP4+dKgu\nV1FlEGSTQIMFEZErz1t9FYimXhC42WMzek9c5SLjK1qzOcUNLbRNY6nh4sE0\ns8+fqUt/URymmsMRN8hYFsXaqwTLytVjgvHvb957jWn9vkC9HhWkO3/+vDbv\nivYxRBrStzcRIar7RFQB3l9e52aGQdJ6mLmX3b0FLdeY3k3ubRMkG+j5z5tX\nwsrQyuk0HQOvQDcIWXgQ6l1DCrjYsk0J4LQ0knGZv4gOqJcmoNgdKR7w+AV1\n6qVXwoC65gwIdc8dom8Wj0aNx4mSlIKV0bL7wvRJx3irE59ddIYKDlRif7zv\n5F98f008tizFYTz0HnRJtyT0B8qe2xawxfNfn7Yik+mJK3X7k3KUsAWyBTak\nObpE5fLK9qKBR+Dxp5YrHOHrxGNafZFtNivJXYYkBNGJolkphhHz5Pxqknuh\nyCLbKmakEqOUCYZejVHl5T0KHc4QIsGfGiF4FdbLGngl96KqkGl98IOyQLJX\n6ncZ\r\n=Zmd9\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCgFKLDQVAmiH3UYbkaiVcmVj99aC4G/wpfyAGxYHsv+QIgSeOHU63f4rAFrGmLjnSwj9MY32Veds2Zl11Ot2tpn+M="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190411T112616Z.51ab84e.UL-67_1554981992835_0.4219789798146256"},"_hasShrinkwrap":false},"1.0.0-dev.20190411T114749Z.8f82d90.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190411T114749Z.8f82d90.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"8f82d907e5c661be9d5be4edf9b950742d812186","_id":"ul-imports@1.0.0-dev.20190411T114749Z.8f82d90.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-yE7ty39mZsnjs73umB6tPr/3Q5aQ+0ggruY9+a5qah9aBojncVVbinm26oey463GubOPnUrG7Q3udoW+fztRwQ==","shasum":"462af633c1775f5566ccd1b8e1e7adebe540e3d5","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190411T114749Z.8f82d90.UL-67.tgz","fileCount":200,"unpackedSize":467019,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcryl8CRA9TVsSAnZWagAAjgwP/3CJVP1GD/uiKG6t1idg\n6egaqgxMOpGatUiRoMOZE8Vwtp495L6tPAnuWrr1IFomzu61ojEI3ytKLyDY\nkq8bRRgUXD8J4dLwOo8H5OvBsriMUEL8rAnMd8ybpZw6gHazSLaJCXXPUEug\n+3iXAcIlxjXrDVNLWkc+PA7RKzHWNj1aHrXZETluCOWIK6AwkDbvoIBuazpY\nWz82vJ57OUbGAg6x8lEBzmU0kyd9QrwUqi3KZeBy1bLA97TpbaiTduqffmXQ\nmW6SycqveVWbj/T90/mSoTCGbcupHg/XqAi702QDY2H8rRsGXN2XDxtprTfq\nEsCjxEh1Xg7w0mVL1x94jyecULB8WQZpP2vP3XHDHjVQWD9L2i04GoBArtRZ\nkLDO3fVJUILkvQNX4I6k1aybbkksucSqEOJcWYZuokkfSbyVW4sw+GY1g4Ij\nhqdmuo3pQPaB4npqyyd+5Znx0mxN1H3NbJlfFrfOUo58laI+xVbq4GJkYBBx\nAKYMUEoJFFtC87PpUhqdAfP6ij6Qe0GEfzY+IzUOzg1uxB3rnpdINqYOr9+D\nDvue7JwmA1W5b51lBNAyWC/qb9w/u2sCJNGzm/zqfAS2M4O60EwFwpJ8EJT4\ntDP6/5y9QHJ24CT35SGA4HxJmw9RWHDN05JeO9PoLNfec+w9+zN47RNvdBWW\nBulA\r\n=XA/o\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCYo0IYgoO353kymXkxPGpt58SU6uvq/tNQp4K3nvx0kwIgEAHjkpqh3MixCaUJVn9gOhPEK+6842+eeIPsbgpTRsk="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190411T114749Z.8f82d90.UL-67_1554983292301_0.7232345954215857"},"_hasShrinkwrap":false},"1.0.0-dev.20190416T091249Z.f6879eb.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190416T091249Z.f6879eb.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"f6879eb121a6c19f3387ba30bfd95c839ca816d6","_id":"ul-imports@1.0.0-dev.20190416T091249Z.f6879eb.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-vmwMnd5c20PSLFvaECmY7DIGNf5ASkqdse3orKEQp8bESiCkVpjeNdFHJrjCwwSjg+hgiHpzoKMcTtj0Q8buHg==","shasum":"a6c1a48b1f87638aee5b01f51ac5f0d6ec958043","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190416T091249Z.f6879eb.UL-67.tgz","fileCount":200,"unpackedSize":467397,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJctZyhCRA9TVsSAnZWagAA8CkP/1k+OahTsUnaXDiGnObm\npzD/oCzrmEp/QS9p1m6irnZR5dKJDGbR+JnlvU6ErHD8b/9eakWfdo0KpWuv\n1BDyzOwLDWh9e2KH3TjbHOu9+ScVDRpc/7FirQFZ7PAc0KmIMMYEArQnsC7K\nw0DZ8x22O533onBDVDMxn6lElDsRSCByJ0xMVquE3IX21/NBiBTp+83LSHtg\ngh8LV9D09aXhocEhddfJyol1NTd3Q+k6B52M9DPp1C0TdaC+XfhwLusqrONQ\n3aHlKJ/phexa5+FMF2HOIo2mpmnEWT/ofx9Fa8SOE29QJIUux+rwclNErBjE\nuv3pvd9V+/+jUeDYb5CUq11OEOTh+JUQx7SBZ/8rqzhqfLHbsAFxbOLwQdCG\noMQP09Gvsf80blah1vXIhOBqB946F6ruL99WMAxItVPUfCKFWtgmn36QR7z/\np4W/04CZ0iuvCuHNVvmXa+hpOFu/kJsTFI55YasBJtbfv622QEzLuU2822EY\n3dtK1zqh/D49tRRJdwNRaEwudSz1lBJmVHYKNEs+DSHaLkbXbi1Bdu5rj5rY\n9+CfR5rNdojNjjX2Kn6JfNv2UH9rTgxlpfUM5W4WF98GJgst5hQrisHa1imB\nYbV4pnrviloGpwlsuwKRlqcW/EtkZJ8+5jCxeeOxazDPQuhbqXIAdlrHGzm9\np7sU\r\n=uqmt\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDUnWixazENXrTuFglET91Yko5YIANXn7NGawa/h2XjFAiEA9tv/ABb+/kqfIFm54WJEbFpubryhfeCUVg73opDesUU="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190416T091249Z.f6879eb.UL-67_1555405985053_0.6995424376220045"},"_hasShrinkwrap":false},"1.0.0-dev.20190501T145554Z.f3921d3.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190501T145554Z.f3921d3.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"f3921d387840da2a48496e7ec8542680976e12b6","_id":"ul-imports@1.0.0-dev.20190501T145554Z.f3921d3.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-CsfzqTkccUVbdlxU5KG41/L7BOVvtHb6XKhhsmhqI9PjKtkZ+EwqEGr3ac5uNeNfuo9YEVpfe5HfuONcO8M/+g==","shasum":"87976bbb31ea163adb78876aebace9918373d687","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190501T145554Z.f3921d3.UL-67.tgz","fileCount":204,"unpackedSize":472432,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJcybQECRA9TVsSAnZWagAAEyAP/ij50GOv5RCV0EHztfHd\n6DW6RWxfmSKSpoO6dFQwjjCtNNXuRm5R5983mTKYJLwqFuv5Yb6iVQ/O9baY\nC0irICVuvmkV3OJsxpnZKEBYXXKssq+btwSiiHcbLGk/Zsj+xwy6qcsrlLDz\ngQ5QnPCpv3Ml6r6xDliwIeryxRj+j19C56Xw7k3Exin9sF93m6pXETNN8HK1\ncOK436PLQUqDIDiPMcF/z5dV6jz82OJDSdshHGuzbOJ+ufv1tOP41wB0hdxq\n6KPzeu+Sfu9EGyHWVd4SheUjGJxo0Wo6iMQQe4Jy+jZ60R/Gz4gwjUCeRWYB\npqUPLO1rfMx20A/T1aeYdxPAn9FflibRI4z4YisUC7OsWiaciIjT2k8oVWQd\n2J1xA2EXmBTKSLNNxo7u3z7nhNIcog2ySEVHq6vco1vt32bZ77RyThR2DVWk\nDpejvZvdTdI8mXU5euODZeP5JiifJilwROwnPV012u3wkAr6ezUCVM6ylmJy\n6hFqfIqs2uP2A6zu+9u+gvBGqR0zf88hh+dTmziMcmsMHGXHv+DsYdBQUwoS\nifldWleFzeX7bPc6hafvR9pNE3xUjKjOp/fVd335sKaaFnPg9ZGddko4Av+c\nttwEyFTZydLPzxLy94CtwRelAQwIYxG2KKtdj6ExiPjn+K64YY6/N6tReAn5\nX5zm\r\n=4fXV\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICNSsUc+cQqr3VTjNNpW0Fc86xzK4pxxHpvsVJgUhYJpAiAWkuJ2oQs4qNLbrxuRDlSjI1ayQWmQW/P8NuwujgGT1g=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190501T145554Z.f3921d3.UL-67_1556722691176_0.19876947128311118"},"_hasShrinkwrap":false},"1.0.0-dev.20190516T120548Z.8a18ee4.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190516T120548Z.8a18ee4.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"8a18ee4710c2bc1337ad12cfd476881166a8344e","_id":"ul-imports@1.0.0-dev.20190516T120548Z.8a18ee4.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-GMuFZLHDzbF88HkkkvYJmwNcy3Ryv7WEGlIPu3WSGeJyv7AnexGzc9rJzPJJCC+1h27cGoFxhDIctVtB8J4QjQ==","shasum":"c8cfa04bec8434bf6d987a8c308f041c25e13a16","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190516T120548Z.8a18ee4.UL-67.tgz","fileCount":204,"unpackedSize":473177,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJc3VIqCRA9TVsSAnZWagAAb/YP/0p+17Oyc6Qzmu1hzo64\ngDVkfEZAxfrWqUPTt2dQKh5oXcv8ip1XuoGEmwmU5daOadWppoFgqHrFXn9j\n8PIJAgOB5ZaMAW8pSOo8Z/ycN5gj1+qMHiHeMAlXKvnJP79/2JXaJGaU6DBY\np3/i3/FcDq9XFC+ubrckqOkBa57eo0s7iYLhPTZqM+bRRgal+0M3UKEmvpFf\nLaJHIJh9Ls1Mk0PEvSeI20+eBeEd41KUiYz5tI+U8pG3LHrIt0MTl+sE8x4l\nSH2K49z2tmtAPMvyBExy2yoXBhP9/yKTxnu1gZ6qTaTYcyiAxT7fQpvz6T0b\n4qoaPwA2xY4bEfZNgWsCIsDeM86zb7nqxlaPA4pKsWhB8SrxBWbimn0Wk+1C\nARGeyPK9SFN4+y/JiHETytxBFd/GKTsCPNklQoNltnzcIGCW5JJ33TIRGLYt\nFOIqrYlaVyPq33wjOnoCiDfSRTcxeV7B+Evq/meXrIShsoqAy2EWUxIrMR1r\nMQJhPth6c8m9nBFloFGdJKc7WfQBwlBC8jSobiZ2HmPfoBHRtKA5wRVLRE1y\ntWUGwaPkMONwSc2BmQdEP9d+K9/Fa9t4r1WSCdD8k9uor8U7rxHGsKp5GlGb\n4Lgm7fMEtg+oqaRz8SYheVGeVZvu/5IIgeyRrASWCRh1igXv0v8RfsLNvqtA\nqNMD\r\n=3mnT\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIA1kyGTGGiPyeDYdw7TfXcAgaympE1t1uLLaQKnjHkPiAiAfUfVf0BdCkRZLLlfauRacagT5M9hnpCFs3UMSGg+png=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190516T120548Z.8a18ee4.UL-67_1558008361769_0.04790237316351642"},"_hasShrinkwrap":false},"1.0.0-dev.20190522T134629Z.39fae0a.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190522T134629Z.39fae0a.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"39fae0a5f56bfc7e0c1c0f04a68648463c39ab2c","_id":"ul-imports@1.0.0-dev.20190522T134629Z.39fae0a.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-YqosIplMnJLgAJLLSVi5p6dfFrGl8Pcyoa0H63AUKwJXXgjGazKx0Qe25MTgXhM1PF6ITtf8G77D0J9aPqe/Sg==","shasum":"7ea76169c91a0fd8163af19f98624ea9bea66fa9","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190522T134629Z.39fae0a.UL-67.tgz","fileCount":204,"unpackedSize":473576,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJc5VMnCRA9TVsSAnZWagAAqW4QAIyunJ/9kHtE//Dwgali\nk3YQOaptJtag85kn1lMXWcEa6/qkVrH8SK7d4XAmw/9tqK6USDyraftY+Khz\nt85HIoKUrDGWplwwDfIWOqwW/LWYjVulLxGwdy4iaZmAEGfRgrynIgKtOFUv\nt701ezacazA+VT7oCrgA9xN5D+hdIANJ0P7nYwC1xVsWqt3Q4ezxzv2Uzpfo\nlEfhBK4im5xC98b1T4bTEgld9fim7gf/tl6Xh/FL6V7lJOOaimMRCH+sF6ED\nluzS3s5r2xN0X3oL5Lity7YFjMZHEaTMbkTxT+vz2WRFjx6k0qO/zo9Ywe3w\ni3002/x+TELaxSrRvzadY5Y+v/iMuAUnhM2AGZt+rn6tmGRMRsJFMrY+ow0D\n0+AfvKl5AkHiS9Phfd8NKxVpCN91ihThPXhEwwCAs1x8OSMk74OuWR+pf7Yp\nI0vU2orIgP5tL5lLrjC2CH9TuFizET5IBaGgKo/HetZgkyFkk4AhwpiBHyVj\nXmLOCmHszhCyyJfeS/6hE7Zt/atwWPwn1qvh1scIz2wtj4dYpVvozZ0ctkEa\n6KKXIGukqyyIT+FT8Jjomd6YjFt9suWbIyKz0DjyEJcDqXe4GAs9PZIWNjMy\nNtR8EV9FFTY4dQFvK4tU4Po8LrrYVvFqMUfDgG00s40bedFINRRnfZ7W2tOB\nardt\r\n=1im2\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCM4eCWT3AQggoevBzYicBr64ulU75OJSxAHjCjyK9SAgIhANiQe/uJk97bayzIhdFNmPKAPZJ+KGNTeh6mo18jKFys"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190522T134629Z.39fae0a.UL-67_1558532903043_0.8428282415784512"},"_hasShrinkwrap":false},"1.0.0-dev.20190531T085455Z.6b1da4e.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190531T085455Z.6b1da4e.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"6b1da4e6e96fbc3eaa70e2cff8d741e1f510e46a","_id":"ul-imports@1.0.0-dev.20190531T085455Z.6b1da4e.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-fp5AWlPW0ne1xE0ol4XAm0LxrRGgMzr8tOsLmZQUy4IOkX/wffmaIvHGJ4+lKTJrTT83YhihIeHwjtkYID8iew==","shasum":"4767027a730ced24e60ae35aa17cdbf97c9d6e1b","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190531T085455Z.6b1da4e.UL-67.tgz","fileCount":206,"unpackedSize":507414,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJc8Ov0CRA9TVsSAnZWagAAOvoP+we1NZi568ZX2ApeHieY\n3f/bfVBDqbE3lZps8r5uINxwDmOAjH3u+RiHBsdU2lZCMENsObWYVx1pVIeN\npys9MJ0qbBjRQ25WJAvuK5WqKbM+w5luC3BSoOZ2uOAMKAa/MipidCQmgYpa\nwlnUDu+Cccr8dUtx9SEX566KcX8HOk9EDQHiNuheJQtAT8bCRFhLVoDje5PQ\nQL9ac/eIUdUEiG2BXeUYDkFIyiz+S6MLIbBdu+MSlufo97OJhuVI9MiHsFpo\nlYHN7BSaG+Gqx/IeSIhvdEQPFuLSWvrc7+pdTyUrc6XWoGJkqdnGNFTU8aTd\nf371eS3aEyFufb8kITTisbYf47wGCEZfKytr1mgU5fOu3iAK1jJXvgLKktLY\npj1TYrZ+BW14DSkg1+wy5JNa526fyT2yhf1ycIYaEJ23Zr5iAFkjT8fFugHZ\nZSUnGr2c3SEDMRJ1qFiJbYzPLX9AbCGa7+Ai4tx3CKzI4CQwWUYDe215ptb0\nl0YT3UJ83WltWE9RkQH+3YyEj7mNdLTy/JI7TeOrPgGvWIaykKCREB3ywCf+\nvR4DasUnQa1myku1wwkY8JJWrho64UR+IKEU6nHBFsZIxZ7zkLk2VjzA4Qgj\nPSb7Ur0SDCXXCJY4+PAtdHuarEJyRn4jCekFQIMb5w3gSgdBbTxB2qFhzQan\nKvw9\r\n=SnSq\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHPxSaPHvSGnxR1kIDAlsvGtdHSxwm8YHHhLvcjZZKqCAiEA8iAnk+8h7klRgwpwk5yE8a6kCjy966VnzsAXDMZ/gMg="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190531T085455Z.6b1da4e.UL-67_1559292915588_0.2708681543754774"},"_hasShrinkwrap":false},"1.0.0-dev.20190604T121938Z.8fb8951.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190604T121938Z.8fb8951.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"8fb8951119a5401844a6f73e011e6e5405da3cb0","_id":"ul-imports@1.0.0-dev.20190604T121938Z.8fb8951.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-XLfsbF0xVq6SZk1rssEURNSuEaBVX3X04xv2QNKVkrXxfBc/MPHYfDVpENsqZO7IdS38p7j4Cl/1VgyAR7QInQ==","shasum":"de0769e39e59e9bafa8fe917ad1043f312c184cc","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190604T121938Z.8fb8951.UL-67.tgz","fileCount":205,"unpackedSize":460497,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJc9mH4CRA9TVsSAnZWagAAOSgQAIcN7wHLf1M97KnewVP8\nfxfSfLq4cg8VmGNpyDCCGSHBGEM6H6ApZO49mD9N/THTjY+YbFmNvPb+AXNR\n/Hx+5MF2TNGzFCygFUY/6SPhKCCZTTdNwT6meoRiSwlv8nJ/3rmc2xV3hMwS\nPS/zOEXKnyFzOipCXhO+tAbBP94866E6u/xXediVn8YCfmC62p3a0NwxeWZO\n+yGp9g5vTudcrIVNkqeu0EJTBtTn7CJmvTHgbqKxS7XGYih9pMyBu4f+tt8l\na0fU/kXt6uOjAcnoxaVSHRdkSu1NoCsCvFsGvK61iMZEhXihOPgYBHS4vUCV\nGNL37+fF2XjLb/JmaTF5nJRLCKRUW7oAK/1r30ZyEPFfFtwxdO5btHczluBi\nZ/9uEwo2yOwj6n4F0qJHlfq6+tiF0p6lr8RQaLH8v7wuRlaeaXcKt0s+ydWK\n0xm+0dw1i0p7bD9Pim1qKiFtui376oibIuOkijucTWtW5Ti+6ryT5uLvYJSh\nZWlM+dPvY8txlA3MwxlWx9fzrEBzKnbnoStwQNn7cv2LsjpJSZIgT5Kh9FBf\nQ9b4Btr06Y6DCtF2A3WztEKlPdgn5aa+su26ikGzLEYsZ5zLMIJHLhS3RM5P\nrB1m9dwinhr21yixML5mDXHOxJyw9A/UwQ42NVgbHjNy4DK/UBRTiBVEYG3h\neYUg\r\n=xbri\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIDTK+ut6y1GSP1DU7E7MC5lbILKJBdMvc/SROkyC0NQfAiEAzhVnsAlNq2kDgov8xX15IZD5LdXtSBuabhfai17DzwY="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190604T121938Z.8fb8951.UL-67_1559650806731_0.8913095446778538"},"_hasShrinkwrap":false},"1.0.0-dev.20190605T082432Z.51be4ca.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190605T082432Z.51be4ca.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"51be4ca1544999921ff0ee04d3360271c35f0cd3","_id":"ul-imports@1.0.0-dev.20190605T082432Z.51be4ca.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-+Gm/lkx1Y2VPAfhmxgYrv+WhvgAOyD41og8EuM+2NOl1aJhwB1bepG7u5VkBREXtmZeKZIuZedY521R4mE6qOQ==","shasum":"df4f17575b05c7c9884a98e9a7c26c4801cb79a8","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190605T082432Z.51be4ca.UL-67.tgz","fileCount":205,"unpackedSize":461385,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJc93xSCRA9TVsSAnZWagAAThwP/353j9qqqoJxsclAmbfJ\nrBk9kUAQODNEGAAMGKYs2/NaDhivZG6Z4vlSQuZAncUPhO0HrTmMJvERGqZJ\nkA6iqiR6XGxF03bu2CDpbEmT8roexeUf+4xAG93CW738bH45Y+RnHK6vERiB\nA2IVwxWjOdMNuk5GkpVhPPGg+5g7h/YwbVZTBGV1xNUWX6Ijx7BFj0PNEE1b\nDLeI+3YhPYRoFOQ/S2OBVunZ4G3kVH/+B4m1xX1u0MaVK8OKOqbUQ7jfn9CB\ne0O0xTTMBgw1aNN3j7P6tOUPlyVvlSOK/5SGQpn7JlI5lCJmEh7vO9/58G1F\nq6ohojNoHP+OAMfi7XIOpr0ATt5a5574khyswC5i8xlHEITauTbLe4IXkl/K\nqW6VcXcuHXXXC6A5nb1wkOxzh3IXtibd7We69yAVxyoOfguy4ElsrxiDKNM+\naJFCmUQqioQTYBD6hbagjxCAMjXp0HIyr7lEoMewP34cd72Dxz1fYZcFYtXQ\nQNylhjGpMs5Dvgut7+cjNY+p2kUZYulbKSlRqhB9uHXyHYabzkFIv8UNRGLD\nJrd51mDcTp+J/G30KMUFBb1BV2KJB+ECJrhwNQdlxlzju6PjBw+8W6i1dqm+\n8tVsNabz2nkLE0rH2SuWBLI818KhbLRPKJHGB7BiipbcX7VvUihmFmmVVC/D\n/zNc\r\n=0NKq\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDPzLLYlcmptDI2LosNjAGp4k4a59VwILqtFXz2tjMw4wIhAJYih9CJIr7bFx5QGQtSgnCcJTTwNAUhvXx8CtfEeQ0Y"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190605T082432Z.51be4ca.UL-67_1559723089605_0.6316558989407823"},"_hasShrinkwrap":false},"1.0.0-dev.20190612T092837Z.06181ce.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190612T092837Z.06181ce.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"06181ce13c37f110c5965094d28f4ce821219f86","_id":"ul-imports@1.0.0-dev.20190612T092837Z.06181ce.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-nt2xKwV9YwTlD6uY0PF1tmXp2Qd0AdV2TZGtOSgS3xG1ooFzFKpni318EXYajRCLfwGuGwMNat4UnaWkTb91oQ==","shasum":"a2932ccbb16cb571495952d234fb4f1375bd8f3a","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190612T092837Z.06181ce.UL-67.tgz","fileCount":205,"unpackedSize":461785,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdAMXaCRA9TVsSAnZWagAAgooP/jkwSi4pBptAl9uSa86L\nBqtm6N4fg2fQP+0WDPOgy6/yAjKW3oa48YhfCN8Q3HciJNbrloTNHm4M1Mir\nFNA9Zk8yhN/AZwo63in7sBYcZX9RhUCEkZvpQU4T0KNpzbw6swRfcw7aB9U5\nHXu4FWjZLHb1eyzzVYyeBcHtxPdcjKMr3bm/AUYD5Z5kMLzMiPXyV+anoC4J\nF4cyOlhVpsnk5oagcMJ5khnA5vAa9oWZKqiaox51wjljFHGAfLJHuU5fGxTs\ngY1IWkj0sbYpI8wtRR84E0EuAWnricYYvfeFJYOhCGBcZdsEsjwPjkQQDtQt\nTO6prq+ham167I9Na01G1Aoto2lqvvyppHV16/4lz+GdSIMaEl/6Q3mwCeKe\nljH0tdSZIppG2lEcWWfmqEa+IuXjp3yHWWZUgK6oznDvZ7nm5GhJfB0p6GtG\nbLWS1xFBpvxCSyGUl1ZL3A3OUsoTx9rXLbv4faWrulY40eAld+daUqlMNyhG\n6JhMz1CgSfrphI9dc4hwcTzHuGiYXuvpCy9g1jCa4gQ4Rbk8PmRC0+Q27n5j\nwrUCeG17F89vxdxd/EEs54yum5mnA+g8wcAxdjlAhkK33ES8XrmQwkuDEGiw\nFd4hkx3vw7bcSjdsT54E+rEoP31pJ0SAplPbeDmzuDSjdhtsay2ZzSr00JNq\nRFSo\r\n=HlLM\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDLsCo9lyeY5R2yEApuo+E44R+++YM/+ARf0V4DowS6gAIgf1iw228bK855rghuHIYszDhZIiKQ1YIA0qiuNMKC3CQ="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190612T092837Z.06181ce.UL-67_1560331737904_0.4489954638779132"},"_hasShrinkwrap":false},"1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"0d2b6cda1498f7d6d341f1f3c1b9d54c57ed5555","_id":"ul-imports@1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-mpY7yLeqh9e8xfa5fWc4WbabEcl0QmbIzr84QBUJr6kgBvTSsQBnWilfvg8oYxMqglaTwgY20lSgdQ2rAKIT3w==","shasum":"45399fd63c0226e1d152b368160ecb79922a840b","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67.tgz","fileCount":206,"unpackedSize":473294,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdCgG5CRA9TVsSAnZWagAAFVEP/0Y3+zB/z2suIHFADMe3\nOENjrmP1uZEdQ5bFcZ/5cGGDvLo9yCw7eSavex34mQTIvqV+O6d2R6mbfkF6\n8Lq3mwfVBQuRxucUw17e0Mi/pdMm+/aWuedKjUMfSp8kmLN89N+xjkeM/XhP\nzx1R8ykKGFjt56+YwAoJqEaAay/3oRPkgBpPe23IBlktTXN6yHtEDeQYooG8\n+cELDCRhWWvJwlb8lvv3pMJ2daAB9eDJi1GmPnpkJm8tY2N6s70cMpzFnp3z\nNaNzrAba4mi7OSl1THFKIimpbHfYUxBAtEhYMqn/GUofdlabruYH+bO+Kqzk\n+kYWxVjG97u5jj/AxbOQdPBAV2o4+ncHYDvjQw15il1ip9fg7/qgD5i0xknq\nez5Qi5UtWz05rf7M3WjNR97hxFO1Z5YSlJHFzEdJYppmsKSpLe+M0FljOIvL\nuUt46rzXUvmDxrumcRPixlLsMbvih+xnZ4RmEkYvCGXTVse69b23oZwTMLOr\n0q1O7l7sPIFGV9vOGLCn6U/hkS2QI72PnwAJc+7ZWDQaiCGEscYwXDKrM79D\n3h/noE6Xp5OG23G33abW1uFtWX1eqpIKJMMnfSSFETQABtAyrlH5k8MnLeK4\n+XZr1mnHuasDNt21Elv++YXbdoMC2XuDlKVwhfKbpcOdZeGX2Z2pjVqWPomF\nRvG+\r\n=aZXN\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIGdc5X6jnB9A9NXxvfJz8c2jk6HMefxavK3/qMF+bzPKAiAtv9MlhI5E28Fwf4peGIjMfrme8LkyhlhLyMmR4dI/+g=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67_1560936889275_0.3731055456090615"},"_hasShrinkwrap":false},"1.0.0-dev.20190624T091248Z.b5d3183.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190624T091248Z.b5d3183.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"b5d31835dd74593197ef05778d47039e549d4bee","_id":"ul-imports@1.0.0-dev.20190624T091248Z.b5d3183.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-rgzEZ30B0hUlrzk6duOBUC4cqa9rYmkjTE0ETVRbJfXLIY3GW31iLVpgXAlrTtKHnBXGCqSgak36U1Pv3ajl7A==","shasum":"8c5c04ceedc8beb0dc73c7664c3ff73fc8bb62bf","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190624T091248Z.b5d3183.UL-67.tgz","fileCount":206,"unpackedSize":488415,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdEJU+CRA9TVsSAnZWagAA/PIP/iQYs9eHPfLmPL+z/42L\nOaJe/qUgxT2xcQms/CwafP8BY+PwiTO1QQs9XPb4en/dVF7OvjIIiUjBQ2qG\nLMBmxDs3R5vmrEdI7GtEW8a9zAhcQWSoYi+f5BoxfrAqlQ7oQuhYxx8iSHEU\ncQFakuJ3i7bozSRs7CNxv91dXcqXIM1oDRdrUYD/ttvHWAJNeINniQWu8jlc\nlkOX4an44rXGLkhWgS9jssccdGmbN3T05OWxDmgHenr1/rVjv/qqR8LC0CDR\nToE4/ZflxIAhhq/CRs5RA4KSU6WZWwKrQSb3dKzeLFijb+qiyICQaW315pCW\nm88j6zqs+aPbdxrQC/RkGeAiGagIVYvlyTOjkNWaAalPFXZJsZlBpDUSb5tf\nmSM8CrG02+XwDAycs7+O1nVoMR2nQBPRxFUSxpFfIEyhMoZ/9qecok0vP3oN\nfxRlpge5PxLHHzNCjMmbUGYL8AS6vo+rdlBR5T/0KnsIsC8HnyNlKdUlY+Bb\ndhKYW1UmHtgbeLTg/QREBO12YGiPYb8Osmsezk0XEclUEIqrN3fqtSFVpx4/\n8jsOZMos0m37fOW512OVrb5wwpN6DKpMUlVo8IxwTMsZqzpzj/IxDYsZcCBB\nZbzfs+EjNMvcGrYAuC2vKQgV/BdOFMDvCBDYv7zlC+KPzOhaoWE51LGwXkoJ\n7thS\r\n=Ipqt\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCEh8m1PmwxV5umWnydA7jI/LnUN+zTQMyJTytGaAjTMwIgDFI+GY1aFAXMg0uNDOfHWZF3pxmgS51M0DkbFj7aFSs="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190624T091248Z.b5d3183.UL-67_1561367869811_0.05248513456626247"},"_hasShrinkwrap":false},"1.0.0-dev.20190710T141339Z.4d40489.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190710T141339Z.4d40489.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"4d40489d1a1442bc3b16edc7c0c1a1d250b434b1","_id":"ul-imports@1.0.0-dev.20190710T141339Z.4d40489.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.9.0","dist":{"integrity":"sha512-LXaM8jTk/+D05pjgIdioz1JfEjGJ1uSwWmIlEIser1Ryw04/4UVKMetuNh0e0iCNeZRqIOxBeMsXsaGrD66P8g==","shasum":"bbb690c5cb173440c5a57fbb2bd106a6d5a54364","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190710T141339Z.4d40489.UL-67.tgz","fileCount":206,"unpackedSize":509599,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdJfMOCRA9TVsSAnZWagAAOE0P/3Nref7EL6dJhsbSXSib\nr+9vk9IenVVij2juWWj9ZPWt57YDYnZWiPvqk1Um37PXF9tnVBKABmWIFCbs\nx1KS2/K0fDxLoBkHgarzRPiMFZm8iZXhi/DspZ144DLz3LLbYP8rftM1rkf9\n5y0Xna+MMHHsblTx5dOAwS/3iFUY6Tep0ROqEFjkHo90VF9WsN5RDcCaK9HC\nKLy/ULCin5sfATIWu2yN5bopGbwasSzRkKsXsZ+LWexR2/USCKAQLSP4R87g\nWdXOBkpMAwba0bqfJLbh+4ahNV0j6+S8BO5tYQP7xy2SE1YAMEfH4xWE663o\n203fmoZRc6r/Xof4WoaM0MsxA7T+mx1zo1z7lWRN1sFzs1SBoaLts0ExsGSr\niqxFIbaTcfcSLZTg3K614BprPZrD6+r4Rsf+XM8+dcxiJAlUgME9I1fkdLNc\ntrJzgKiedTZ1QRWY+0IyARikSGYRkqV2mRRjK6XagRV9YX7aJLa5qU9in3bu\nXew/fE/fZF8N9xGIDAEh4YZa65hYQ1QwRCvfFJ2c4jAO2AaIT1EpMU49C/3D\n4xVCwrOFfBVV1rRpyjNAbv/bXAmXHv50syiX1ZFiQnMtCyG3QIXUw5UfHHXu\nMkULfgGiX5CnPL0NXHaJ3opRdD2iLLPmFETkEfyCofwDoFYuV28qzNaMBFSS\nuJPp\r\n=9CFI\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIEr1TCVySGKsBSUAkNrRkS27rS8dWtCLNgXABj8/ukVdAiEAjnS/0fw13xb+YGKDfc1NqXaHX96jjUlvzKcEL29DMU4="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190710T141339Z.4d40489.UL-67_1562768141711_0.3582488719638428"},"_hasShrinkwrap":false},"1.0.0-dev.20190805T093416Z.db90ad5.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190805T093416Z.db90ad5.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.2","fast-csv":"2.4.1","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.4","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190212T124747Z.23b4bc89d","json5":"2.1.0","juice":"5.2.0","kettle":"1.10.1","markdown-it":"8.4.2","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"5.1.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"2.6.3","ul-api":"1.0.0-dev.20181126T142401Z.124fbf0"},"devDependencies":{"eslint":"5.15.3","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-express-user":"1.0.1","gpii-pouchdb-lucene":"0.1.0-dev.20170523T095210Z.7508cd5","grunt":"1.0.3","grunt-jsonlint":"1.1.0","istanbul":"0.4.5","node-jqunit":"1.1.8"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100 requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are new records, create them.  If there are updated versions of existing records, update them. \n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\" record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n 1. In production: `npm run eastin-import`\n 2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n 1. In production: `npm run eastin-image-import`\n 2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n \n 1. In a production environment:  `npm run gari-import`\n 2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n# Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name` and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new data,\nyou will need to first run the SAI import or a full sync (see above for details on both).\n\n# Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order, \nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"db90ad5ca58dc906eaa0403f08ecdbcebd6a4214","_id":"ul-imports@1.0.0-dev.20190805T093416Z.db90ad5.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-JLvp3XVXHO6O3KiD0QVXPLjEJobaPtCC8kucuWBdIgp8IFfoWwdnKna8+Gfm0JOSu8cbFHW8O2ubl4Swu0NynQ==","shasum":"085511eeaabb3f4afb7610fbd5806e0eaf0739d5","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190805T093416Z.db90ad5.UL-67.tgz","fileCount":205,"unpackedSize":577495,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdR/gmCRA9TVsSAnZWagAA+JMQAKGMCZv03vlM2NonHIFO\n0hjiYx3PSOby89dJsOh31Kn6UX6JOr8WS91WFMgpGgK4yBvDmYDQTU1rwu90\nGfpDt1PeMfr4s5yzIozai+Li+Sf6B/ZXJBuJv5iIB+mEEhrECNWVgXsBxcti\nfx4qywhXX0D+HPqqWreoDQ/MRlnTkuEXhzsDeAj6GQahuHUeywF6FzsfTpUl\naxtb9pdLSYUuTYpBF0tKD6xaCg18aCm3QwgTKYUXvKRNTR71t3gbTnx1JWm8\n2VT+ahiy3c0apNRlzdLlzBXTm1XjrN4A8+0wba5FSAzkn/K05S/LRs8BgFZ5\nylMkSa73zxDigi0fahW8WTZkgue/EJQakOAsQTJl2+w886KXVGgGgLN1S9lB\nEeDPFVPWW393YVjxCWQWLkrLV/TzBxFPUZ1KiDMAAeOMZI5JMRctNLAkWxru\nY4WBFxgO8nW6PMHwD5Ui4g7EX5nwnRpV4mukXYG2dkRenuLUXYe1d9xMyVJW\njQxhgHl5J/rRa0QCrVqun80gQAgauiH63WNTC+0og5jHFQSbM+jEP1qw4Dxx\n/fN+0rN562BaZRozkM4WaU2nazd5x7jAaCoCM1c7iEEBpxqMyDeEbImnPsNu\nxIZWb8A6j2ZvB6DBfv6M58oPFkJ+ZNaZYwH6RPvnzbTOOWjboLsyaALGOl1c\nzpJj\r\n=63y1\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDGze/KHVlQPjaKaiViWhtKUAz5jRRbxS5AzLcuhX444AIgfumEY7nPjp4gBRd/ouWtXKLBzMR0YY2AyfnOeImhRbE="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190805T093416Z.db90ad5.UL-67_1564997669802_0.4344217392080809"},"_hasShrinkwrap":false},"1.0.0-dev.20190902T095239Z.cd2d254.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20190902T095239Z.cd2d254.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190808T142952Z.f8576acb6","json5":"2.1.0","juice":"5.2.0","kettle":"1.11.0","markdown-it":"9.1.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.0","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.2.2","eslint-config-fluid":"1.3.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.1","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","istanbul":"0.4.5","node-jqunit":"1.1.8","ul-api":"1.0.0-dev.20190829T121743Z.3698d7d.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"cd2d254e49812a6991789155af04755b8e0435b2","_id":"ul-imports@1.0.0-dev.20190902T095239Z.cd2d254.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-T+HMkiW54du1g6jiV/uLpHNByKRRfsaevl/TSIT5x5eL0PvjcA+oNFTZkNUnkoxp3g6LkPXOl4qIQVu7n8w++g==","shasum":"e8749189df957c7d5ae8ff2c23c8bb0014b25b3a","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20190902T095239Z.cd2d254.UL-67.tgz","fileCount":204,"unpackedSize":583111,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdbOaYCRA9TVsSAnZWagAAz6sP/0xJxJfNydh9a3b4qWb3\nZWADfy/44wM/+t7PlsLPxHFZi83Ysq4Dtgj88KY9rBVPgelRkI8wSCjxmxxd\nRtFgmEk336YCVZPeZeTFqnz7A9afbXktM2ed+9gXr/XCYXiWes3N+9Sa8bis\nymvukPqzmGDCgf8XyrtbiMCg+Gy2z33SljEL45sLbpPRl194gTC5JiXac44V\n5aCD47q1wCdLPu80DF120qZzLWsmrFaWZthdtpuojjb0bmF+d/yJvvGMK/rb\ngBIwmupMr+ByAv/BI4y8/YUQeMUXttELW9xiVIm8Vx7ihxr0GkC/FJoqjwv0\nBtK1710H0Co3w5ySSQF+NmQW/waVfQ65kH1ytGnxnp3KO/OMUWPYdrDIWSuN\n8eTX3QCfr3DMOQVwgUdCA6Q/R6c7MWi5EZrT7v1y4KFE6O0kxaIEsMl0KkT2\nK5WAHWole9AmMqkrX4I4Ql1a+iNusg93E+zmY0d+gWbtiXseFbd5RBqiqxBq\ncQEg4pBXSvXjfkhyJnR76v5dXc87E9foAKJi65hVQZNbSRZc4XSb8b43uL5S\nkDgN+LjnBEkWTwHs6LF+KesPVRPSU85cHA8HHMG0dCR4cEyoUYnhuvyaFQZq\nAsNRd2lQ3mD246hz9isdC2n/vq7w4Yf5iUq2vkU9L6as65BYZn+XzAmYpGbk\nTVCI\r\n=Cvgg\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQD7KB6Yk2vMi6RpM+5dv8V4BJRYC3sMngdJVqrpQevzPQIgIaY8RT+VmAQ3HGjoZit9R+iUKrsCnFzaT4svcoyNUXc="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20190902T095239Z.cd2d254.UL-67_1567418007909_0.19541793355405956"},"_hasShrinkwrap":false},"1.0.0-dev.20191009T120702Z.83fc001.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191009T120702Z.83fc001.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/istanbul/lib/cli.js cover tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20190808T142952Z.f8576acb6","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.1","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","istanbul":"0.4.5","node-jqunit":"1.1.8","ul-api":"1.0.0-dev.20191009T114656Z.bf585a7.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"83fc001393f400236fe0105a1bd736fbfc67290c","_id":"ul-imports@1.0.0-dev.20191009T120702Z.83fc001.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-qjVMRFuphirIAsu4U0JMKqO1BwgpbVqWiYdN+KMDnYvFFKoawdCz1NVAiHp+lTlk3VS9C9h+LU7QtXcFfEK4EQ==","shasum":"448919db56dadf4d9efec62159ae9fd4be88a5f2","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191009T120702Z.83fc001.UL-67.tgz","fileCount":204,"unpackedSize":583077,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdnc10CRA9TVsSAnZWagAAeRoP/2kD1RgJuRgNJavesn88\nA6zN/alp0S5gXdZXXZfy2pJxvNcdUHDY18naXRxSDpuPGYTI1OJpLTOctmAP\nri7iUVaDYhvlZio5QKTNlNS52QL0P6XpOQxzrD+letgC+VHOjQ1AVax3g99K\n6LkE5bo/bXxTUHxMvcATrm4w6ZuPDjbexbs8jqjjNNt4kI60XUM4CoIdbFZc\nCvzf5v09AbhlI4mmS/RyPlLgqFkHItlTh/MuIBhUUKWdSIUvxsPfnM9kAyTG\nO3/UvLaFvZIEEu4hDZl6seulqv5OsRpcqTuOhUh55MQCAMM7W29ngp4feWbT\nSFWeLtM2MZK6rIJdteSNZQQJu9w9s2gEbkbIGwCk4vgef6hc455z2XAMK6lu\nwHCbQxlgzylDfNIF4bX43974lCAp5cjt5MVBdvAn9w43f9xxxtiudxiVKPoE\ngKnz8oDlHUkRqZlBeT0OdpNZCZbq+RYuIoJQIvmQD4V/l5LH6l09w/SeSwG6\nZQIR8+WBNzyk94ngTSyIxbITPq3ZzDA0/3FqY2HXySoq5wK/d//oyUIxsL34\n8VADWpCjLhn8N4Z/cTF32ox4jvYOa+6amjbD3uiXdw1dLAfiRd+BRaal+VLB\nphqOQg2SQQAcp75n3vekHvi+Pawns8G6zcX06wsLIcziLuxuzqRwwuc+7344\nz0Qu\r\n=64D9\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICcw9refybZVGIj8Ei1o5CI4NAb0SvBgeicbIAqIWBYRAiBMh0//PNmXqEOJek7S04bHh48DV2MpTCgemxv1l0OrkQ=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191009T120702Z.83fc001.UL-67_1570622835414_0.07129428939370697"},"_hasShrinkwrap":false},"1.0.0-dev.20191011T090801Z.65e4c42.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191011T090801Z.65e4c42.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191011T081350Z.de1da4c.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191011T085109Z.0ea5dd9.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"65e4c426457f4435a2efb8eafdc078d40aa01e3c","_id":"ul-imports@1.0.0-dev.20191011T090801Z.65e4c42.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-E7ynz22fE76gJczHKCRSwyoh0LWVMrX+DXR3EzNXwCGeWVc9o3/HxCHs2mZn09DTX+KQmhEAQJsEie66vvcLDA==","shasum":"606a12f138344b7ffb4680f451b5b76a8ee0ec8e","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191011T090801Z.65e4c42.UL-67.tgz","fileCount":207,"unpackedSize":647633,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdoEaTCRA9TVsSAnZWagAA7eIQAJVL9ukIjKTrU+hSyyyL\naGetXHyWFRMkdnUioEZ1YAx4Z5LFHrEjUQ/WGF/9beXlHBU7Ja94BDyIIvzK\nWid+/JA6ULI8uFeNhNI0/RIU4KVNn3/3YzTG4SnFTbp2rq3+qtV0IKf2LFvZ\ndbJQF7MhuMK5MSL2lNSo0l1siEBNQJBuXAPpqhHrH2SUikeBiTwLj2lfggjq\nk987GQJM+su8ZcKT8yJus1ObZqKpkKoFq/Mx1zP/xkwt5jemfUHAD5cmz0TB\n165pr8x5afYYa6HkHG92QNKZA1G7wej52wLE3Aco4uT8cqDZvN39THw4rpor\nvqilWDIoH5wqkEoA2lN8ul3PlOyuP/9P3dWQg9/oQQeQMNDx2XTYIVr+FS+d\nsc3vOcRYPak5EuqcKuiA89UZJmd+uKvAWyw4kK136acnwmpcxMpbljgcp33B\nu5KFjCjmYezmfdxDjWrLh0zbVbmyXsJaGOBv4B0YBQNnBAdfiy58WgizLBCi\nBVpUj2jx6D1T1apv9gW3Sflgw4UYcX1ait3EO15j4vQXsIs85Mi11oZiAKXD\nEG+jou3GNnCv3DQUeA/epoEv7lH38ADvSAJUkmRVkDpHFf0PKqU5S5vplf2N\nv6SSkvQEtGkJpSdjjUVNzVJjGgqBkmRr2GyFAR58ChjnXN4GK5xpQ5OVmYLx\nMoXH\r\n=mY7J\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIHcu5iM/PyE5dSw+lBnxHL00TIieWwm/lrWxae5R1mYwAiEAyGcGksthM2Kf6hgxjP28gj2n5PDKtYOPDBKjJb4EJb0="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191011T090801Z.65e4c42.UL-67_1570784914790_0.9451793470222418"},"_hasShrinkwrap":false},"1.0.0-dev.20191011T163526Z.5ac8d76.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191011T163526Z.5ac8d76.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191011T081350Z.de1da4c.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191011T163340Z.226532a.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"5ac8d76d125a457dcb01022ea3c40e73b8b458d2","_id":"ul-imports@1.0.0-dev.20191011T163526Z.5ac8d76.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-wMiQhv+vLNtrheXtuF+07rsVTdX726NrB5wAc5poNCuLUwSq6IMc29z6V2Z2a5KtwrdsnM+X5k2BpfA2V9OPVw==","shasum":"456aa059a8bd5d6c136e4fd1205034e0fee96ba2","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191011T163526Z.5ac8d76.UL-67.tgz","fileCount":207,"unpackedSize":647633,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdoK9hCRA9TVsSAnZWagAAvDwP+wVMl6EWIpODfI4GpAq5\nqE23rB8El7lqdx7LnVVMPzDC33M5ER3OOe+D5lwQcHzEbi8dsnib0QrWsjU9\naX6U4KZDwQlmU1WzdzMs52QLkqW6S4hm++XJBqSSUoTbh2nm82NHnRPHRp2C\nL7GNk9Nq7MPPvVNr4axitBqv0jkuCbpxO0QDEEf8LWK3UcxZFd9Nvmwq5Pt9\ni5ioCcM4+ujZsKYOmHSln1vNqeVMmJWHpkB4zfzpeEDCLlUEAMfQOmn8IaPg\nE29iIvToOft+H1Zwvw5PEaFDXcf+mwa7z67lzY4vHJSNjupIO/Wx5WwyEmKk\njVfilVl9uLqta1u0IkiCGBTeys8v1lPom1/sswwCNN6sMnekHyo/pX7R7odq\nAKbZAYm6pEwIHJicNdvbdS1St9xe6YC66kwX+8Ofu0YIW8vAI/BCkfb24lJt\nj7WuAeLxPOR2KfMmjsxnMnKfufOENJV+plS6ocmgQZtPA0mWSRqNYhC38oTF\nDGe2E+FyPq2UgeLVA+PNVTp17NuNg+FlcLx+kBjxA3WNG/Sp+ixOtX0ZI9ye\n4ftrIwQonB+Su8oMtlC396iq2mtDl8wl91i2cO40LXkJccqNPVJc8qCxTh+K\nzHSErLMqUrCIVxTnIpQDfRR49HvuJJnquNejFWgwY34zJFP/946oq/a4IUII\nlgqv\r\n=T+hf\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCSo1GJEeN/FEdNDuv0tQgIJ2yk3op6uNSFfawaPfW5WgIgRRV6AXW9sltHdNCPCc1iYcALZx7ZOghUWCAwRYu6P58="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191011T163526Z.5ac8d76.UL-67_1570811744357_0.44089663910425503"},"_hasShrinkwrap":false},"1.0.0-dev.20191014T134843Z.b03bb6a.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191014T134843Z.b03bb6a.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T133849Z.154a17f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191014T134607Z.a97da2a.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"b03bb6ae3be1427c75689aa74eac68f11b67eec4","_id":"ul-imports@1.0.0-dev.20191014T134843Z.b03bb6a.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-j94DOc5o1N0Ovkm4qyzn2Kmi2xNDdG+8c0w6Xs8/To8ZTeZJpgcuCL7rU2AusQvNlmqkD8hZ00+EQKd2yhPKTA==","shasum":"781ed34324727f29a5e245408f897dc0c5517081","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191014T134843Z.b03bb6a.UL-67.tgz","fileCount":207,"unpackedSize":647633,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdpHzNCRA9TVsSAnZWagAAcXUP/A2Ntu3BVmrN6/TfFVkY\nl/lqhHE8+6IhGeTU/jV8CEHeKasxpqwlnFMV3Kzu8DFoTlXXeCXK6Qu/bnsE\n9mfDOPApGIfSNWFuDauYXQD2vQAmUjk9Hm356oGnDR8hzIjq0q2WPRZTpnC3\n/U7XXYWAElh7A8J1/mgQuVIzRCOCqUaZahXXbT+/72s+5OFVwOxsOMoqmS2G\nJ5Md+gn6o1aQ1xIYsbeS9I082KlQz1/GIV6aaCBxXTJaUwbw+5FrjfbAxdbq\nvEgz5xHZ+WNfrjXsgYoAK8vFQ03d2nMLMyrAUGnR5WDnEQzM7GL2s2lDI4Re\nf5MQ8lVWH2HCHdWx+VE923FXwzdTxtwAP+1wTvw9bBi2r3hsghR5YL+AhdJY\nQd4yvnjWmp4oFmFZiXzrKi0tMKh3ykEAwr2Yf0kJeyc8JX4JOYw6xmGRTgSl\n2zhdrNQEYXvfoSLv3TWuLKKSALh8ji+C9LJNa1I+kiXLjrhdx25AIl87llCA\nIZ54CheTIMaB95QBYiIU9hyd0oTwNIXbwA/ASXcPLuGY6GhJibCxrcy1g/py\nfTUHErsqVzwU9hY6G0PVlu39BKiHX6qkP2CLxuCpgdNtE21hG+2edRYWb4N5\nQskpCrq0J0uisNqlVMRENXzL2xSXpzrcX5PZULg/6USxNWZEJ2wd4No2B8l/\nN2ui\r\n=5bAt\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCO/BBPh4/l9z87PIdA3xTwn/A86mh++P5DWcx5WlXH9AIgXWCj+hT2Vs5uOUUC6uvUGGNZl42kB8eXwNZPo0Kcnkk="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191014T134843Z.b03bb6a.UL-67_1571060940467_0.4492654428385623"},"_hasShrinkwrap":false},"1.0.0-dev.20191105T131713Z.f747aea.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191105T131713Z.f747aea.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import-cleanup":"node src/js/abledata/legacy-import-cleanup.js","abledata-legacy-import-prep":"node src/js/abledata/legacy-import-prep.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"f747aead11abd93d1e7d2d858f543996c4cb3972","_id":"ul-imports@1.0.0-dev.20191105T131713Z.f747aea.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-aANAOQb9rbLPx2fHaB7Xffnbs8bPLEBiDbJH5Lv4/wwkjyzdjvmUpRHSNcRbUTTBzIG/ZpZkXJyp93p4RoA2DQ==","shasum":"d93d7734201d7749e905c33206de6cf298dc14e4","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191105T131713Z.f747aea.UL-67.tgz","fileCount":215,"unpackedSize":103445283,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdwXcNCRA9TVsSAnZWagAA3WQP/jSrT79wgjus9jjaiuJc\n7sweHmhBUbjbUSqP5rh+UKYCaeiKKmWXi+6XW/kXrY13+IHFyidJWQx9HDGY\nkV9LEhx7dbNVUvAWodHeE0pbB2HChDwwiCkW/2RJi0h2og0agbM9SDfHlSpr\n9CKUoRiCzik4FruoLtHg4N1R7A/KX0lqaGFsCqnPhSR4Cm126tru4aQboLkI\nBVJS7hQ4GW9qarX4rAqQKQ+yHzZYF2lrR3lXjaaVHwEyNRVDwIUnV7Mr/B/0\nXn5TAr95TtpbsXOXZ1abN1I/6ZfrockaLTKcDGnG/nXMPTc91+wb2vumwEPO\ntcZd+wKvILT2ZYHFHtFDYrbiLGUHi+jUaL0yKxdMDPsYUIwCS4dslpptcsBo\nYAX6/C3ofkh4rauQTbsri8+YsjurcTL76l5JFDy2yCgTyuvQ/wG/2Q0k//00\nM0g+DvaZ9zCZiC1n6NE9mUf/lXhW6q1BKtznNFh1KoVI5/x3KVOoGpDcX7CQ\n9Qw71KRW4O5IlkCbpCXDM2C6PgPT1h+FhiHHeVZk7Wt5eYGqU1xB8Ry+Zrc/\npMaxk8UzzOgj8HWwpArPRlIud8/Di98JnoBrfbYsU/TNA/Z99vj565gskvkF\npC0vreXMl6dreCkPOYvEcfKWj6VmlurlFBPIOIzJiPGhLFpThNgQuM/sPYBI\nTuVs\r\n=WZmo\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCB6MV7z50obnv15hiASgdwPee4DS4zioqwLb0gMBetQAIgFboozMOjdTmg0nNa/kSumBcCqK+46/xrdAqLLCHMl58="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191105T131713Z.f747aea.UL-67_1572960012385_0.4064649301283074"},"_hasShrinkwrap":false},"1.0.0-dev.20191106T091507Z.5302765.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191106T091507Z.5302765.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"5302765adc286b66c3a7dcb052d5149b08105f0a","_id":"ul-imports@1.0.0-dev.20191106T091507Z.5302765.UL-67","_nodeVersion":"10.15.2","_npmVersion":"6.10.1","dist":{"integrity":"sha512-OdXt1UdsirpXthFSwrVj83llxQMZT58u4hNbLIReygiQzlnccJcwiedAKwj1K+g+tnRb3g1TTBkPfSPa+U2NtQ==","shasum":"03423881d03a1dfdb5632164e42f9174e2763213","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191106T091507Z.5302765.UL-67.tgz","fileCount":215,"unpackedSize":103445220,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJdwo+CCRA9TVsSAnZWagAANLsP+wRoyveavwe1R/OivoBu\nvZXzpOdDOSEwaC+gzcGBMazZ6cIDp2tH2w1/diyFOteJmIIe9lm+PBPORpeB\n5t2g8hRDM7sLZ4dJ9TBAhgSlFvuhWpDRjo6ZfRAlnv1kpQIpuj4LyEonv2MN\nOiIMltcaprFsmQPzJBsME93kN56N8dTF/JR05aOftos73Zf6qiPmOWhLvnUJ\nlxxKRgy5rLn0z2e0kr78KscEaUl5AJ1pA9rkWzNXAjjO+F9e5ZgeI11Ffg5q\nmqPjab+lgR9eupBcv/2Zw1HJJtfdPwBRmOKU9V1NVMsP+CVNHpHQK1XOXceP\n+2OAeDLWMpK3SP3zr1+PA/QCCGi2IJl40RjOH+X5hLwoRgrcgCl5gVvl0XE1\nkHtRf2y+awTMF1kvSzN/T5aieEw2HYT64BsfLViwSMds3ZUaSSn5j7HuRTfP\nISeNGo+/++Q6ASCq1U9qhKPPTRyYcdaLiyUhM9SjudGuGxf4R9tPmpaonG0s\nA6Zp782voLU0I4eOdAZ2bkUNWOHGCEX0u1h9Ttkfx0e61bN8ruFbmfNo1SEB\nsSyFJ8n6CavNBHk5esGtGD11uTp6xLkjy9VbAt0Ane0LuhNf5ES7ofcV40vB\nrWyjvna4IiFZ0mHcqIRKug2mGVfeL+XvwC+L1xfinf5m6gB5KVK5ARrGE97z\nE0ZD\r\n=6NHu\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIBed6MlYQZLyJwd/odHzPY82TCQagFI3oRndnanE6aLYAiEAqTSyjk3OmPoDNxnQTFXFbdUxTT+/Ak+LZzSI4S9LbXw="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191106T091507Z.5302765.UL-67_1573031809881_0.6243237656810592"},"_hasShrinkwrap":false},"1.0.0-dev.20191204T142144Z.22a916b.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191204T142144Z.22a916b.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"22a916b66d1a36ed023c2ad50f845cc7d36f7856","_id":"ul-imports@1.0.0-dev.20191204T142144Z.22a916b.UL-67","_nodeVersion":"10.17.0","_npmVersion":"6.11.3","dist":{"integrity":"sha512-KZY2oGzI9nXTB0Cme41jr3imgZUYu7bDOLAGZqzeJDAtbrC3NlL/gABV2OPWf9cVq8aibYIFFPMiI8I6Ez5M7g==","shasum":"745346608a4e4c39d218806c1ea3381c7d1c6059","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191204T142144Z.22a916b.UL-67.tgz","fileCount":214,"unpackedSize":69122918,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd58FaCRA9TVsSAnZWagAA2KEP/0h/L8AN8s4bDx2yJyLO\nB1XEVliA2jxlcdK1bIJONcrhVxAtaddVV1vooleFxUTI66cqT4873UgKKUUX\n9IP40aiTUYL9XWXICSqs2aHkZAbUMxRFKhlcHEvkSk4DXuY1rIZJWXJDfl82\n9AMNm5fdZObhbfp5gdfvMHAA2nrWuI/cu3FB7uKyyOQJOMs4ar9aNx7CABZW\nh7C0u6N+Rwly7i0VzqBJjf3sGOqcygZTIOAwyedMEOnfT+aKoayJSmPe6Mk5\nEDSL/TniuFzfrVoOmXe+SmkWCqMSsLvPiNAvh+XfI6xK6ECNVuj2s10kIaI6\nzqE8MBViE5YkMO9o65qdY6kzed7+89M1TNoNE/PiI8/alaHvaKyOGIPQbg+i\n5bGVdvxCSHTzwBx6Z+a7Ql630xfnWDIGH9lpJkfZnTDGIEkO3TPtyx4zCHcM\n1b3HdmLlFYSXJw3FZWd/SKfsZap1vwwSZ0yJ3DQ/4/hhnx1HqTHM7OU2vugh\n5fJoa8q4NbTfOQY4/isvcVi8JMkdAYlXm3SCoY0cb/FEK9nNaXG4gk5HA3nH\nV4+MbaN660KODX58JpFhLaPyB+Jckiya232EcpP7qMEo+G7ZfOT57+WRiuLz\nVgkg2cNdclRkI3ccPdITGeXjjOM5H327X7rXy6L5+Evn3KA4dGSaAnu8duG8\nKV7P\r\n=kx8G\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCB8UqQP+gqFS7Busezvr6vQlvL3+0GLgSPEOYXNyUbPgIhAOwSvZKmy7afhtUvKYHyxQcpFLgQj70XkFd07Xd1jrfF"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191204T142144Z.22a916b.UL-67_1575469401681_0.3076804559819186"},"_hasShrinkwrap":false},"1.0.0-dev.20191212T125642Z.22c7eea.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191212T125642Z.22c7eea.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"22c7eead2c6af4589ea2e16d1692aa252aaa9c76","_id":"ul-imports@1.0.0-dev.20191212T125642Z.22c7eea.UL-67","_nodeVersion":"10.17.0","_npmVersion":"6.13.2","dist":{"integrity":"sha512-YYdWeO2hQ7+Ae+7xaqS0b1/quQ0OgryXqLfMZPgdXTbC/hd3qAfssiX8TVAncdGRn7iMjsLb+upV1/ghKj6dyQ==","shasum":"c32ea448daa87511289a46beae4932eeeeb68b30","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191212T125642Z.22c7eea.UL-67.tgz","fileCount":214,"unpackedSize":69123501,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd8jkeCRA9TVsSAnZWagAAaQYQAIPlfr4uQ4fD0+/HrUvL\nUMjTKbIf46BPue/Yx/WenwuH+SorrINesu2QGrrYs90GT+jOCufi7j7fDrFZ\n21Izdo7fptW7sFlwmedtXcgLSNDd65Lc1mfm454OYyjAVQuMZNkqiKznzxns\ngyI3TkLh3F5/8C4AQk/tFSPIYpKey9EoqftxsFtx1tAJ1GXFVOkROnpv0ZrQ\n163bH+8DIjw4q0yq+zb5MJBzcEUqvzDVHzBb3mcrC6IFQdY9Ik2//MqWzfQf\nL8JJfKCtv8kl+sBUviB7txFM6Sd08RJTDTG+pxxxqppq2pwhxNWXHNQc6X5L\nSKFvIDakP6yApqJ8yE/dA+0w9qsF5mr4Iq7Ajj1nbRo1sr1bV4Y8uFPpaY5s\n2Qo7dsoZ46L0mmtR692TGEkKRsX903A+XvM1BsyrRWuYRDj6maD9SiZEDz3j\nJqGBcpBeX6jtzyMhmgdKfGnTrcCrNyIpMGT9Vj3WlcT4EzXOFf1oQ67ky1nc\ndVujT2EB8RD+sGYjaAG9WJ3+CO1Ohsu85hdjbWN/aTPExNEZZ8ZIDxTLbGe8\nokohgezjSaeB2tfromB50qgbd4M8Op9OE6d9It/JoSdNAL4G6Zf/L8twRkUV\n35rq80YUdILleb4O+sr+7J/hCgp4Xc3c0b48aJMWMFe+LCRJt1dT4DAaZXdr\nCcfG\r\n=2Scq\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIBLxPhb9aG/CLz/qVjrZ6ryJt2vm0Gwk836/6eF1XyyXAiEA4gvwfNB6bydwkWwIEj4ptF17i2ZMjGN1i4xQBEpgeKI="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191212T125642Z.22c7eea.UL-67_1576155422024_0.10465019772258999"},"_hasShrinkwrap":false},"1.0.0-dev.20191216T131313Z.2658baa.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191216T131313Z.2658baa.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"2658baaa3b28fea1796a10ecffc78a1b55200f65","_id":"ul-imports@1.0.0-dev.20191216T131313Z.2658baa.UL-67","_nodeVersion":"10.17.0","_npmVersion":"6.13.2","dist":{"integrity":"sha512-ebyfdMjU/NeLUtqjR9Lj1U4ZrQdW6eVCvB3QXOApQ33qihQwJ4LcNjRXfOr8VKXbpdMcDUYD+2IuMbd1l1Fv6A==","shasum":"6f1388905d75d217353fc7df74316b0a8153b673","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191216T131313Z.2658baa.UL-67.tgz","fileCount":214,"unpackedSize":69124195,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd94MQCRA9TVsSAnZWagAAQEUP/i934ptZM/yxKpfsuZHB\nYXqolNF2e8kL8z/mMwOMMtgFLu4M/zBFWzzqsFR1YP89DbrtTbqrzHvoRJF3\n9P/nmIOBnJMKO4m0Ybvhek6Eyn1EeOKL/Stm49Fl/ZAB30hYZ/jgi+nFWfLl\nMwH5NmQMzcldMVl1FasTvPUgKE7Tr+A8q/tIJgMJKuFYrchyfPj+D5WDkboP\nxEb93QUtBrxBESQ0CLUzRRDGeRS6a+NbR0MfxKT220rFWbtRZ03gU/LKIvii\nseSN52akCvTCMUByr5K3IjgOGCw7I7jwTsyU2GNX5uQEnZVOca122+Xxlosk\nCdXS+TLeZeo0xHBag1HF80J76pqX2RAG4vCZoKfo+MbsjmjyGEzPT637N95h\n2vWjK0bo97f1exBZrRhKc9+k+sWxEbuAKA+qGXkHsUiKrE2NPyuw0ZxWo1Yg\n8CtNmlLe3VNSi1ihqZBSXUnd15VUfrMZZB2f7bZbNEVNuOqvTdYoeuuaWQKb\ntM/ncBe9JIH+8tOXl2pl5UEU57pM4zbB1ZJzWvw+WVRjKyfMxImaHM63PK0b\nIEHKwhIgdzAoGjHB9I+gSVy7X9njAry7OBqg6Bd8mldsbNVlKYL8uHDJJ5dS\ncGWocPPIovaOOHt1WMUN67xDe9ki69M7ztqFdmmaP1KADHigcp40rvPvdPa+\nWljL\r\n=SfaT\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDYJa1wUus/yttLkrkTe3UkcMjWFCN/nBecHzyKNWvxygIgNGfeUhQGKTSlo0uczIZjpwBjzi1+Oygzp1mLPTmbUH4="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191216T131313Z.2658baa.UL-67_1576502031879_0.026822108288582225"},"_hasShrinkwrap":false},"1.0.0-dev.20191217T095812Z.c37ddce.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191217T095812Z.c37ddce.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20181016T230646Z.e3d03b4c","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"c37ddce9ac1ff23a1ed866582d012c44b6403479","_id":"ul-imports@1.0.0-dev.20191217T095812Z.c37ddce.UL-67","_nodeVersion":"10.17.0","_npmVersion":"6.13.2","dist":{"integrity":"sha512-Iz4U69NkJe+IYYIEUYLbphwQld0RleMsYu++7lx2qaxjUX7abXb6CQUxbksoKlpQWGb/KJqjqhu+J0tpfivdJw==","shasum":"b0d18b4afd13fb01a92b5326487cf9e4e7407630","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191217T095812Z.c37ddce.UL-67.tgz","fileCount":214,"unpackedSize":69124690,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd+KbXCRA9TVsSAnZWagAAvA4P/ic0iw/uRagk5Ivn58JO\nlgOmJNXOGp3O8cDwDzScXhTaLVeF/dt4dF/aLtcxZUYEp7eoWZXMaYPqBiub\nztt/MyrcEhMaIAGkgngDUi4o6se5yu/0BnQ4yLSS6geOhT9p+THOw7QRrLPb\nWUM/hZ7uWvTZIQuzno1+4kZDHSTiCOTeHhJ9QvvoTPy05QJejLQFy5Zk4jnv\nY9K4AV78ILY5LJWSNBKr7wCmnhRPPjk/hfeBtEDGQCuTU26zCORA11do2ouj\noRI5YmmCyCXbEAiV4lTRVpol10HdHlQst2huCcFSJsKgtu2qYwB4Os5sWJRs\nRYCeVqL8UpeLBGhnqapcmPAIMvPuIAXeBRnbAIG83+LhqArVs3wdqMQ/Inw+\n3sOfOLj5jvQMC9UbmZEsX8p/h7LDG9wBPsVka8nPb6w5oy0Xgv8JxEVnNi3a\nU5+VGyuW6BE+ttwhUwmg0r1eNMnNkZStw2Wzhczj7AlmNTV/45AkCBXd6oba\nHHEOMND3VQRoEif+TiIl0ylr1I7UcHIjmsirk6WSOJEt5/Ryjm2xXkXP7c+Z\n9cIWmUJvfXOYwgwfXbeIClsbmKYBxKbtH/bNh486hW0KaNYkgcj2SqtqDxo3\n0G8UiozxuVeLMKr5j97rRwdcYvshQLhWUzQCANYbFKztrEkqHiSFs+GJYImN\nO4yO\r\n=1wCd\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIAhGXbNJiWUPhNQQYlrNKqw4/W7JYggvlZeVy/COo9PBAiEAtzmTDQFrwigd6O6PFeQmT7zrX4aNgrYxpCgVCy6M5+E="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191217T095812Z.c37ddce.UL-67_1576576726672_0.14734106198750663"},"_hasShrinkwrap":false},"1.0.0-dev.20191217T101352Z.ee2e90b.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20191217T101352Z.ee2e90b.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191211T142911Z.0af24fabe","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20191015T074901Z.8b1dd87.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"ee2e90bfeb743d76fc26ab801e3b338742d48141","_id":"ul-imports@1.0.0-dev.20191217T101352Z.ee2e90b.UL-67","_npmVersion":"6.4.1","_nodeVersion":"8.16.2","_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"dist":{"integrity":"sha512-iH9TwJKyUwlk9tETZAyf/SzFaBl0jlmFcipJte7ODaboxqwqzeSt7/40e8mLk/M1QcoCzReZ4oXr9HijGHZpAQ==","shasum":"a35886844a0a293ecb2da22ac925c8a083b653ca","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20191217T101352Z.ee2e90b.UL-67.tgz","fileCount":214,"unpackedSize":69124691,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJd+Kp5CRA9TVsSAnZWagAARXYP/RtdRbiqcMN0MN3XWDju\nR9thjt3x4NRFL+wdcdjXGgrh8TFYlVlHwPsVqJKQUmOYQozSt2yQMcv1g91r\n+crA+cPnrPnITJZYSGA0zpqMY+H0nx42uc4JJZfXtouFOnrEbAn1IcMi0wnq\nZ+LU2xWwvVET0ZJT/Kioj5lA1sYQylf4aGHFF20+FtxbpAgJcmzztZrRebj0\nlU2u8dE9eXUsQ/QBggg+7AgtgMwJKAmu8aKuqgSGRDu2pusuxY464eRqnlpD\nrCRLMtiw35k8h6h6MwL4jMTOlBP9F2OuzdBMXY13GZopp4BVpcn0mtq2wB3h\nZxYPOYr7DuwuGwDYmW/kiddKqiTtXFO1irdgekcHFCyQR+tZbv5a0seGACaR\nD6KsJ6auLtvGnTtKyzlmJASb1J+lS6p/tvTbzovQ8TvLD1oQeOnsKeeCoGSD\nwHW1rILBnzG7dmEH8ssJ3JgdFBD+gH9MORT50uW2EcU15HawOZOFBNyMcWWM\nhpTIF7JHprnUa+JQqhiQwcePH814hMNtnEBTFASktBqxrDaNdTFmSQSK0Oyw\nTVwd6j+9+hXm2XQ5M7GAjO/zsDX3n19KOpIYnn/+9B9tYOrl3tja6XWJZCAa\nPPCJga/XK/YMx5camueuXHMV3XJtvDaqVQTTGGsYUHa80y+eGfFX6GPwu3P5\nCfV1\r\n=iFiJ\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIHp4m2N5FSi81FUBPLxl0+6PWIm29oGDGe64PEZAukR8AiBdHE+j7Ti3dESni+ITRVCLhrGtCaKoDJODBjXgbbE5jQ=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20191217T101352Z.ee2e90b.UL-67_1576577654919_0.3287976667543433"},"_hasShrinkwrap":false},"1.0.0-dev.20200109T111334Z.7bf9919.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20200109T111334Z.7bf9919.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191211T142911Z.0af24fabe","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20200109T111055Z.0aac7fa.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"7bf9919dd50004e07a873efab026c9bb8e6eca93","_id":"ul-imports@1.0.0-dev.20200109T111334Z.7bf9919.UL-67","_nodeVersion":"10.17.0","_npmVersion":"6.11.3","dist":{"integrity":"sha512-n/6sb3xZKOVO7knX87fqvU6N/RUfSr/7owIKafwrbR6Bh0CD/Wg7au5UJvv1VAMBjys0c9coU5enZF7XsyHCZA==","shasum":"da29661503e8d632fa951cf2c45fa561fe2089a3","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20200109T111334Z.7bf9919.UL-67.tgz","fileCount":214,"unpackedSize":69124691,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJeFwryCRA9TVsSAnZWagAANSkQAIRP16Y3qBx1T62OpdYk\n6zNfkemKuwfD7o6y3A6idE7ORForXxFbn/zYShT3/Cwmbpy2HX+YBG7/IYUE\n7OTaD51RpJXSvy81qCxJvT8mwQpI/3mLaJ6b4GKrPrAV78svFKeTPdvtAf5b\n/XkYLjyE08sOjKq/fj7MtZQLcw9sF5TrjSXKKsQ6kplWVTOX2HSms7YshRaL\ni5klVmc/BVSPy335O0FYTZN+mO7iaxT7GFSMKSnkYX1xNbPPB5GlCmuxgxXR\nOdeF9Hvc+NAGfEU88tQk1WD1pqVXt+bRAtggdoXgRb+5wpIZR0O2wmLNl8bY\nwreEfMFOHjMcK1vYe73EZnGSKgy/oBwms5bLwfg53A3Ke3fNy7SnyYXZCVSK\neijl/Jly9cIC032VQCEP7pEH+1BisGerOrEWz+TzYL3NYF5zn9gGicQT8WuW\nCbrqplaFesyFoI6Uk2+FB5fLtxyxjuj8YFduMvO2s+foyx8m75vhOe/WGgaF\nx1Q8IYco3fLK+ZDS5Z0zO25zAsD4guftbchzh2T/h+hh/joN5dV/BxzOpS8f\nugF7Mhn0/ncnaHvouS1EQiyxeIrb8qoFw3IXs0bDtJoNfqs6aqE3xuORGFu+\nHR87gUTfkLDANBfBA2MtCLyMTQw1X+Ms3EQ6wLoefoK9Z8ge0yM+MbO+ki3T\nB4aI\r\n=KnwM\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQC82l1JPjEeq+7ZZkJjA4W3FQQN6Ugrwq2PZGRuHuQWCgIhAO0PBMRdaZUx/85rVPwQQQHKVVkfBbntLtdVZ6yRLjZm"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20200109T111334Z.7bf9919.UL-67_1578568433810_0.03244826374660281"},"_hasShrinkwrap":false},"1.0.0-dev.20200122T143521Z.c6d4e12.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20200122T143521Z.c6d4e12.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191014T122830Z.3b83f5651","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20200109T111055Z.0aac7fa.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"c6d4e12c8185cfd9a5284dd2fa29a2dd3940502e","_id":"ul-imports@1.0.0-dev.20200122T143521Z.c6d4e12.UL-67","_nodeVersion":"10.18.1","_npmVersion":"6.13.4","dist":{"integrity":"sha512-IUQXUXLJkMQyEqoLLiBVn6APx6QC3heDdqHf2hIGwCt0niCExGgqDBYhuhyG6wmxuGIGdAEOutWVZHz8H0qvbg==","shasum":"f0b2bb18ec5ca68257e3bfaa49373d36b77095d5","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20200122T143521Z.c6d4e12.UL-67.tgz","fileCount":217,"unpackedSize":69187376,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJeKF3gCRA9TVsSAnZWagAAYFUQAJpP6cccQfqNBLIkYb8o\nbiywgOZHBzioMUMVDKJpHg8OHrzdEN8VMwAaUOayN5aBj9qOi+xvQroxvUqN\n9RE55JWAUNJTZ2nJKudkTHs0FIfr8TGIV65OIpDEcxRAPsZ9EVPybdqpSryw\nQbCb5JKJ73GgFS0HzGw3ndFsLvVVywrTA0V5N1dLmX8TNMFtCIOMA9x1bldY\nclzbE7HUSLW5naptt9uzxHxQKgD6eu0aKTjmgMOcGa8jn+xWSJB5zCGD0iL6\nEeOFsUFQBzGe/4+ydAJ7Tb7EL+i2lCC/hyXi4Qhoh/V+TDz1QLCERLVQ2iUh\ny81ATg9kA4uE5clr213tw0v2I9GrOSaysuhrYol2GjjvvzCwL3ezVEX8TJ1G\n8wDG2g2ZWGrc78YTJCkSjWZvpK+1b29rYB7erTMPJlhqrQs/6RAa0TSUux7k\ntbQZbNnYYrbmDixilFeeySj1SYWRdF6MKyBnYErrBTjbdaUYzUdYLvs1u+p3\nHFALrbAnpsAFFhaX2cRcmeyp7hZnGjqdOEBrBC5zWGl6RE+o9RRcjodYkDGZ\nUa+7qNLPerH0Nlld9uGxtYdqp5RhyG7mDfuhvfVO6WaAdQT2Htp4WP9FU2HB\nZSsK7t+2HYcbWbFr42cKcKr8DoPWX3cPoGKa5aLn7c2AWAgtwqZIduGY2oz+\nFxwc\r\n=uj/h\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIAtyd5bF3RB8Ft0SWRyEyIjvdggrJecJSqXKJpgWhoifAiEAmzKFxTBXAxz8ZitvIwRMQGK359z7sbR9WNcXhIO1pJo="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20200122T143521Z.c6d4e12.UL-67_1579703775697_0.8016884335418459"},"_hasShrinkwrap":false},"1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"3.4.5","gpii-diff":"1.0.0-dev.20171128T134233Z.8584f1c","gpii-express":"1.0.15","gpii-launcher":"1.0.0-dev.20171221T122035Z.edade23","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191014T122830Z.3b83f5651","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.1","juice":"5.2.0","kettle":"1.11.0","markdown-it":"10.0.0","md5-file":"4.0.0","mkdirp":"0.5.1","nodemailer":"6.3.1","nodemailer-smtp-transport":"2.7.4","pako":"1.0.10","request":"2.88.0","rimraf":"3.0.0"},"devDependencies":{"eslint":"6.5.1","eslint-config-fluid":"1.4.0","fluid-grunt-eslint":"18.1.2","gpii-couchdb-test-harness":"1.0.0-dev.20190827T101451Z.024d2c3.GPII-3531","gpii-express-user":"1.0.2-dev.20191014T143404Z.150969f.GPII-3921","gpii-grunt-lint-all":"1.0.5","grunt":"1.0.4","node-jqunit":"1.1.8","nyc":"14.1.1","ul-api":"1.0.0-dev.20200109T111055Z.0aac7fa.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`gpii-launcher`](https://www.npmjs.com/package/gpii-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`gpii-launcher`](https://github.com/the-t-in-rtf/gpii-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`gpii-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"bd0c4cf008724e34c8e7818036021a71edac7ccb","_id":"ul-imports@1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67","_nodeVersion":"10.19.0","_npmVersion":"6.13.4","dist":{"integrity":"sha512-lBF7uiweH/MAmmtMThLUkaUT3spkCuIG4whY/2htijPckMy/P8h/HNWJEUi30h0LH7OFgGXqX56dKWQhYnxAZQ==","shasum":"64da6a3100740384c5851f9dab3c2b25e2b8a1ea","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67.tgz","fileCount":217,"unpackedSize":69188855,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJejJ/eCRA9TVsSAnZWagAA7zgQAIHLed2lBU1Sa9HVgA9O\n1USQJO/uafDrUYviEMqAdrx2TwYw9pUD1VXe9w4A49Gzesk6ADlmVRUik49j\nbeaqJHs903d4LTDu07emOWLbYN4ARqikNzrciTwmCOuBE6veJaMjooB6UyGb\ntBsDX5VDd5H06OJ51YJ2nDT0FMm7TyXAQ10K/bft7OQYplkS2pAcIe0y6iOH\n4qrOVMnGaW328RNOopTMPLLWWUCPVv4kocLDv8TcMzGx0XGW9BzY87DuBlol\n/zGM2Ya+EBMG54/wKMEqeS2kpQuzlEXAuMfAuf5HyY8cdjdVddZINr7HzOHF\np5MdtVDO47vTUXxYx+P0zt35zfRiOIaMuB86LaFJKW0rkds7j/ZGpSopugRA\ngGaaqq8Ygs/ubMkvZ9M8KUeuoDE3qm9WAx+zz+Hmg9fp/Z9Ukp0HuaVUSOuU\nGdYnIim3Pbkqelt4BX3Qlgzl6OIRVOJxrOGBGtasYA71oynLxIGrSGLdOgyw\nIF29Fz2uORxz6jBWALdgB9nFd0MWUKarapoWlHq4+0OP1aZT2J2+RG7BVAGY\nb4uKSrstxzCfP+WVLgP6DkS7bdOC7oO1UCJE5T1SyNEqQxU1M1nAZYnjU6bu\nAUsEy78ajltf5VVgbx6VHTJpsmwd4ug4u2UVOYqf2NpsWVyLvT5ol7RN/tNO\nYDGB\r\n=f7yB\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQD0qO8jJCrZOHnhvl3ZJst4KLcJX8WBggA9T7tylObSWAIhAL5cMetlWVqRoc/sr/2+4fSk5eaMaMF4aUvP30Dd9b/O"}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67_1586274269994_0.8837598757041616"},"_hasShrinkwrap":false},"1.0.0-dev.20200731T125434Z.5fbc882.UL-67":{"name":"ul-imports","version":"1.0.0-dev.20200731T125434Z.5fbc882.UL-67","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run all-image-imports && npm run sai-curation && npm run curation-unified-orphans","all-imports":"npm run eastin-import && npm run sai-import && npm run abledata-import","abledata-import":"node src/js/abledata/index.js -- --noCache","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","abledata-sync":"npm run abledata-import && npm run curation-known-duplicates && npm run unifier","abledata-legacy-import":"node src/js/abledata/legacy-import-prep.js && node src/js/abledata/index.js","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import && npm run abledata-image-import","abledata-image-import":"node src/js/images/abledata-image-sync.js","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"6.4.1","fluid-diff":"1.0.0","fluid-express":"1.0.16","fluid-launcher":"1.0.0","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191014T122830Z.3b83f5651","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.3","juice":"7.0.0","kettle":"1.15.1","markdown-it":"11.0.0","md5-file":"5.0.0","mkdirp":"1.0.4","nodemailer":"6.4.11","nodemailer-smtp-transport":"2.7.4","pako":"1.0.11","request":"2.88.2","rimraf":"3.0.2"},"devDependencies":{"eslint":"7.5.0","eslint-config-fluid":"1.4.0","eslint-plugin-markdown":"1.0.2","fluid-couchdb-test-harness":"1.1.0","fluid-express-user":"1.0.2-dev.20200731T100501Z.31a286a.FLUID-6524","fluid-grunt-eslint":"18.1.2","fluid-grunt-lint-all":"1.0.8","grunt":"1.2.1","node-jqunit":"1.1.8","nyc":"15.1.0","ul-api":"1.0.0-dev.20200731T124130Z.aa35fff.UL-65"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`fluid-launcher`](https://www.npmjs.com/package/fluid-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`fluid-launcher`](https://github.com/the-t-in-rtf/fluid-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`fluid-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"5fbc882a219e53e6a80cd7b7b489f43714f50572","_id":"ul-imports@1.0.0-dev.20200731T125434Z.5fbc882.UL-67","_nodeVersion":"12.18.2","_npmVersion":"6.14.5","dist":{"integrity":"sha512-8gqpHxHK/X8HUFlMQA/Jyg9vEbbVvKOfM+ZBp3pABUkOMs4IZLMR1YmsAWIsK++mFsca55g6MZr1G9SUZJQKxA==","shasum":"9fd7b3a3f85d36123891557dd60e50b60398549e","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20200731T125434Z.5fbc882.UL-67.tgz","fileCount":217,"unpackedSize":69175179,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfJBShCRA9TVsSAnZWagAALaQP/AyjoFF3v/FNRYXmqChj\nLOdZbDnDOyt22Ee9fluN2lGD65XVlUbPAHf0hBbFAEOnAOmoL4IyMY5Mizsu\nplSr1lO/SO5BI5LKmCpRlqluJyAzMfgDvcWwyjNsrV9aMUvP6ZGH4Y+ytrJj\npC51boKupRzWYj9vX71l0yMlkvYv6pvkwEvNj7UgwBk3/23pJ6QKoDMIwMD3\n6GSKZWhDX3YmAT4HxzyzD2CkX8xtYHRyGEJ9ccZupjD0Rpce71GY4TU9qGVS\nMLtCwwbzUr701Qf+A3KyIvTq1N7tEAUnEbAdfoUorgEVppd160TDMUWY/FC3\nfNU5RQ+HIAon1+0T7md969dXdbnG1x7vJmno0cH3tyZCicRo4JSU5trzJyqy\n7HC3ISVPJyaiu9LiKm05CzeS02KOy+bG1YOi8AeTd/xbOEuLJb50kkeMyH3g\n+fxSJ3vpqsX6QBzyxRVBL3opT42Tf8OQZDf0FS6dIwc1WoolguBn0CXybH1v\nS3W7kV4trHE2+F8/V2/xaL/aByOetD6utNjdzJ2n+wn3dBvJ6wRFU/xYJYo+\nwOnQJsnJJOf581cqIgokaQxLr9gc4qromhaGmD6zysbfZfzVDLhF5SMx+aee\nPAmZlSx0U1ySsyvl6Jk316D0hFG3fOuA5scbP/WPgbULTOaeSuBzMYwE1/fd\n2zMN\r\n=/XCg\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIAEBXHc2v2Jw0dpu1aIv5UjonInGQl/TFw8lx5swzgZpAiA8yYfGsjgEmO/xflflnLjXDoIjS0qE1r1kVWSOn8NTxA=="}]},"maintainers":[{"name":"amb26","email":"amb26npm@ponder.org.uk"},{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"}],"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20200731T125434Z.5fbc882.UL-67_1596200096769_0.6762036524425021"},"_hasShrinkwrap":false},"1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion":{"name":"ul-imports","version":"1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run sai-curation && npm run curation-unified-orphans && npm run all-image-imports ","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"6.4.1","fluid-diff":"1.0.0","fluid-express":"1.0.16","fluid-launcher":"1.0.0","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191014T122830Z.3b83f5651","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.3","juice":"7.0.0","kettle":"1.15.1","markdown-it":"11.0.0","md5-file":"5.0.0","mkdirp":"1.0.4","nodemailer":"6.4.11","nodemailer-smtp-transport":"2.7.4","pako":"1.0.11","request":"2.88.2","rimraf":"3.0.2"},"devDependencies":{"eslint":"7.5.0","eslint-config-fluid":"1.4.0","eslint-plugin-markdown":"1.0.2","fluid-couchdb-test-harness":"1.1.0","fluid-express-user":"1.0.2-dev.20200731T100501Z.31a286a.FLUID-6524","fluid-grunt-eslint":"18.1.2","fluid-grunt-lint-all":"1.0.8","grunt":"1.2.1","node-jqunit":"1.1.8","nyc":"15.1.0","ul-api":"1.0.0-dev.20210413T091744Z.bfa8131.legacy-infusion"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`fluid-launcher`](https://www.npmjs.com/package/fluid-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`fluid-launcher`](https://github.com/the-t-in-rtf/fluid-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`fluid-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"c56b898bfbca9b70c3ccfb1df2d2611207976864","_id":"ul-imports@1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion","_nodeVersion":"15.14.0","_npmVersion":"7.7.6","dist":{"integrity":"sha512-mkbPmtSYHhkYBK+XtUxp8+EJBMm27kidElCcl0gbQ9vN6kYB0iwF5sALfTNbswtnIzcg51r09PvGLktt/UShbA==","shasum":"df4269b2d5d339add671fa270d2c5446bd1fa461","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion.tgz","fileCount":217,"unpackedSize":69177647,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgdX1WCRA9TVsSAnZWagAAONcP/iniPFHRu5tTY1pXbScJ\nPmmPbyTMjV4YYLYCL+/0XS1cR9AiQCGuy8E5bMjeRSf6ZmBhIpdMusa5lHi0\nLLUf4g9i5tkOCJW5m5p595mH9I4BzfPOefwPprbx0dIjpKPWA0ZaVBA3kFc7\nmJ21lCbGsNCvrsslnD2zNEKFNNgAvu2bqoZdmafKt2YaYw86SsBKR0Yir6Sk\nwPpK23ebrfqE373SLPJOYoiVy0aKU9iOYM8fTTz32wCQdhm/FCxut3F+1CYn\nnZ0eCxqIbw2wBBrMIsB7n39Fyedjsx+z4mDubG8usAuLb1y6RrnFOOkkp0+7\n3tiPZBAt6FR0ImbHebFdmKwvNOmzhHkBNQ/yiXJ8XyrbArKaSw7ASd9qg+92\niF/6xBmQ2CcdIz/HMWvz5LCh0Keb8GaO9V4Oobw3qyUL4IoQeV8O95kRIMBb\nTVFP5lrWqxwXHebTS9ALIZud6K3tNlzaOj7AsB9PARnX392ZT+C15qTdTGnD\n43VA8ThhfXGP47Qktjb9+WCgZIjWmXtogIugQW1c8ez7hc/RC6wp0N9Cx2YI\n4u9zcAd37aHUgpEftBafmOsD3bVWcv3kZrkjAzC3thc7k9bMRy9eWDK4+q20\n7Xx6ucAjJSeJeNRZuIicHPOxivAm2+iot4o8uRB9NqfFGMxaAwTWTruPeOMZ\nnWCV\r\n=9I6g\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCY0RUFXXWektXqHvTSJRTsFuu/xBg04e21lbxt+otIvgIhAI3aF52iYfuOJJic/sje8eC8osqenEOtCp/OqSFNTk94"}]},"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"maintainers":[{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},{"name":"amb26","email":"amb26npm@ponder.org.uk"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion_1618312533740_0.7115847166006193"},"_hasShrinkwrap":false},"1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin":{"name":"ul-imports","version":"1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin","description":"The scripts used to import data into the Unified Listing.","main":"index.js","directories":{"test":"tests"},"scripts":{"test":"node node_modules/nyc/bin/nyc.js node tests/all-tests.js","full-sync":"npm run all-imports && npm run curation-known-duplicates && npm run unifier && npm run sai-curation && npm run curation-unified-orphans && npm run all-image-imports ","all-imports":"npm run eastin-import && npm run sai-import","eastin-import":"node src/js/eastin/launcher.js -- --noCache","sai-import":"node src/js/sai/index.js -- --noCache","gari-import":"node src/js/gari/index.js -- --noCache","sai-sync":"npm run sai-import && npm run unifier && npm run sai-curation","unifier":"node src/js/unifier/index.js -- --optionsFile %ul-imports/configs/unifier-prod.json","all-image-imports":"npm run eastin-image-import && npm run sai-image-import","eastin-image-import":"node src/js/images/eastin-image-sync.js","sai-image-import":"node src/js/images/sai-image-sync.js","gari-image-import":"node src/js/images/gari-image-sync.js","curation-check-source-urls":"node src/js/curation/check-source-urls.js","curation-fix-duplicate-children":"node src/js/curation/fix-duplicate-children.js","curation-fix-duplicate-source-urls":"node src/js/curation/fix-duplicate-source-urls.js","curation-fix-missing-creation-date":"node src/js/curation/fix-missing-creation-date.js","curation-invalid-records":"node src/js/curation/invalid-records.js","curation-known-duplicates":"node src/js/curation/fix-known-duplicates.js","curation-unified-orphans":"node src/js/curation/fix-orphaned-unified-records.js","image-curation":"npm run image-curation-missing-files && npm run image-curation-md5-checksums && npm run image-curation-orphaned-files","image-curation-missing-files":"node src/js/curation/delete-image-records-missing-files.js","image-curation-md5-checksums":"node src/js/curation/add-missing-md5-checksums.js","image-curation-orphaned-files":"node src/js/curation/delete-orphaned-image-files.js","sai-curation":"npm run sai-curation-merge && npm run sai-curation-metadata","sai-curation-delete":"node src/js/sai/delete.js","sai-curation-merge":"node src/js/sai/merge.js","sai-curation-metadata":"node src/js/sai/metadata.js","sai-curation-merge-loops":"node src/js/curation/fix-merge-loops.js","sai-curation-missing-records":"node src/js/curation/flag-missing-sai-records-as-deleted.js","sai-curation-duplicate-children":"node src/js/curation/fix-duplicate-children.js","updates-report":"npm run updates-report:diffs && npm run updates-report:reports","updates-report:diffs":"node src/js/updates-report/process-incoming-import-results.js","updates-report:reports":"node src/js/updates-report/process-incoming-diff-results.js","noop":"echo This container should be launched manually using a command like: \\'docker-compose run ul-imports full-sync\\'."},"repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"license":"BSD-3-Clause","bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","dependencies":{"cheerio":"1.0.0-rc.3","fast-csv":"3.4.0","foundation-sites":"6.4.4-rc1","fuse.js":"6.4.1","fluid-diff":"1.0.1-dev.20210427T114423Z.d2f3616.GH-5","fluid-express":"1.0.16","fluid-launcher":"1.0.0","gpii-sort":"1.0.0-dev.20170420T114825Z.804ac19","gpii-universal":"0.3.0-dev.20191014T122830Z.3b83f5651","infusion":"3.0.0-dev.20191009T141140Z.32c9263b4.FLUID-6148","json5":"2.1.3","juice":"7.0.0","kettle":"1.15.1","markdown-it":"11.0.0","md5-file":"5.0.0","mkdirp":"1.0.4","nodemailer":"6.4.11","nodemailer-smtp-transport":"2.7.4","pako":"1.0.11","request":"2.88.2","rimraf":"3.0.2"},"devDependencies":{"eslint":"7.5.0","eslint-config-fluid":"1.4.0","eslint-plugin-markdown":"1.0.2","fluid-couchdb-test-harness":"1.1.0","fluid-express-user":"1.0.2-dev.20200731T100501Z.31a286a.FLUID-6524","fluid-grunt-eslint":"18.1.2","fluid-grunt-lint-all":"1.0.8","grunt":"1.2.1","node-jqunit":"1.1.8","nyc":"15.1.0","ul-api":" 1.0.0-dev.20210420T120913Z.95bff0f.eastin-integration"},"readme":"# Unified Listing Imports\n\nThis package contains scripts that are used to import data from vendor databases, and to curate imported data.\n\n## \"Core\" Imports\n\nThe general process of importing data from a vendor is as follows:\n\n1. Check for a local cache of vendor records (to avoid downloading data too frequently).\n2. If the cache is found, load the records.  If not, download all vendor data in batches, making no more than 100\n   requests simultaneously.\n3. Transform the vendor records into \"source\" records.\n4. Retrieve the list of records for the vendor source(s) and compare them to the newly-retrieved records.  If there are\n   new records, create them.  If there are updated versions of existing records, update them.\n\n## The \"Unifier\"\n\nThrough the web interface and various API endpoints, by default only \"unified\" records are displayed.  The \"unifier\"\nis a script that looks for any \"source\" records that are not already associated with a \"unified\" record.  For each of\nthese \"orphaned\" records it finds, the \"unifier\":\n\n1. Creates a new \"unified\" record using the same \"title\", \"description\", \"manufacturer\", etc. data from the \"source\"\n   record.\n2. Updates the \"source\" record to indicate that it is associated with the new \"unified\" record.\n\n## \"Image\" Imports\n\nThere is a secondary set of scripts that:\n\n1. Extract image data from \"source\" records.\n2. Download the original images and cache them locally as needed.\n3. Update the associated \"unified\" record to indicate that it has image data.\n\nFor this script to work properly, it should be run after the \"unifier\" has completed its work (see below).\n\n## Performing a Full Sync\n\nThe basic process for performing a full sync is as follows:\n\n1. Run each of the \"core\" imports.\n2. Run the \"unifier\".\n3. Run each of the \"image\" imports.\n\n## Syncing Using NPM\n\nThis package has a few `npm` scripts defined to cover each part of the above.  To run a \"full\" sync, use a command like:\n\n`npm run full-sync`\n\nFor individual commands, see the `package.json` file.  Note that all of the npm scripts are hard-coded to use the\n\"production\" configuration settings.  To run in a dev environment, you will need to run the direct commands (see below).\n\n__NOTE:__ The GARI imports are no longer included in a \"full\" sync.\n\n## Syncing Using Direct Commands\n\nThis package provides a range of scripts that use [`fluid-launcher`](https://www.npmjs.com/package/fluid-launcher) to\nrun various commands, and to change the effective options used when running the command using options files,\ncommand-line switches, and environment variables.  The `configs` directory contains a range of configuration files.\nUsually, you will want to work with one of the \"merged\" config files that combines:\n\n1. Options common to all scripts.\n2. Options common to the operating environment (production, development).\n3. Options common to all scripts that work with a particular vendor's data (EASTIN, GARI, et cetera).\n4. Options specific to a particular script.\n\nAll commands provided below are intended to be run from root of the directory in which the repository has been checked\nout.\n\n### EASTIN\n\nThe Unified Listing pulls \"source\" records from [EASTIN](http://www.eastin.eu/en/searches/products/index), which is\nitself a federation of various partner databases ([see their website for details](http://www.eastin.eu/en/partners/index)).\n\nTo import \"core\" data from EASTIN:\n\n1. In production: `npm run eastin-import`\n2. In a development environment: `npm run eastin-import -- --optionsFile %ul-imports/configs/eastin-dev.json`\n\nTo import \"image\" data from EASTIN:\n\n1. In production: `npm run eastin-image-import`\n2. In a development environment: `npm run eastin-image-import -- --optionsFile %ul-imports/configs/eastin-image-sync-dev.json`\n\n### GARI\n\nThe Unified Listing pulls \"source\" records from [GARI](http://www.gari.info).  To import \"core\" data from GARI:\n\n1. In a production environment:  `npm run gari-import`\n2. In a development environment: `npm run gari-import -- --optionsFile %ul-imports/configs/gari-dev.json`\n\nTo import \"image\" data from GARI:\n\n1. In production: `npm run gari-image-import`\n2. In a development environment: `npm run gari-image-import -- --optionsFile %ul-imports/configs/gari-image-sync-dev.json`\n\n### The SAI\n\nThe Shopping and Alerting Aid is a front-end to the Unified Listing to assist users in finding products that meet their\nneeds.  To import \"core\" data from the SAI:\n\n1. In production: `npm run sai-import`\n2. In a development environment: `npm run sai-import -- --optionsFile %ul-imports/configs/sai-dev.json`\n\nTo import \"image\" data from the SAI:\n\n1. In production: `npm run sai-image-import`\n2. In a development environment: `npm run sai-image-import -- --optionsFile %ul-imports/configs/sai-image-sync-dev.json`\n\n### \"Unifier\"\n\nSee above for details regarding the \"unifier\".  To create a \"unified\" record for each \"source\" record that is not\nalready associated with a \"unified\" record:\n\n1. In production: ```npm run unifier```\n2. In a development environment: ```npm run unifier -- --optionsFile %ul-imports/configs/unifier-dev.json```\n\n## Curation Scripts\n\nThis package includes scripts that can be used to detect and where possible clean up particular problems with imported\ndata.  For details, look at the contents of the `src/js/curation` directory.\n\n## SAI\n\nThis package includes scripts that make key updates to \"unified\" records based on data coming from the SAI:\n\n1. For each SAI record that has been flagged as `deleted` in the SAI, the unified record is updated to indicate that it\n   has been deleted.\n2. For each SAI record that has been flagged as a \"duplicate\" of another (\"original\") record:\n    1. The associated unified record is updated to indicate that it has been deleted, and to redirect future requests to\n       the \"original\" record.\n    2. All \"child\" records associated with a \"duplicate\" record are updated to be associated with the \"original\" record\n       instead.\n3. For each SAI record that has an updated `name` or `description`, the associated unified record will have its `name`\n   and `description` updated.\n\nNote that image data associated with \"duplicate\" records is not migrated.\n\nTo run these scripts in production, use the command: ```npm run sai-curation```\n\nPlease note, the curation script does not directly reload the data from the SAI API.  Instead, it compares the SAI\n\"source\" records that have already been imported into the UL with their associated unified records.  To pick up new\ndata, you will need to first run the SAI import or a full sync (see above for details on both).\n\n## Running Scripts with Custom Options\n\nThe scripts in this package use the [`fluid-launcher`](https://github.com/the-t-in-rtf/fluid-launcher) package to allow\nyou set options from the command-line or environment variable.  For example, let's say you want to run a \"full sync\"\nwith a custom password.  You can do this using an argument, or an environment variable.  With an argument, you might\nuse a command like:\n\n```npm run full-sync -- --password myPasswordValue```\n\nNote the `--` characters are required to help npm understand where its arguments end and where the script's arguments\nbegin.\n\nUsing an environment variable to set the same password, you might use a command like:\n\n```PASSWORD=myPasswordValue npm run full-sync```\n\nFor more details about supported options, run any of the scripts in this package with the `--help` argument.  You can\nalso create your own custom options files and use those instead of the included files.  For more details, see the\n`fluid-launcher` documentation.\n\n## Generating \"Updates\" Reports\n\nEach import that is run saves data on updated records to a file.  These files can be used to generate emails describing\neach updated record, and also a rollup HTML report that summarises all the changes for a given import.   This process\ninvolves two steps:\n\n1. Look for unprocessed import output.  Compare the raw updates for a given source to the original records and produce a\n   \"diff\" data file.  Import output is gzipped and archived at the end of this process.\n2. Look for unprocessed \"diff\" data files.  Generate emails and an HTML report for each file.  Incoming \"diff\" data\n   files are then gzipped and archive the file.\n\nThere are npm scripts provided in this package to handle both steps.  There is a rollup script to run these in order,\nwhich is run using a command like `npm run updates-report`.  You can also run the steps individually, see the\n`package.json` file for details.\n","readmeFilename":"README.md","gitHead":"f5a1f921ce6213c1c976c5a3e37a2866caadeb28","_id":"ul-imports@1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin","_nodeVersion":"14.17.1","_npmVersion":"6.14.13","dist":{"integrity":"sha512-AMJmf4ZqPxyrEPhgxzKKAc6ZB5WkG5db3JHtjTCyMrj1WfIeMgO4A5PK00MboNHxZsBZ/l1UVOJhM2dZLBXf0Q==","shasum":"31b59bea142498223352b1ce0a6612049b8bb6bb","tarball":"https://registry.npmjs.org/ul-imports/-/ul-imports-1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin.tgz","fileCount":221,"unpackedSize":69187997,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQCf/loW3LX49b51qMLONh7L7dzCJggF6bhDyC/UgRn5RQIhAKCUz9yfHylrrxCweRHMqq6A8w9ZYa5sfXqu5S81DBWb"}]},"_npmUser":{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},"maintainers":[{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},{"name":"amb26","email":"amb26npm@ponder.org.uk"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/ul-imports_1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin_1632827224609_0.32530754950351537"},"_hasShrinkwrap":false}},"readme":"","maintainers":[{"name":"the-t-in-rtf","email":"tony@raisingthefloor.org"},{"name":"amb26","email":"amb26npm@ponder.org.uk"}],"time":{"modified":"2022-06-28T03:38:44.415Z","created":"2017-01-23T14:16:54.527Z","1.0.0-dev.20170123T141639Z.9e14821":"2017-01-23T14:16:54.527Z","1.0.0-dev.20181025T111840Z.7b93e2f":"2018-10-31T12:39:07.126Z","1.0.0-dev.20181101T100429Z.bd8ed6b":"2018-11-01T10:04:53.227Z","1.0.0-dev.20181107T083400Z.2ed57ad":"2018-11-07T08:34:21.012Z","1.0.0-dev.20181112T151503Z.1e997a2":"2018-11-12T15:15:28.027Z","1.0.0-dev.20181114T133929Z.c780f41":"2018-11-14T13:39:48.994Z","1.0.0-dev.20181114T141610Z.52a303e":"2018-11-14T14:16:27.095Z","1.0.0-dev.20181114T145001Z.b60c93a":"2018-11-14T14:50:42.191Z","1.0.0-dev.20181114T145550Z.9d7148a":"2018-11-14T14:56:03.720Z","1.0.0-dev.20181114T145712Z.f60fc66":"2018-11-14T14:59:55.131Z","1.0.0-dev.20181115T091929Z.759364e":"2018-11-15T09:21:19.500Z","1.0.0-dev.20181205T105745Z.accef71":"2018-12-05T10:58:05.505Z","1.0.0-dev.20181205T115420Z.5af93eb":"2018-12-05T11:54:51.611Z","1.0.0-dev.20181205T131852Z.396b0a9":"2018-12-05T13:19:18.473Z","1.0.0-dev.20181205T132454Z.ad7f2ad":"2018-12-05T13:25:10.452Z","1.0.0-dev.20190227T132007Z.eda3df9.UL-67":"2019-02-27T13:20:19.756Z","1.0.0-dev.20190411T112616Z.51ab84e.UL-67":"2019-04-11T11:26:32.987Z","1.0.0-dev.20190411T114749Z.8f82d90.UL-67":"2019-04-11T11:48:12.436Z","1.0.0-dev.20190416T091249Z.f6879eb.UL-67":"2019-04-16T09:13:05.217Z","1.0.0-dev.20190501T145554Z.f3921d3.UL-67":"2019-05-01T14:58:11.516Z","1.0.0-dev.20190516T120548Z.8a18ee4.UL-67":"2019-05-16T12:06:01.924Z","1.0.0-dev.20190522T134629Z.39fae0a.UL-67":"2019-05-22T13:48:23.251Z","1.0.0-dev.20190531T085455Z.6b1da4e.UL-67":"2019-05-31T08:55:15.737Z","1.0.0-dev.20190604T121938Z.8fb8951.UL-67":"2019-06-04T12:20:06.906Z","1.0.0-dev.20190605T082432Z.51be4ca.UL-67":"2019-06-05T08:24:49.778Z","1.0.0-dev.20190612T092837Z.06181ce.UL-67":"2019-06-12T09:28:58.130Z","1.0.0-dev.20190619T093425Z.0d2b6cd.UL-67":"2019-06-19T09:34:49.370Z","1.0.0-dev.20190624T091248Z.b5d3183.UL-67":"2019-06-24T09:17:49.981Z","1.0.0-dev.20190710T141339Z.4d40489.UL-67":"2019-07-10T14:15:41.977Z","1.0.0-dev.20190805T093416Z.db90ad5.UL-67":"2019-08-05T09:34:29.944Z","1.0.0-dev.20190902T095239Z.cd2d254.UL-67":"2019-09-02T09:53:28.072Z","1.0.0-dev.20191009T120702Z.83fc001.UL-67":"2019-10-09T12:07:15.632Z","1.0.0-dev.20191011T090801Z.65e4c42.UL-67":"2019-10-11T09:08:34.985Z","1.0.0-dev.20191011T163526Z.5ac8d76.UL-67":"2019-10-11T16:35:44.541Z","1.0.0-dev.20191014T134843Z.b03bb6a.UL-67":"2019-10-14T13:49:00.779Z","1.0.0-dev.20191105T131713Z.f747aea.UL-67":"2019-11-05T13:20:12.901Z","1.0.0-dev.20191106T091507Z.5302765.UL-67":"2019-11-06T09:16:50.474Z","1.0.0-dev.20191204T142144Z.22a916b.UL-67":"2019-12-04T14:23:22.075Z","1.0.0-dev.20191212T125642Z.22c7eea.UL-67":"2019-12-12T12:57:02.509Z","1.0.0-dev.20191216T131313Z.2658baa.UL-67":"2019-12-16T13:13:52.317Z","1.0.0-dev.20191217T095812Z.c37ddce.UL-67":"2019-12-17T09:58:47.096Z","1.0.0-dev.20191217T101352Z.ee2e90b.UL-67":"2019-12-17T10:14:17.152Z","1.0.0-dev.20200109T111334Z.7bf9919.UL-67":"2020-01-09T11:13:54.227Z","1.0.0-dev.20200122T143521Z.c6d4e12.UL-67":"2020-01-22T14:36:16.605Z","1.0.0-dev.20200407T154402Z.bd0c4cf.UL-67":"2020-04-07T15:44:30.536Z","1.0.0-dev.20200731T125434Z.5fbc882.UL-67":"2020-07-31T12:54:57.202Z","1.0.0-dev.20210413T111504Z.c56b898.legacy-infusion":"2021-04-13T11:15:34.270Z","1.0.0-dev.20210928T110407Z.f5a1f92.post-eastin":"2021-09-28T11:07:04.972Z"},"homepage":"https://github.com/the-t-in-rtf/ul-import#readme","repository":{"type":"git","url":"git+https://github.com/the-t-in-rtf/ul-import.git"},"author":{"name":"Tony Atkins","email":"tony@raisingthefloor.org"},"bugs":{"url":"https://github.com/the-t-in-rtf/ul-import/issues"},"license":"BSD-3-Clause","readmeFilename":""}