const fs = require('fs-extra')
const fs2 = require('fs.extra')
const Path = require('path')
const BASE_PATH = process.cwd()
const request = require('request')
const Url = require('url')
const exec = require('child_process').exec
const compress = require('file-compress')
const json2str = require('json-stringify-safe')
const zlib = require('zlib')
const { MD5 } = require('crypto-js')

function md5 () {
	return MD5.apply(null, arguments).toString()
}

function getSaveDir (config, urlPath) {
	var path = (config && config.urls && config.urls[urlPath] && config.urls[urlPath].name) ? config.urls[urlPath].name : md5(urlPath.toLowerCase())
	return path
}

module.exports = {

	run (urlPath, saveDir, cleanRun = true, overwrite = true, archive = false, crawl = {}, crawlFolder = 'web-crawler-statics') {

		console.log(`[ Crawl ] - Evaluating Remote file - ${urlPath}\n`)

		const logFile = Path.join(BASE_PATH, 'logs/web-crawler/success', new Date().getTime() + '.log')
		fs.ensureFileSync(logFile)
		const netLogPath = Path.join(BASE_PATH, 'logs/web-crawler/network', String(new Date().getTime()))
		fs2.mkdirpSync(netLogPath)

		function download (uri, filename, opts={}) {
			opts = Object.assign({
				overwrite: true
			}, opts)

			return new Promise((resolve, reject) => {
				request.head(uri, (err, res, body) => {
					if (err) {
						reject(err)
						return false
					}

					var path = null
					if (fs.existsSync(filename) && fs.statSync(filename).isDirectory()) {
						filename = Path.join(filename, 'index.html')
						var path = Path.parse(filename)
					} else {
						var path = Path.parse(filename)
					}

					if (!path) {
						reject(`[Crawl] - Invalid path ${filename}`)
						return false
					}

					let logFile = Path.resolve(netLogPath, `${path.base}.log`)
					fs.ensureFileSync(logFile)
					fs.appendFileSync(logFile, `${uri}\r\n${filename}\r\n\r\n${json2str(res)}\r\n\r\n${err}\r\n\r\n${body}`)

					var contentType = res.headers['content-type']
					var contentLength = res.headers['content-length']

					// if (!/text|audio|video|image|font|javascript/.test(contentType)) {
					// 	reject(`${contentType} is not static file`)
					// 	return false
					// }
					if (opts.overwrite || !fs.existsSync(path)) {
						fs.ensureDirSync(path.dir)
						if (res.headers['content-encoding'] == 'gzip') {
							var stream = request(uri).pipe(zlib.createUnzip()).pipe(fs.createWriteStream(filename))
						} else {
							var stream = request(uri).pipe(fs.createWriteStream(filename))
						}
						stream.on('close', () => {
							resolve({uri, contentType, contentLength})
						})
						stream.on('error', err => {
							reject(err)
						})
					} else {
						resolve({uri, contentType, contentLength})
					}
				})
			})
		}

		if (cleanRun) {
			fs2.rmrfSync(Path.resolve(BASE_PATH, crawlFolder, saveDir))
		}

		return new Promise((_resolve, _reject) => {
			return new Promise((_res, _rej) => {
				if (crawl && crawl.urls && crawl.urls[urlPath] && crawl.urls[urlPath].lastModified) {
					var earliest = parseInt(crawl.urls[urlPath].lastModified)
					request.head(urlPath, (err, res, body) => {
						if (err) {
							_res()
							return true
						}
						var lastModified = res.headers['last-modified']
						if (lastModified) {
							lastModified = parseInt(new Date(lastModified).getTime() / 1000)
						} else {
							_res()
							return true
						}
						if (lastModified < earliest) {
							_rej(lastModified)
						} else {
							_res()
						}
					})
				} else {
					_res()
				}
			})
			.then(() => {
				exec('node ' + Path.resolve(__dirname, '../node_modules/phantomjs-prebuilt/bin/phantomjs') + ' ' + Path.join(__dirname, 'crawl.js') + ' ' + urlPath, (err, stdout, stderr) => {
					if (err) {
						console.error(stderr)
						_reject(stderr)
						return false
					}

					var urls = stdout.split(/\s*[\r\n]+\s*/)
					urls = urls.filter((val, index) => {
						let url = Url.parse(val)
						if (!url.host) {
							return false
						}
						let clean = val
						if (url.search) {
							clean = clean.replace(url.search, '')
						}
						for (var i in urls) {
							let v = urls[i]
							let u = Url.parse(v)
							let c = v
							if (u.search) {
								c = c.replace(u.search, '')
							}
							if (c == clean) {
								return i == index
							}
						}
						return true
					})

					var cleanUrls = urls.map((val, index) => {
						let clean = val
						let url = Url.parse(val)
						if (url.search) {
							clean = clean.replace(url.search, '')
						}
						return clean
					})

					var promises = []
					var fileMapper = {}
					var fileMapper2 = {}

					urls.forEach(url => {
						if (url) {
							promises.push(new Promise((resolve, reject) => {
								// base64 data stream:
								var mt = null
								if (mt = /^data:(.*?)\/(.*?);base64,(.*)/i.exec(url)) {
									var ext = mt[2]
									var content = mt[3]
									var contentType = mt[1] + '/' + mt[2]
									var contentLength = url.length
									var filename = md5(url)
									var clean = `${filename}.${ext}`
									var path = Path.join(BASE_PATH, crawlFolder, saveDir, clean)
									if (overwrite || !fs.existsSync(path)) {
										if (!fs.existsSync(path)) {
											fs.ensureFileSync(path)
											return fs.writeFile(path, content, 'base64', err => {
												if (!err) {
													fs.appendFileSync(logFile, `${contentType}\t|\t${contentLength}\t|\t${clean}\r\n`)
													fileMapper[md5(clean)] = Path.parse(path).base
													fileMapper2[path] = md5(clean)
													console.log(`[succ]\t${contentLength}\t${url}`)
												} else {
													console.error(`[err]\t${clean}`, err)
												}
												resolve(path)
											})
										}
									}
									resolve(path)
									return false
								}
								// normalize url:
								var path = Url.parse(url)
								var clean = url
								if (path.search) {
									clean = clean.replace(new RegExp('/*' + path.search), '')
								}
								var pathname = path.pathname.replace(/\/+$/, '')
								path = Path.join(BASE_PATH, crawlFolder, saveDir, pathname)
								download(url, path, {
									overwrite: overwrite
								})
								.then(data => {
									fs.appendFileSync(logFile, `${data.contentType}\t|\t${data.contentLength}\t|\t${url}\r\n`)
									fileMapper[md5(clean)] = Path.parse(path).base
									fileMapper2[path] = md5(clean)
									console.log(`[succ]\t${data.contentLength}\t${url}`)
									resolve(path)
								})
								.catch(err => {
									console.error(`[err]\t${url}`, err)
									resolve(path)
								})
							}))
						}
					})

					// do Archive:
					if (archive) {
						return Promise.all(promises).catch(err => {}).then(() => {
							var archiveConfig = require(Path.join(BASE_PATH, archive))
							if (archiveConfig) {
								if (archiveConfig instanceof Array) {
									for (let i in archiveConfig) {
										let acnf = archiveConfig[i]
										for (let j in acnf) {
											let mat = /\{:\s*crawl[_-]save[_-]to\s*:\}/i.exec(j)
											if (mat) {
												archiveConfig[i][j.replace(/\{:\s*crawl[_-]save[_-]to\s*:\}/ig, crawlFolder.replace(/[\.\$\^\[\]\{\}\(\)\\]/g, '\\$0'))] = acnf[j]
												delete acnf[j]
											}
										}
									}
								} else {
									for (let i in archiveConfig) {
										let mat = /\{:\s*crawl[_-]save[_-]to\s*:\}/i.exec(i)
										if (mat) {
											archiveConfig[i.replace(/\{:\s*crawl[_-]save[_-]to\s*:\}/ig, crawlFolder.replace(/[\.\$\^\[\]\{\}\(\)\\]/g, '\\$0'))] = archiveConfig[i]
											delete archive[i]
										}
									}
								}

								fs.appendFileSync(Path.join(BASE_PATH, crawlFolder, saveDir, '.fnmap'), JSON.stringify(fileMapper))
								compress.archive(archiveConfig, fileMapper2)
								console.log('done')
							} else {
								console.error('Please provide a valid archive config file. If you wish not to do archive, then prune --archive option.')
							}
						}).then(() => {
							_resolve()
						})
					} else {
						return Promise.all(promises).then(() => {
							console.log('done')
						}).catch(err => {
							console.log('err done', err)
						}).then(() => {
							_resolve()
						})
					}
				})
			})
			.catch(err => {
				_reject(err)
			})
		})

	},

	evaluate (urlPath, name, save = 'web-crawler-statics', cleanRun = true, overwrite = true, since = false, archive = false, crawl = 'crawl.conf.js') {
		if (!save) {
			save = 'web-crawler-statics'
		}

		var crawlConfig = Path.join(BASE_PATH, crawl)
		if (!fs.existsSync(crawlConfig)) {
			crawlConfig = {urls:{}}
		} else {
			crawlConfig = require(crawlConfig)
			if (!crawlConfig) {
				crawlConfig = {urls:{}}
			}
		}
		if (!crawlConfig.urls || typeof crawlConfig.urls != 'object') {
			crawlConfig.urls = {}
		}

		if (!urlPath && !Object.keys(crawlConfig.urls).length) {
			console.error('[ Crawl ] - Error - Should be at lease one url in your config file or assigned from the command line arg --url, -u')
			return false
		}

		var crawlFolder = crawlConfig['crawl-save-to'] ? crawlConfig['crawl-save-to'] : save

		if (!(urlPath in crawlConfig.urls)) {
			crawlConfig.urls[urlPath] = {name: name}
			if (since) {
				crawlConfig.urls[urlPath].lastModified = since
			}
		}

		urls = Object.keys(crawlConfig.urls)
		var doRun = (index) => {
			if (index == urls.length) {
				return false
			}
			var url = urls[index]
			var saveDir = getSaveDir(crawlConfig, url)
			return this.run(url, saveDir, cleanRun, overwrite, archive, crawlConfig, crawlFolder).catch(() => {}).then(() => doRun(index + 1))
		}

		doRun(0)
	}

}
