// Dependencies
var cheerio = require('cheerio');
var	request = require('request');
var fs = require('fs');
var rp = require('request-promise');
var dateTime = require('node-datetime');
var fs = require('fs');
var targz = require('tar.gz');
var exec = require('child_process').exec;
var mkdirp = require('mkdirp');
var tar = require('tar-fs');
//define variables
var pinterest_images =[];
var unsplash_images =[];
var final_unsplash_images =[];
var twitter_images =[];
var flickr_images = [];
var final_flickr_images = [];
var pinterest_images =[];
var final_array =[];
var high_resolution_images =[];
var hashtag; // search word for each source to be scraped
var limit; // limit of images to be returned to user
var download_directory = './images/'; // specify download directory
var trending_hashtags =[];
var most_popular_hashtag;

var exports = module.exports = {};

exports.most_popular_hashtag = function() {

  var options = { // specify url to scrape and load into cheerio
      uri: 'https://hashtagnow.co/',
      transform: function (body) {
          return cheerio.load(body);
      }
  };
  return rp(options)
      .then(function ($) {
        $('a','div.trending-hashtags-twitter').each(function(){
          var img = $(this).attr('href');
          // hashtags begin with /hashtag/ so this is removed, leaving only the name of the hashtag
          var trim = img.slice(9);
          //push to array after trim
          trending_hashtags.push(trim);
        })

        // most popular hashtag
        most_popular_hashtag = trending_hashtags[0];
        // return to user
        return most_popular_hashtag;
    })
};

  // export functionality for scraping multiple hashtags from https://hashtagnow.co/
  exports.multiple_trending_hashtags = function() {
    var trending = [];


    var options = { // specify url to scrape and load into cheerio
        uri: 'https://hashtagnow.co/',
        transform: function (body) {
            return cheerio.load(body);
        }
    };
    return rp(options)
        .then(function ($) {
          $('a','div.trending-hashtags-twitter').each(function(){
            var img = $(this).attr('href');
            // hashtags begin with /hashtag/ so this is removed, leaving only the name of the hashtag
            var trim = img.slice(9);
            //push to array after trim
            trending.push(trim);
          })

          // return all of the scraped hashtags to the user
          return trending;
      })
  };



// export functionality for scraping flickr.com
exports.flickr = function(hashtag,compress,limit) {
  var flickr_images = []; // orginal array before it is trimmed
  var final_flickr_images = []; // final array of images from flickr.com

  var options = { // specify url to scrape and load into cheerio
      uri: 'https://www.flickr.com/search/?text=' +hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };
  return rp(options)
      .then(function ($) {
        $('div.photo-list-photo-view').each(function(){
          var img = $(this).attr('style');
          // add all images found in the above div under the 'style' attribute to an array
          flickr_images.push(img);
        })


        for(var i = 0; i < flickr_images.length; i++ ){
            // get the text in between the brackets (this is the URL)
            var string_url = flickr_images[i].substring(flickr_images[i].lastIndexOf("(")+1,flickr_images[i].lastIndexOf(")"));
            // trim the forward slash at the beginning
            var final_strings = string_url.slice(2);
            // add http:// to beginning of each string so it can be recognised at a url
             final_strings = 'http://'+final_strings;
            // push to final array
            final_flickr_images.push(final_strings);
        }

        // check for limit, if defined trim the number of images returned
        if(limit){
          final_flickr_images = final_flickr_images.slice(0, limit);
        }

        // get current date and time
        var dt = dateTime.create();
        // format date and time
        var formatted = dt.format('Y-m-d-H:M:S');

         // make folder with the name as the current date and time, place in images folder
         var mkdir = 'mkdir -p ' + download_directory+formatted;
         // set permissions for folder
         var child = exec(mkdir,'0777', function(err, stdout, stderr) {
             for(var i = 0; i < final_flickr_images.length; i++ ){
             // loop through array of images and add to folder
             request(final_flickr_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/flickr'+hashtag+ '' +i+ '.jpg'));
            }

            // check if compress variable is defined, if so, compress the folder which was just created
              if(compress ==true){
               // read folder from directory
                var read = targz().createReadStream("./images/"+formatted);
                // create tarball
                var write = fs.createWriteStream(formatted+'.tar.gz');
                read.pipe(write);
                }
         });
       // return final array of images
       return final_flickr_images;
    })
};


// export functionality for scraping unsplash.com
exports.unsplash = function(hashtag,compress,limit) {
  var unsplash_images =[];
  var final_unsplash_images =[];

  var options = { // specify url to scrape and load into cheerio
      uri: 'https://unsplash.com/search/'+hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };
  return rp(options)
      .then(function ($) {
        $('a','div.Nqw-T').each(function(){
          var img = $(this).attr('href');
          // add all images found in the above div under the 'href' attribute to an array
          unsplash_images.push(img);
      })


      // loop through array
      for (var i=unsplash_images.length-1; i>=0; i--) {
      // remove all that do not being with 'http' (get all images)
        if (unsplash_images[i].substring(0, 4) !== 'http'){
      unsplash_images.splice(i, 1);
      }
    }
    // loop through array and trim
    // standard unsplash url is "http://unsplash.com/photos/LW3FskrgQ9M/download?force=true"
    // this trims the string to cut off everything past 'download'
    // all urls have a length of 47 up to this point
     for(var i = 0; i < unsplash_images.length; i++ ){
        var subStr = unsplash_images[i].substring(0,47);
        final_unsplash_images.push(subStr);
    }
    // check for limit, if defined limit the number of images returned
    if(limit){
      final_unsplash_images = final_unsplash_images.slice(0, limit);
    }

    // get current date and time
    var dt = dateTime.create();
    // format date and time
    var formatted = dt.format('Y-m-d-H:M:S');

     // make folder with the name as the current date and time, place in images folder
     var mkdir = 'mkdir -p ' + download_directory+formatted;
     // set permissions for folder
     var child = exec(mkdir,'0777', function(err, stdout, stderr) {
           for(var i = 0; i < final_unsplash_images.length; i++ ){
           request(final_unsplash_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/unsplash'+hashtag+ '' +i+ '.jpg'));
            }
            if(compress ==true){

              var read = targz().createReadStream("./images/"+formatted);
              var write = fs.createWriteStream(formatted+'.tar.gz');
              read.pipe(write);
            }
       });
        //return the final array aof images from unsplash.com
        return final_unsplash_images;
    })
};


// export functionality for scraping pinterest.com
exports.pinterest = function(hashtag,compress,limit) {
  var pinterest_images =[];

  var options = {
      uri: 'https://www.pinterest.com/search/pins/?q='+hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };


  return rp(options)
      .then(function ($) {
        $('img','div.SearchPageContent').each(function(){
          var img = $(this).attr('src');
          // add all images found in the above div under the 'src' attribute to an array
          pinterest_images.push(img);
      })

      // remove all elements returned which do not begin with https as these will not be image URL's
      for (var i=pinterest_images.length-1; i>=0; i--) {
        if (pinterest_images[i].substring(0, 5) !== 'https') {
            pinterest_images.splice(i, 1);
        }
      }


     //images are made smaller as standard, changing the size will result in higher resolution
      for(var i=0; i < pinterest_images.length; i++) {
       pinterest_images[i] = pinterest_images[i].replace(/236x/g, '564x');
      }

     // check for limit, if defined limit the number of images returned
     if(limit){
       pinterest_images = pinterest_images.slice(0, limit);
     }
     // get current date and time
     var dt = dateTime.create();
     // format date and time
     var formatted = dt.format('Y-m-d-H:M:S');

      // make folder with the name as the current date and time, place in images folder
      var mkdir = 'mkdir -p ' + download_directory+formatted;
      // set permissions for folder
      var child = exec(mkdir,'0777', function(err, stdout, stderr) {

          for(var i = 0; i < pinterest_images.length; i++ ){
            // create an image for each url & add to images directory under the current date and time folder
           request(pinterest_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/pinterest'+hashtag+ '' +i+ '.jpg'));
            }
            if(compress ==true){
              // read images folder
              var read = targz().createReadStream("./images/"+formatted);
              // create tarball
              var write = fs.createWriteStream(formatted+'.tar.gz');


              read.pipe(write);
            }
       });

       //return final images
       return pinterest_images;

    })
};

// export functionality for scraping twitter.com
exports.twitter = function(hashtag,compress,limit) {
  var twitter_images =[];
  var options = {
      uri: 'https://twitter.com/search?q=%23'+hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };


  return rp(options)
      .then(function ($) {

      	$('img','div.stream').each(function(){
          var img = $(this).attr('src');

          twitter_images.push(img);
      })

      // Twitter returns many images which are unwanted (gifs/profile images etc.)
      // The desired images begin with this string, which is what I am searching for
      var search_term = 'https://pbs.twimg.com/media';

      // loop through array and remove all strings which do not begin with the above search term
			for (var i=twitter_images.length-1; i>=0; i--) {
    		if (twitter_images[i].substring(0, 27) !== search_term) {
        twitter_images.splice(i, 1);
        }
      }

      // chec if limit is defined and adjust array accordingly if it is
      if(limit){
        twitter_images = twitter_images.slice(0, limit);
      }

      // get current date and time
      var dt = dateTime.create();
      // format date and time
      var formatted = dt.format('Y-m-d-H:M:S');

       // make folder with the name as the current date and time, place in images folder
       var mkdir = 'mkdir -p ' + download_directory+formatted;
       // set permissions for folder
       var child = exec(mkdir,'0777', function(err, stdout, stderr) {

          for(var i = 0; i < twitter_images.length; i++ ){
           request(twitter_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/twitter'+hashtag+ '' +i+ '.jpg'));
            }

            // check if compress variable is set to true, compress if it is
            if(compress ==true){
              // read images directory
              var read = targz().createReadStream("./images/"+formatted);
              //create tarball
              var write = fs.createWriteStream(formatted+'.tar.gz');
              read.pipe(write);
            }
       });
       // return final set of images to user
       return twitter_images;

    })
};







exports.all = function(hashtag,compress,limit) {
  var pinterest_images =[];
  var twitter_images =[];

  var pinterest_options = {
      uri: 'https://www.pinterest.com/search/pins/?q='+hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };

  var twitter_options = {
      uri: 'https://twitter.com/search?q=%23'+hashtag,
      transform: function (body) {
          return cheerio.load(body);
      }
  };



  return rp(pinterest_options,twitter_options)
      .then(function ($) {
        $('img','div.SearchPageContent').each(function(){
          var img = $(this).attr('src');
          // add all images found in the above div under the 'src' attribute to an array
          pinterest_images.push(img);
      })

      $('img','div.stream').each(function(){
        var img = $(this).attr('src');

        twitter_images.push(img);
    })

    // remove all elements returned which do not begin with https as these will not be image URL's
      for (var i=pinterest_images.length-1; i>=0; i--) {
        if (pinterest_images[i].substring(0, 5) !== 'https') {
            pinterest_images.splice(i, 1);
        }
      }


     //images are made smaller as standard, changing the size will result in higher resolution
      for(var i=0; i < pinterest_images.length; i++) {
       pinterest_images[i] = pinterest_images[i].replace(/236x/g, '564x');
      }

     // check for limit, if defined limit the number of images returned
     if(limit){
       pinterest_images = pinterest_images.slice(0, limit);
     }

     // Twitter returns many images which are unwanted (gifs/profile images etc.)
     // The desired images begin with this string, which is what I am searching for
     var search_term = 'https://pbs.twimg.com/media';

     // loop through array and remove all strings which do not begin with the above search term
    for (var i=twitter_images.length-1; i>=0; i--) {
      if (twitter_images[i].substring(0, 27) !== search_term) {
       twitter_images.splice(i, 1);
       }
     }

     // chec if limit is defined and adjust array accordingly if it is
     if(limit){
       twitter_images = twitter_images.slice(0, limit);
     }
     // get current date and time
     var dt = dateTime.create();
     // format date and time
     var formatted = dt.format('Y-m-d-H:M:S');

      // make folder with the name as the current date and time, place in images folder
      var mkdir = 'mkdir -p ' + download_directory+formatted;
      // set permissions for folder
      var child = exec(mkdir,'0777', function(err, stdout, stderr) {

          for(var i = 0; i < pinterest_images.length; i++ ){
            // create an image for each url & add to images directory under the current date and time folder
           request(pinterest_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/pinterest'+hashtag+ '' +i+ '.jpg'));
            }

            for(var i = 0; i < twitter_images.length; i++ ){
              // create an image for each url & add to images directory under the current date and time folder
             request(twitter_images[i]).pipe(fs.createWriteStream(download_directory+formatted+'/twitter'+hashtag+ '' +i+ '.jpg'));
              }
            if(compress ==true){
              // read images folder
              var read = targz().createReadStream("./images/"+formatted);
              // create tarball
              var write = fs.createWriteStream(formatted+'.tar.gz');


              read.pipe(write);
            }
       });

       //return final images
       return pinterest_images,twitter_images;

    })
};

// export all of the functionality
module.exports;
