Jump To …

request.js

The request object encapsulates a request, creating a Node.js HTTP request and then handling the response.

 
var HTTP = require("http")
  , HTTPS = require("https")
  , URL = require("url")
  , QueryString = require("querystring")
  , Emitter = require('events').EventEmitter
  , sprintf = require("sprintf").sprintf
  , _ = require("underscore")
  , Response = require("./response")
  , HeaderMixins = require("./mixins/headers")
  , Content = require("./content")
;

The Shred object itself constructs the Request object. You should rarely need to do this directly.

 
var Request = function(options) {
  this.log = options.logger;
  processOptions(this,options||{});
  createRequest(this);
};

A Request has a number of properties, many of which help with details like URL parsing or defaulting the port for the request.

 
Object.defineProperties(Request.prototype, {
  • url. You can set the url property with a valid URL string and all the URL-related properties (host, port, etc.) will be automatically set on the request object.
 
  url: {
    get: function() {
      if (!this.scheme) { return null; }
      return sprintf("%s://%s:%s%s",
          this.scheme, this.host, this.port,
          (this.proxy ? "/" : this.path) +
          (this.query ? ("?" + this.query) : ""));
    },
    set: function(_url) {
      _url = URL.parse(_url);
      this.scheme = _url.protocol.slice(0,-1);
      this.host = _url.hostname;
      this.port = _url.port;
      this.path = _url.pathname;
      this.query = _url.query;
      return this;
    },
    enumerable: true
  },
  • headers. Returns a hash representing the request headers. You can't set this directly, only get it. You can add or modify headers by using the setHeader or setHeaders method. This ensures that the headers are normalized - that is, you don't accidentally send Content-Type and content-type headers. Keep in mind that if you modify the returned hash, it will not modify the request headers.
 
    
  headers: { 
    get: function() { 
      return this.getHeaders();
    }, 
    enumerable: true 
  },
  • port. Unless you set the port explicitly or include it in the URL, it will default based on the scheme.
 
  port: {
    get: function() {
      if (!this._port) {
        switch(this.scheme) {
          case "https": return this._port = 443;
          case "http":
          default: return this._port = 80;
        }
      }
      return this._port;
    },
    set: function(value) { this._port = value; return this; },
    enumerable: true
  },
  • method. The request method - get, put, post, etc. that will be used to make the request. Defaults to get.
 
  method: {
    get: function() {
      return this._method = (this._method||"GET");
    },
    set: function(value) {
      this._method = value; return this;
    },
    enumerable: true
  },
  • query. Can be set either with a query string or a hash (object). Get will always return a properly escaped query string or null if there is no query component for the request.
 
  query: {
    get: function() {return this._query;},
    set: function(value) {
      if (value) {
        if (typeof value === 'object') {
          value = QueryString.stringify(value);
        } else {
          value = QueryString.stringify(QueryString.parse(value.toString()))
        }
      }
      if (value) { this._query = "?" + value; }
      else this._query = "";
      return this;
    },
    enumerable: true
  },
  • parameters. This will return the query parameters in the form of a hash (object).
 
  parameters: {
    get: function() { return QueryString.parse(this._query||""); },
    enumerable: true
  },
  • content. (Aliased as body.) Set this to add a content entity to the request. Attempts to use the content-type header to determine what to do with the content value. Get this to get back a Content object.
 
  body: {
    get: function() { return this._body; },
    set: function(value) {
      this._body = new Content({
        data: value,
        type: this.getHeader("Content-Type")
      });
      this.setHeader("Content-Type",this.content.type);
      this.setHeader("Content-Length",this.content.length);
      return this;
    },
    enumerable: true
  },
  • timeout. Used to determine how long to wait for a response. Does not distinguish between connect timeouts versus request timeouts. Set either in milliseconds or with an object with temporal attributes (hours, minutes, seconds) and convert it into milliseconds. Get will always return milliseconds.
 
  timeout: {
    get: function() { return this._timeout; }, // in milliseconds
    set: function(timeout) {
      var request = this
        , milliseconds = 0;
      ;
      if (!timeout) return this;
      if (typeof options=="number") { milliseconds = options; }
      else {
        milliseconds = (options.milliseconds||0) +
          (1000 * ((options.seconds||0) +
              (60 * ((options.minutes||0) +
                (60 * (options.hours||0))))));
      }
      this._timeout = milliseconds;
      return this;
    },
    enumerable: true
  }
});

Alias body property to content. Since the content object has a body attribute, it's preferable to use content since you can then access the raw content data using content.body.

 
Object.defineProperty(Request.prototype,"content",
    Object.getOwnPropertyDescriptor(Request.prototype, "body"));

The Request object can be pretty overwhelming to view using the built-in Node.js inspect method. We want to make it a bit more manageable. This probably goes too far in the other direction.

 
_.extend(Request.prototype,{
  inspect: function() {
    var request = this;
    var headers = _(request.headers).reduce(function(array,value,key){
      array.push("\t" + key + ": " + value); return array;
    },[]).join("\n");
    var summary = ["<Shred Request> ", request.method.toUpperCase(),
        request.url].join(" ")
    return [ summary, "- Headers:", headers].join("\n");
  }
});

Allow chainable 'on's: shred.get({ ... }).on( ... ). You can pass in a single function, a pair (event, function), or a hash: { event: function, event: function }

 
_.extend(Request.prototype,{
  on: function(eventOrHash, listener) {
    var emitter = this.emitter;

Pass in a single argument as a function then make it the default response handler

 
    if (arguments.length === 1 && typeof(eventOrHash) === 'function') {
      emitter.on('response', eventOrHash);
    } else if (arguments.length === 1 && typeof(eventOrHash) === 'object') {
      _(eventOrHash).each(function(value,key) {
        emitter.on(key,value);
      });
    } else {
      emitter.on(eventOrHash, listener);
    }
    return this;
  }
});

Add in the header methods. Again, these ensure we don't get the same header multiple times with different case conventions.

 
HeaderMixins.gettersAndSetters(Request);

processOptions is called from the constructor to handle all the work associated with making sure we do our best to ensure we have a valid request.

 
var processOptions = function(request,options) {

  request.log.debug("Processing request options ..");

We'll use request.emitter to manage the on event handlers.

 
  request.emitter = (new Emitter);

Set up the handlers ... 

  if (options.on) {
    _(options.on).each(function(value,key) {
      request.emitter.on(key,value);
    });
  }

Make sure we were give a URL or a host

 
  if (!options.url && !options.host) {
    request.emitter.emit("error",
        new Error("No url or url options (host, port, etc.)"));
    return;
  }

Allow for the use of a proxy.

 
  
  if (options.url) {
    if (options.proxy) {
      request.url = options.proxy;
      request.path = options.url;
    } else {
      request.url = options.url;
    }
  }

Set the remaining options.

 
  request.query = options.query||options.parameters;
  request.method = options.method;
  request.setHeader("user-agent",options.agent||"Shred for Node.js, Version 0.5.0");
  request.setHeaders(options.headers);

The content entity can be set either using the body or content attributes.

 
  if (options.body||options.content) {
    request.content = options.body||options.content;
  }
  request.timeout = options.timeout;

};

createRequest is also called by the constructor, after processOptions. This actually makes the request and processes the response, so createRequest is a bit of a misnomer.

 
var createRequest = function(request) {
  var timeout
  ;

  request.log.debug("Creating request ..");
  request.log.debug(request);

Choose which Node.js standard library to use. Warning: Shred has not been tested much with https.

 
  var http = request.scheme == "http" ? HTTP : HTTPS;

Set up the real request using the selected library. The request won't be sent until we call .end().

 
  request._raw = http.request({
    host: request.host,
    port: request.port,
    method: request.method,
    path: request.path+request.query,
    headers: request.getHeaders()

Provide a response handler.

 
  }, function(response) {

    request.log.debug("Received response ..");

We haven't timed out and we have a response, so make sure we clear the timeout so it doesn't fire while we're processing the response.

 
    clearTimeout(timeout);

Construct a Shred Response object from the response. This will stream the response, thus the need for the callback. We can access the response entity safely once we're in the callback.

 
    response = new Response(response, request, function(response) {

Set up some event magic. The precedence is given first to status-specific handlers, then to responses for a given event, and then finally to the more general response handler. In the last case, we need to first make sure we're not dealing with a a redirect.

 
      var emit = function(event) {
        if (request.emitter.listeners(response.status).length>0) {
          request.emitter.emit(response.status,response);
        } else {
          if (request.emitter.listeners(event).length>0) {
            request.emitter.emit(event,response);
          } else if (!response.isRedirect) {
            request.emitter.emit("response",response);
          }
        }
      };

Next, check for a redirect. We simply repeat the request with the URL given in the Location header. We fire a redirect event.

 
      if (response.isRedirect) {
        request.log.debug("Redirecting to "
            + response.getHeader("Location"));
        request.url = response.getHeader("Location");
        emit("redirect");
        createRequest(request);

Okay, it's not a redirect. Is it an error of some kind?

 
      } else if (response.isError) { 
        emit("error");
      } else {

It looks like we're good shape. Trigger the success event.

 
        emit("success");
      }
    });
  });

We're still setting up the request. Next, we're going to handle error cases where we have no response. We don't emit an error event because that event takes a response. We don't response handlers to have to check for a null value. However, we should introduce a different event type for this type of error.

 
  request._raw.on("error", function(error) { 
    request.log.error("Request failed: " + error.message); 
  });

We're almost there. Next, we need to write the request entity to the underlying request object.

 
  if (request.content) {
    request.log.debug("Streaming body: '" +
        request.content.body.slice(0,59) + "' ... ");
    request._raw.write(request.content.body);
  }

Finally, we need to set up the timeout. We do this last so that we don't start the clock ticking until the last possible moment.

 
  if (request.timeout) {
    timeout = setTimeout(function() {
      request.log.debug("Timeout fired, aborting request ...");
      request._raw.abort();
      request.emit("timeout", request);
    },request.timeout);
  }

The .end() method will cause the request to fire. Technically, it might have already sent the headers and body.

 
  request.log.debug("Sending request ...");
  request._raw.end();
};

module.exports = Request;