node-get.js | |
---|---|
node.js libraries | var http = require('http'),
sys = require('sys'),
fs = require('fs'),
events = require('events'),
url = require('url'),
path = require('path'); |
Local node-get libraries | var encodings = require('./encodings'),
getutil = require('./getutil');
var default_headers = {
'Accept-Encoding': 'none',
'Connection': 'close',
'User-Agent': 'node-get'
}; |
Get a Get object. Takes an argument, that is either a plain string representing the URI, or an object: | function Get(options) {
if (typeof options == 'string') {
this.uri = options;
this.headers = default_headers;
} else {
if (!options.uri) {
throw Error('uri option required in get constructor');
}
this.uri = options.uri;
this.max_redirs = options.max_redirs || 10;
this.headers = options.headers || default_headers;
}
}
sys.inherits(Get, events.EventEmitter); |
Create or reuse a HTTP client
| Get.prototype.client = function(callback) { |
TODO: handle non http/https protocols | this.uri_o = url.parse(this.uri); |
Validate the URI at this step so that invalid redirects are also caught. | if (!(this.uri_o.protocol &&
(this.uri_o.protocol == 'http:' || this.uri_o.protocol == 'https:') &&
this.uri_o.hostname)) {
return callback({ message: 'Invalid URL.' }, null);
} |
Reuse existing client connections in the case of redirects. | if (this._client &&
(this.uri_o.port || 80) == this._client.port &&
this.uri_o.host == this._client.host) {
return this._client;
}
if (this.uri_o.protocol == 'https') {
this._client = http.createClient(443, this.uri_o.hostname);
client.setSecure('x508_PEM');
} else {
this._client = http.createClient(this.uri_o.port || 80,
this.uri_o.hostname);
}
this._client.headers = this.headers;
return this._client;
};
Get.prototype.guessEncoding = function(location, response) { |
The 'most reliable' measure is probably the end of files, so start off with extname. | var ext = path.extname(location).toLowerCase();
if (encodings.ext[ext]) return encodings.ext[ext];
}; |
Innermost API function of Get
| Get.prototype.perform = function(callback, times) {
if (times > this.max_redirs) {
callback({
message: 'Redirect limit of '
+ this.max_redirs
+ ' reached'
});
return;
}
times = times || 1;
var c = this.client(callback); |
The client can fail if the url is invalid | if (!c) return; |
Handle DNS-level errors, like ECONNREFUSED | c.addListener('error', function(e) {
callback({
message: e.message
}, null);
}); |
TODO: fix when/if gzip is supported. | var request = c.request('GET',
((this.uri_o.pathname || '')
+ (this.uri_o.search || '')
+ (this.uri_o.hash || '')) || '/', {
host: this.uri_o.hostname
}
);
request.addListener('response', function(response) {
if (response.statusCode >= 300 &&
response.statusCode < 400 &&
response.headers.location) { |
RedirectionServers can send a full redirect location or a short form, like a hyperlink. Handle both. | if (url.parse(response.headers.location).protocol) {
this.uri = response.headers.location;
} else {
this.uri = url.resolve(this.uri, response.headers.location);
}
this.perform(callback, times + 1);
return;
} else if (response.statusCode >= 400) { |
failure | callback({
message: 'Server returned HTTP ' + response.statusCode,
statusCode: response.statusCode
}, response);
} else { |
success | response.setEncoding(this.guessEncoding(this.uri, response));
callback(null, response);
}
}.bind(this));
request.end();
}; |
Stream a file to disk
| Get.prototype.toDisk = function(filename, callback) { |
TODO: catch all errors | this.perform(function(err, response, encoding) {
if (err) { callback(err); return; } |
Pump contents from the response stream into a new writestream. | getutil.pump(
response,
fs.createWriteStream(filename),
function(err) {
callback(err, filename);
}
);
});
}; |
Get the contents of a URL as a string
| Get.prototype.asString = function(callback) { |
TODO: catch all errors | this.perform(function(err, response, encoding) {
if (err) { callback(err); return; } |
Fill an array with chunks of data,
and then join it into a string before calling | var out = [];
response.on('data', function(chunk) {
out.push(chunk);
});
response.on('error', function(err) {
callback(err);
});
response.on('end', function() {
callback(null, out.join(''));
});
});
};
module.exports = Get;
|