node-get.js | |
---|---|
node.js libraries | var http = require('http'),
sys = require('sys'),
fs = require('fs'),
events = require('events'),
url = require('url'),
path = require('path'); |
Local node-get libraries | var encodings = require('./encodings'),
getutil = require('./getutil'); |
A wrapper for response that autodetects certain files that are often served by nonrepresentative URLs (zip files) | function AutoResponse(response, def) {
var zipdetect = function(chunk) {
response.removeListener('data', zipdetect); |
Detect zip files, by detecting their common four-byte header | if (chunk.charCodeAt(0) == 80 &&
chunk.charCodeAt(1) == 75 &&
chunk.charCodeAt(2) == 3 &&
chunk.charCodeAt(3) == 4) {
response.setEncoding('binary');
if (this.stream) {
this.stream.encoding = 'binary';
}
} else { |
Otherwise, assume that the value is the default given as def | response.setEncoding(def);
if (this.stream) {
this.stream.encoding = def;
}
}
this.emit('data', chunk);
};
response.on('data', zipdetect);
return new SkipFirst(response);
} |
An odd object - a wrapper for an event emitter
that doesn't emit the first | function SkipFirst(em) {
var first = true,
that = this;
this.encoding = em._decoder.encoding;
em.on('data', function(chunk) {
if (!first) {
that.emit('data', chunk);
} else {
first = false;
}
});
em.on('end', function() { that.emit('end') });
} |
Get a Get object. Takes an argument, that is either an object with a required 'uri' property, or a string that's taken to be a URI | function Get(options) {
if (typeof options == 'string') {
this.uri = options;
} else {
if (!options.uri) {
throw Error('uri option required in get constructor');
}
this.uri = options.uri;
}
}
sys.inherits(Get, events.EventEmitter);
sys.inherits(AutoResponse, events.EventEmitter);
sys.inherits(SkipFirst, events.EventEmitter); |
Create or reuse a HTTP client
| Get.prototype.client = function() { |
TODO: handle non http/https protocols | this.uri_o = url.parse(this.uri); |
Reuse existing client connections in the case of redirects. | if (this._client &&
(this.uri_o.port || 80) == this._client.port &&
this.uri_o.host == this._client.host) {
return this._client;
} |
TODO: move | this.file_ext = path.extname(this.uri_o.pathname).toLowerCase();
if (this.uri_o.protocol == 'https') {
this._client = http.createClient(443, this.uri_o.hostname);
client.setSecure('x508_PEM');
} else {
this._client = http.createClient(this.uri_o.port || 80,
this.uri_o.hostname);
}
return this._client;
}; |
Innermost API function of Get
| Get.prototype.perform = function(callback, times) { |
TODO: make configurable | if (times > 10) callback({ message: 'Redirect limit reached' });
times = times || 1;
var c = this.client(),
that = this; |
TODO: fix when/if gzip is supported. | c.headers = {
'Accept-Encoding': 'none',
'Connection': 'close',
'User-Agent': 'node-get'
};
var request = c.request('GET',
((this.uri_o.pathname || '')
+ (this.uri_o.search || '')
+ (this.uri_o.hash || '')) || '/', {
host: this.uri_o.hostname
}
);
request.end();
request.addListener('response', function(response) {
if (response.statusCode >= 300 &&
response.statusCode < 400 &&
response.headers.location) { |
redirection | if (url.parse(response.headers.location).protocol) { |
The redirect is a full location | that.uri = response.headers.location;
} else { |
Follow the redirect like a link | that.uri = url.resolve(that.uri, response.headers.location);
}
that.perform(callback, times + 1);
return;
} else if (response.statusCode >= 400) { |
failure | callback({
statusCode: response.statusCode
}, response);
} else { |
success | response.setEncoding('binary');
callback(null,
new AutoResponse(response,
encodings[that.file_ext] || 'utf-8'));
}
});
}; |
Stream a file to disk
| Get.prototype.toDisk = function(filename, callback) { |
TODO: catch all errors | this.perform(function(err, response) {
if (err) { callback(err); return; }
var ws = fs.createWriteStream(filename);
response.stream = ws;
getutil.pump(response,
ws, function(err) {
callback(err, filename);
});
});
}; |
Get the contents of a URL as a string
| Get.prototype.asString = function(callback) { |
TODO: catch all errors | this.perform(function(err, response) {
if (err) { callback(err); return; }
var out = [];
response.on('data', function(chunk) {
out.push(chunk);
});
response.on('end', function() { |
TODO warn for encoding non-strings | callback(null, out.join(''));
});
});
};
module.exports = Get;
|