Jump To …
node-get-file.js encodings.js getutil.js index.js node-get.js

# node-get.js |
--- | ---
¶ node.js libraries |
    var http = require('http'),
        sys = require('sys'),
        fs = require('fs'),
        events = require('events'),
        url = require('url'),
        path = require('path');
¶ Local node-get libraries |
    var encodings = require('./encodings'),
        getutil = require('./getutil');
¶ A wrapper for response that autodetects certain files that are often served by nonrepresentative URLs (zip files) |
    function AutoResponse(response, def) {
        var zipdetect = function(chunk) {
            response.removeListener('data', zipdetect);
¶ Detect zip files, by detecting their common four-byte header |
            if (chunk.charCodeAt(0) == 80 &&
                chunk.charCodeAt(1) == 75 &&
                chunk.charCodeAt(2) == 3 &&
                chunk.charCodeAt(3) == 4) {
                response.setEncoding('binary');
                if (this.stream) {
                    this.stream.encoding = 'binary';
                }
            } else {
¶ Otherwise, assume that the value is the default given as def |
                response.setEncoding(def);
                if (this.stream) {
                    this.stream.encoding = def;
                }
            }
            this.emit('data', chunk);
        };
        response.on('data', zipdetect);
        return new SkipFirst(response);
    }
¶ An odd object - a wrapper for an event emitter that doesn't emit the first `data` event sent. |
    function SkipFirst(em) {
        var first = true,
            that = this;
        this.encoding = em._decoder.encoding;
        em.on('data', function(chunk) {
            if (!first) {
                that.emit('data', chunk);
            } else {
                first = false;
            }
        });
        em.on('end', function() { that.emit('end') });
    }
¶ Get a Get object. Takes an argument, that is either an object with a required 'uri' property, or a string that's taken to be a URI |
    function Get(options) {
        if (typeof options == 'string') {
            this.uri = options;
        } else {
            if (!options.uri) {
                throw Error('uri option required in get constructor');
            }
            this.uri = options.uri;
        }
    }

    sys.inherits(Get, events.EventEmitter);
    sys.inherits(AutoResponse, events.EventEmitter);
    sys.inherits(SkipFirst, events.EventEmitter);
¶ Create or reuse a HTTP client @return {http.Client} a proper client. |
    Get.prototype.client = function() {
¶ TODO: handle non http/https protocols |
        this.uri_o = url.parse(this.uri);
¶ Reuse existing client connections in the case of redirects. |
        if (this._client &&
            (this.uri_o.port || 80) == this._client.port &&
            this.uri_o.host == this._client.host) {
            return this._client;
        }
¶ TODO: move |
        this.file_ext = path.extname(this.uri_o.pathname).toLowerCase();

        if (this.uri_o.protocol == 'https') {
            this._client = http.createClient(443, this.uri_o.hostname);
            client.setSecure('x508_PEM');
        } else {
            this._client = http.createClient(this.uri_o.port || 80,
                this.uri_o.hostname);
        }

        return this._client;
    };
¶ Innermost API function of Get @param {Function} callback @param {Number} times number of times re-called. |
    Get.prototype.perform = function(callback, times) {
¶ TODO: make configurable |
        if (times > 10) callback({ message: 'Redirect limit reached' });
        times = times || 1;

        var c = this.client(),
            that = this;
¶ TODO: fix when/if gzip is supported. |
        c.headers = {
            'Accept-Encoding': 'none',
            'Connection': 'close',
            'User-Agent': 'node-get'
        };

        var request = c.request('GET',
             ((this.uri_o.pathname || '')
            + (this.uri_o.search || '')
            + (this.uri_o.hash || '')) || '/', {
                host: this.uri_o.hostname
            }
        );

        request.end();
        request.addListener('response', function(response) {
            if (response.statusCode >= 300 &&
                response.statusCode < 400 &&
                response.headers.location) {
¶ redirection |
                if (url.parse(response.headers.location).protocol) {
¶ The redirect is a full location |
                    that.uri = response.headers.location;
                } else {
¶ Follow the redirect like a link |
                    that.uri = url.resolve(that.uri, response.headers.location);
                }
                that.perform(callback, times + 1);
                return;
            } else if (response.statusCode >= 400) {
¶ failure |
                callback({
                    statusCode: response.statusCode
                }, response);
            } else {
¶ success |
                response.setEncoding('binary');
                callback(null,
                    new AutoResponse(response,
                        encodings[that.file_ext] || 'utf-8'));
            }
        });
    };
¶ Stream a file to disk @param {String} filename. @param {Function} callback. |
    Get.prototype.toDisk = function(filename, callback) {
¶ TODO: catch all errors |
        this.perform(function(err, response) {
            if (err) { callback(err); return; }

            var ws = fs.createWriteStream(filename);
            response.stream = ws;

            getutil.pump(response,
                ws, function(err) {
                callback(err, filename);
            });
        });
    };
¶ Get the contents of a URL as a string @param {Function} callback. |
    Get.prototype.asString = function(callback) {
¶ TODO: catch all errors |
        this.perform(function(err, response) {
            if (err) { callback(err); return; }
            var out = [];
            response.on('data', function(chunk) {
                out.push(chunk);
            });
            response.on('end', function() {
¶ TODO warn for encoding non-strings |
                callback(null, out.join(''));
            });
        });
    };

    module.exports = Get;