c3d2/c3d2-web
c3d2/c3d2-web
37
3
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

fahrplan_download.js: optimize

Browse Source
This commit is contained in:
Astro 2010-10-17 16:27:26 +02:00
parent 07d56d5da5
commit 832be75cda
1 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
scripts/fahrplan_download.js
View File

@ -18,7 +18,7 @@ var outputDir = '.';
var baseDir = '/fahrplan'; var baseDir = '/fahrplan';
var clients = []; var clients = [];
for(var i = 0; i < 8; i++) for(var i = 0; i < 4; i++)
clients.push(HTTP.createClient(443, 'cccv.pentabarf.org', true)); clients.push(HTTP.createClient(443, 'cccv.pentabarf.org', true));
function get(path, cb) { function get(path, cb) {
@ -36,6 +36,13 @@ function get(path, cb) {
else else
console.warn(res.statusCode + ': ' + path); console.warn(res.statusCode + ': ' + path);
}); });
req.on('error', function(e) {
console.error(e.message);
// Retry:
process.nextTick(function() {
get(path, cb);
});
});
} }
function base64(s) { function base64(s) {
@ -183,13 +190,23 @@ Spider.prototype.see = function(path, from) {
case 'src': case 'src':
that.see(v); that.see(v);
return manglePath(v); return manglePath(v);
case 'action':
return (v.indexOf('://') >= 0) ? v : 'https://cccv.pentabarf.org' + v;
default: default:
return v; return v;
} }
} }
get(path, downloadXml(outPath, rewriteAttr)); get(path, downloadXml(outPath, rewriteAttr));
} else } else {
get(path, downloadTo(outPath)); // just binary download
try {
FS.statSync(outPath);
console.log('.. '+outPath);
} catch(e) {
// statSync() raised ENOENT?
get(path, downloadTo(outPath));
}
}
}; };
new Spider(); new Spider();