From 80cb99ea6f91d376cdaf7d29432e8c558f4f1596 Mon Sep 17 00:00:00 2001 From: Astro Date: Tue, 7 Oct 2014 01:39:08 +0200 Subject: [PATCH] ds14 mitschnitte online! --- Makefile | 5 +- content/news/ds14-mitschnitte-online.xml | 109 ++++++++++++ scripts/ds2014-resources.js | 202 +++++++++++++++++++++++ scripts/encode_gif.sh | 2 +- scripts/encode_mp3.sh | 3 +- scripts/encode_mp4.sh | 10 +- scripts/encode_webm.sh | 11 +- 7 files changed, 330 insertions(+), 12 deletions(-) create mode 100644 content/news/ds14-mitschnitte-online.xml create mode 100644 scripts/ds2014-resources.js diff --git a/Makefile b/Makefile index e4839e1c9..b6bca046e 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,8 @@ DS_FEEDS:= \ build/datenspuren/2010/mitschnitte-rss.xml \ build/datenspuren/2011/mitschnitte-rss.xml \ build/datenspuren/2012/mitschnitte-rss.xml \ - build/datenspuren/2013/mitschnitte-rss.xml + build/datenspuren/2013/mitschnitte-rss.xml \ + build/datenspuren/2014/mitschnitte-rss.xml # Quellen NEWSITEMS:=$(wildcard content/news/*.xml) @@ -298,6 +299,8 @@ build/datenspuren/2012/mitschnitte-rss.xml: content/news/ds12-videomitschnitte-k $(call xml_process) build/datenspuren/2013/mitschnitte-rss.xml: content/news/ds13-videomitschnitte-komplett.xml $(STYLE) $(call xml_process) +build/datenspuren/2014/mitschnitte-rss.xml: content/news/ds14-mitschnitte-online.xml $(STYLE) + $(call xml_process) xhtml5-validate: $(patsubst build/%.html, build/%.html.xhtml5-validate, $(CONTENT)) diff --git a/content/news/ds14-mitschnitte-online.xml b/content/news/ds14-mitschnitte-online.xml new file mode 100644 index 000000000..9afdf00af --- /dev/null +++ b/content/news/ds14-mitschnitte-online.xml @@ -0,0 +1,109 @@ + + + https://datenspuren.de/2014/logo/datenspuren-kolle-mate-etikett.5.svg +

+ Die Videomitschnitte der Datenspuren 2014 sind + online! Sie sind auch als + Podcast-Feed und über BitTorrent erhältlich. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/ds2014-resources.js b/scripts/ds2014-resources.js new file mode 100644 index 000000000..cf1f81c28 --- /dev/null +++ b/scripts/ds2014-resources.js @@ -0,0 +1,202 @@ +var ltx = require('ltx'); +var http = require('http'); +var fs = require('fs'); +var url = require('url'); + + +var running = 0, queue = []; + +var SUFFIXES = [".mp4", ".webm", ".mp3"]; +var WEB_SUFFIXES = [".mp4", ".webm", ".mp3"]; + +SUFFIX_OPTS = { + ".mp4": "-movflags +faststart" +}; + +function resolveSizes(urls, cb) { + if (running > 0) { + queue.push(function() { + resolveSizes(urls, cb); + }); + return; + } + running++; + + urls = urls.map(url.parse); + + var pending = 0, results = [], + done = function() { + if (pending < 1) { + results = results.sort(function(r1, r2) { + var ui1 = urls.indexOf(r1.url); + var ui2 = urls.indexOf(r2.url); + if (ui1 < ui2) + return -1; + else if (ui1 > ui2) + return 1; + else + return 0; + }); + cb(results); + + running--; + var f = queue.shift(); + f && f(); + } + }; + + urls.forEach(function(u) { + var req = http.request({ method: 'HEAD', + path: u.pathname, + hostname: u.host }); + req.end(); + req.on('response', function(res) { + if (res.statusCode === 200) { + results.push({ url: url.format(u), + size: res.headers['content-length'] }); + } else { + console.error("HTTP " + res.statusCode + " " + u.pathname); + } + pending--; + done(); + }); + pending++; + }); + done(); +} + +function mimeByExt(url) { + var ext = url.split('.').pop(); + var type = ({ avi: 'video/avi', + mp3: 'audio/mpeg', + mp4: 'video/mp4', + webm: 'video/webm', + ogv: 'video/ogg', + flv: 'video/x-flv' })[ext]; + return type || 'application/octet-stream'; +} + +function processEvent(event) { + var base = 'http://ftp.c3d2.de/datenspuren/2014/' + event.outputName; + var urls = WEB_SUFFIXES.map(function(fmt) { + return base + fmt; + }); + resolveSizes(urls, function(infos) { + infos = infos.filter(function(i) { + return i.size !== undefined; + }); + + if (infos.length < 1) { + console.warn(""); + return; + } + var resAttrs = { + title: event.title, + size: infos[0].size, + type: mimeByExt(infos[0].url), + url: infos[0].url, + preview: base + ".gif", + poster: base + ".jpg" + }; + if (event.id != "0000") { + resAttrs['details-link'] = "http://datenspuren.de/2014/fahrplan/events/" + + event.id + ".html"; + resAttrs['feedback-link'] = "https://frab.cccv.de/en/DS2014/public/events/" + + event.id + "/feedback/new"; + } + var res = new ltx.Element('resource', resAttrs); + infos.slice(1).forEach(function(info) { + res.c('alternative', { size: info.size, + type: mimeByExt(info.url), + url: info.url }); + }); + console.info(res.toString()); + }); +} + +function printMetadata(event) { + var metaparameters = ""; + function addMeta(k, v) { + metaparameters += + " -metadata " + k + "=\"" + + v.replace(/"/g, '\\"') + "\""; + } + addMeta('title', event.title); + addMeta('subtitle', event.subtitle); + addMeta('album', "Datenspuren 2014"); + addMeta('artist', "Datenspuren"); + addMeta('author', "Datenspuren"); + addMeta('language', "deu"); + addMeta('year', "2014"); + addMeta('composer', event.persons.join(", ")); + addMeta('description', event.abstract || event.description); + SUFFIXES.forEach(function(suffix) { + console.log("ffmpeg -i " + event.inputName + suffix + " -codec copy" + metaparameters + " " + (SUFFIX_OPTS[suffix] || "") + " -y " + event.outputName + suffix); + }); +} + +function slugify(title) { + return title + .replace(/\(Gesundheits\-\)System,.*/, ".gesundheitssystem") + .replace(/'was, was/, "was was") + .replace(/zur Verschleierung von Kontakten/, "") + .replace(/Fake-Hotspot /, "Fake-Hotspots ") + .toLowerCase() + .replace(/\s*[,\-\:–]\s+.*/, "") + .replace(/[\!\?\/\(\)]/g, "") + .replace(/Ä/g, "Ae") + .replace(/Ö/g, "Oe") + .replace(/Ü/g, "Ue") + .replace(/ä/g, "ae") + .replace(/ö/g, "oe") + .replace(/ü/g, "ue") + .replace(/ß/g, "ss") + .replace(/ẞ/g, "SS") + .replace(/≡/g, "E") + .replace(/\s+$/, "") + .replace(/\s+/g, "_") + .replace(/[^a-zA-Z0-9\-_]/g, ""); +} + +function makeEvent(event) { + event.inputName = event.id; + event.outputName = event.id + "_" + slugify(event.title); + console.error("" + event.title + " -> " + event.outputName); + return event; +} + +var scheduleFile = fs.readFileSync('../content/static/datenspuren/2014/fahrplan/schedule.xml'); +var schedule = ltx.parse(scheduleFile); +schedule.getChildren('day').forEach(function(dayEl) { + var dayEvents = []; + var roomIdx = 0; + dayEl.getChildren('room').forEach(function(roomEl) { + roomEl.getChildren('event').forEach(function(eventEl) { + var persons = []; + eventEl.getChildren('persons').forEach(function(personsEl) { + personsEl.getChildren('person').forEach(function(personEl) { + persons.push(personEl.getText()); + }); + }); + dayEvents.push(makeEvent( + { id: eventEl.attrs.id, + time: eventEl.getChildText('start'), + room: roomIdx, + title: eventEl.getChildText('title'), + subtitle: eventEl.getChildText('subtitle'), + slug: eventEl.getChildText('slug'), + abstract: eventEl.getChildText('abstract'), + description: eventEl.getChildText('description'), + persons: persons + })); + }); + roomIdx++; + }); + dayEvents = dayEvents.sort(function(e1, e2) { + return (e1.time < e2.time) ? -1 : + ((e1.time > e2.time) ? 1 : e1.room - e2.room); + }); + dayEvents.forEach(printMetadata); + dayEvents.forEach(processEvent); +}); + diff --git a/scripts/encode_gif.sh b/scripts/encode_gif.sh index a5c7f1dcb..de54c9684 100755 --- a/scripts/encode_gif.sh +++ b/scripts/encode_gif.sh @@ -3,7 +3,7 @@ if=$1 of=$2 -ffmpeg -i "$if" -r 0.01 -b:v 100k -bt:v 10M -s 320x180 "$of.%05d.jpg" && \ +ffmpeg -i "$if" -r 0.005 -b:v 100k -bt:v 10M -s 720x404 "$of.%05d.jpg" && \ rm -v $of.0000[01234].jpg && \ convert -delay 50 -loop 0 "$of.?????.jpg" "$of.gif" && \ mv $of.00005.jpg $of.jpg && \ diff --git a/scripts/encode_mp3.sh b/scripts/encode_mp3.sh index 3ff096c41..92a3289e7 100755 --- a/scripts/encode_mp3.sh +++ b/scripts/encode_mp3.sh @@ -6,11 +6,10 @@ of=$2 # No. CPU cores: threads=8 # Audio bitrate: -ab=96k +ab=128k ffmpeg -i "$if" \ -f mp3 \ -ab $ab \ -threads $threads \ -y "$of" - diff --git a/scripts/encode_mp4.sh b/scripts/encode_mp4.sh index 2a85d757f..27a2e3843 100755 --- a/scripts/encode_mp4.sh +++ b/scripts/encode_mp4.sh @@ -6,20 +6,22 @@ of=$2 # No. CPU cores: threads=4 # Video bitrate: -vb=1600k +vb=400k # Audio bitrate: -ab=128k +ab=96k + ffmpeg -i "$if" \ -f mp4 \ - -metadata:s:a:0 language=deu -metadata title="IP-Debugging" -metadata album="Themenabend" -metadata artist="C3D2" -metadata year=2014 \ -acodec libfaac \ -ab $ab \ + -ac 1 \ -threads $threads \ + -vf hqdn3d=0:0:16:14 \ -vcodec libx264 \ -b:v $vb \ + -s 720x404 \ -profile:v baseline -level 3.0 \ -movflags +faststart \ -passlogfile "$of.ffmpeg2pass" \ -y "$of" - diff --git a/scripts/encode_webm.sh b/scripts/encode_webm.sh index a41b8b348..196c7d50f 100755 --- a/scripts/encode_webm.sh +++ b/scripts/encode_webm.sh @@ -7,28 +7,31 @@ of=$2 # No. CPU cores: threads=4 # Video bitrate: -vb=1600k +vb=400k # Audio bitrate: -ab=128k +ab=96k ffmpeg -i "$if" \ -threads $threads \ -f webm \ -an \ + -vf hqdn3d=0:0:16:14 \ -vcodec libvpx \ -b:v $vb \ + -s 720x404 \ -pass 1 \ -passlogfile "$of.ffmpeg2pass" \ -y /dev/null ffmpeg -i "$if" \ -threads $threads \ -f webm \ - -metadata:s:a:0 language=deu -metadata language=deu -metadata title="IP-Debugging" -metadata album="Themenabend" -metadata artist="C3D2" -metadata year=2014 \ -acodec libvorbis \ -ab $ab \ + -ac 1 \ + -vf hqdn3d=0:0:16:14 \ -vcodec libvpx \ -b:v $vb \ + -s 720x404 \ -pass 2 \ -passlogfile "$of.ffmpeg2pass" \ -y "$of" -