You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
333 lines
11 KiB
333 lines
11 KiB
#!/usr/bin/env sh |
|
|
|
#~ set -x |
|
assert_tools () { |
|
err=0 |
|
while test $# -gt 0; do |
|
command -v "$1" >/dev/null 2>/dev/null || { |
|
>&2 printf "tool missing: $1\n" |
|
err=$(( $err + 1 )) |
|
} |
|
shift |
|
done |
|
test $err -eq 0 || exit $err |
|
} |
|
assert_source () { |
|
dependencies="test printf" |
|
assert_tools ${dependencies} |
|
test $# -gt 0 && { |
|
datafn="$1"; |
|
} || { |
|
>&2 printf "option for data source missing.\n"; |
|
datafn="data.xml"; |
|
>&2 printf "assuming fallback '${datafn}'.\n"; |
|
} |
|
err=0 |
|
while test "${err}" -lt 2; do |
|
test -f "$datafn" 2>/dev/null && { >&2 printf "from ${datafn} …\n"; err=0; break; } || { |
|
>&2 printf "failed.\nfile '${datafn}' does not exist.\n" |
|
err=$(( $err + 1 )); |
|
datafn="data.xml"; |
|
} |
|
done |
|
test $err -eq 0 && { printf "$datafn"; exit 0; } || exit $err |
|
} |
|
build () { |
|
datafn="$1" |
|
dependencies="xsltproc date" |
|
assert_tools ${dependencies} |
|
|
|
# generate references.html based on directory list |
|
# todo: make it a transform (xsl) or drop it with the redesign of the webiste |
|
pwd=$(pwd) && cd references && ./gen.sh && cd "${pwd}" |
|
|
|
# todo: create covers |
|
#~ pdftoppm -jpeg -f 1 -l 1 covers/ds102.pdf covers/ds102.jpg |
|
#~ mv covers/ds102.jpg* covers/ds102.jpg |
|
|
|
>&2 printf "building ds-feed.xml … " |
|
# --stringparam current-date `date +%Y-%m-%d` # possibility for current date-time-group via XSLT <xsl:value-of select="$current-date"/> |
|
xsltproc --stringparam current-date "$(date --rfc-email)" --encoding utf-8 -o ds-feed.xml rss.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on rss.xsl.\n"; exit 1; } |
|
>&2 printf "building download.html … " |
|
#~ xsltproc --encoding iso-8859-1 -o download.html download.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on download.xsl.\n"; exit 1; } |
|
xsltproc --encoding utf-8 -o download.html download.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on download.xsl.\n"; exit 1; } |
|
>&2 printf "building index.html … " |
|
xsltproc --encoding utf-8 -o index.html index.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on index.xsl.\n"; exit 1; } |
|
exit 0 |
|
} |
|
sortxml () { |
|
#~ test "$1" = "sortxml" && { |
|
dependencies="xsltproc" |
|
assert_tools ${dependencies} |
|
xsltproc --encoding utf-8 -o sorted.xml sort.xsl data.xml |
|
mv -i sorted.xml data.xml |
|
} |
|
linkcheck () { |
|
dependencies="curl uuid" |
|
assert_tools ${dependencies} |
|
tmpfile="/tmp/"$(uuid); curl -k https://ds.ccc.de/download.html > $tmpfile |
|
for f in download.html ${tmpfile} ; do |
|
test -e $f || continue |
|
for e in $(cat ${f}|sed 's/<a /\n<a /g'|grep -i " href="|cut -d"=" -f2|cut -d'"' -f2); do |
|
status=$(curl -k --max-time 1 --head --silent --output /dev/null --write-out "%{http_code}" $e ) |
|
test $status -eq 200 && >&2 printf ${e}" – "${status}" OK\n" || >&2 printf ${e}" – "${status}" FAILED\n" |
|
done |
|
done |
|
} |
|
mirror_website () { |
|
dependencies="wget" |
|
assert_tools ${dependencies} |
|
local url=$@ |
|
local domain=`expr "$url" : '^http[s]*://\([^/?]*\)'` |
|
wget \ |
|
--no-check-certificate \ |
|
--recursive \ |
|
--no-clobber \ |
|
--page-requisites \ |
|
--html-extension \ |
|
--convert-links \ |
|
--restrict-file-names=windows \ |
|
--domains $domain \ |
|
--no-parent \ |
|
$url |
|
} |
|
newdataentry () { |
|
dependencies="cat grep head cut rev date printf" |
|
assert_tools ${dependencies} |
|
|
|
datafn=$(assert_source $1) && { |
|
if test "$datafn" = "$1" ; then |
|
shift |
|
fi |
|
} || { |
|
>&2 printf "file '${datafn}' not found."; err=1; exit 1; |
|
} |
|
|
|
err=0 |
|
customdir=$(pwd) |
|
fl="" |
|
|
|
>&2 printf "determining issue number based on '${datafn}'.\n" |
|
lastentry=$(grep -i "<schleuder id=" ${datafn} | head -1 | cut -d'"' -f2) |
|
#~ lastentry=$(grep "<schleuder id=" ${datafn} | cut -d'"' -f2 | sort | tail -1) |
|
newentry=$(( $lastentry + 1 )) |
|
|
|
# todo: switches |
|
#~ for o in "$@"; do |
|
# assigned parameters with equal sign |
|
#~ on=$(printf $o|cut -d"=" -f1) |
|
#~ ov=$(printf $o|cut -d"=" -f2-) |
|
#~ case $on in |
|
# custom data directory for availability test and filesize, default pwd |
|
#~ c|customdir) ;; |
|
# custom issue, default next after top entry in ${datafn} |
|
#~ i|issue) ;; |
|
# custom timestamp, default: current time |
|
#~ d|datetime) ;; |
|
# custom teaser xml-text, default: none |
|
#~ t|teaser) ;; |
|
# custom fs for files |
|
#~ p|pdffs) ;; |
|
#~ e|epubfs) ;; |
|
# help text |
|
#~ h|help) |
|
#~ *) ;; |
|
#~ esac |
|
# or without equal sign … |
|
#~ case $o in |
|
#~ c|customdir) ;; |
|
#~ i|issue) ;; |
|
#~ esac |
|
#~ done |
|
test $# -gt 0 && test -d $1 && { |
|
customdir=$1 |
|
>&2 printf "custom base directory for files such as jpg, pdf, epub: "${customdir}".\n\n" |
|
shift |
|
} |
|
test $# -gt 0 && test "$1" = "help" && { |
|
bold="\033[1m" |
|
nobold="\033[0m" |
|
printf "\n${bold}datenschleuder release xml snippet generation${nobold}\nprints a new data entry for a release of datenschleuder\n\n" |
|
printf "release [options]\n\n" |
|
printf "${bold}option\t\tdescription${nobold}\n\n" |
|
printf "directory\tcustom data directory for availability test and filesize, default pwd\n\n" |
|
printf "number\t\tprint entry for release with custom number, default is the increment of the top entry in ${datafn}\n\n" |
|
printf "date\t\tuse a unix timestamp for a custom date\n\n" |
|
printf "teaser\t\tinclude teaser xml-message as last argument(s)\n\n" |
|
printf "${bold}full example:${nobold} '\$./gen.sh release 99 1568505600 Die Ausgabe 99 ist fertig.'\n\n" |
|
exit 1 |
|
} |
|
|
|
2>/dev/null test $# -gt 0 && test $(( $1 * 1 )) -eq $1 && { |
|
newentry=$1; shift; |
|
>&2 printf "custom issue number '${newentry}' for '${datafn}'.\n" |
|
} || { |
|
printf "next argument not a number. assuming issue based on top entry.\n" |
|
} |
|
2>/dev/null test $# -gt 0 && test $(( $1 * 1 )) -eq $1 && { |
|
>&2 printf "custom datetime.\n" |
|
datetime="$1"; shift; |
|
} || { printf "next argument not a number. using actual system time.\n"; datetime=0; } # remaining arguments should be teaser xml-message. |
|
printf "using UTC (+0000) for expressing the timezone.\n" |
|
|
|
printf "\n<schleuder id=\"$newentry\">\n" |
|
# todo: switch for dtg, integrate custom datetime input early |
|
# or simply implement xslt for handling ISO and nothing else for input |
|
# RFC-5322-Format without seconds using the current timezone |
|
dtg="TZ=:UTC LANG=en_US.utf8 date -u -R |rev|cut -d\":\" -f2-|rev" |
|
tzc="TZ=:UTC LANG=en_US.utf8 date +\"%z\"" # timezone: `date +"%z"` |
|
# ISO dtg as issues may provide datetime-data on months but mostly not on days … |
|
#~ dtg="TZ=:UTC LANG=en_US.utf8 date +%Y-%m-%d %H:%M %Z"; tz="" |
|
test $datetime -eq 0 && { |
|
datetime=$(TZ=:UTC LANG=en_US.utf8 date -u -R |rev|cut -d":" -f2-|rev) # |
|
tz=$(TZ=:UTC LANG=en_US.utf8 date +"%z") |
|
} || { |
|
#~ dtg="TZ=:UTC LANG=en_US.utf8 date -d @${datetime} +\"%Y-%m-%d %H:%M %Z\""; tz="" |
|
dtg="TZ=:UTC LANG=en_US.utf8 date -d @${datetime} -u -R |rev|cut -d\":\" -f2-|rev"; tz=" UT" |
|
tzc="TZ=:UTC LANG=en_US.utf8 date +\"%z\"" |
|
#~ datetime="$(TZ=:UTC LANG=en_US.utf8 date -d @${datetime} -u -R|rev|cut -d":" -f2-|rev)" |
|
# TZ=:UTC LANG=en_US.utf8 date -d @${datetime} +"%Y-%m-%d %H:%M %Z" |
|
datetime="$(sh -c "${dtg}")" |
|
tz="$(sh -c "${tzc}")" |
|
} |
|
printf "\t<date>%s %s</date>\n" "${datetime}" "${tz}" # custom format universal time |
|
fn="ds"$(printf "%03d" $newentry) |
|
for d in covers pdfs epubs ; do |
|
case "$d" in |
|
covers) ffn=${d}"/"${fn}".jpg"; cover=$ffn ;; |
|
pdfs) ffn=${d}"/"${fn}".pdf" ;; |
|
epubs) ffn=${d}"/"${fn}".epub" ;; |
|
#~ *) printf "file "${ffn}" not found, in "${d}".\n" ;; |
|
esac |
|
#~ printf "$ffn\n" |
|
test -e ${customdir}"/"${ffn} && { |
|
test "$d" = "covers" && { |
|
printf "\t<image>"${cover}"</image>\n" |
|
} || { |
|
fs=$(wc -c <${customdir}"/"${ffn}) |
|
printf "\t<link filesize=\"%d\">%s</link>\n" "${fs}" "${ffn}" |
|
} |
|
} || { |
|
mfl=${mfl}" "${ffn}"\n" |
|
#~ printf "file "${ffn}" not found.\n" |
|
err=3 |
|
} |
|
done |
|
#~ printf "\t<preface></preface>\n" # not used on server yet |
|
test $# -eq 0 && { teaser="" ; } || { |
|
teaser="$@"; |
|
printf "\t<teaser>${teaser}</teaser>\n" |
|
} |
|
printf "</schleuder>\n" |
|
test "$teaser" = "" && { >&2 printf "\nmissing teaser message.\n"; } |
|
test -e ${customdir}"/"$cover || { >&2 printf "\nmissing cover file: ${cover}\n"; } |
|
test $err -gt 0 && { >&2 printf "\nmissing files:\n${mfl}\n"; } |
|
} |
|
upload () { |
|
>&2 printf "not implemented yet.\n" |
|
# check user ownership in www |
|
#~ find /usr/local/www/ds.ccc.de/references -not -user www |
|
# check rights for directories and files, so they can be updated via `scp` etc. |
|
# newly uploaded files will be owned by the uploading user |
|
# that might cause problems for the next one to update |
|
# 775: pdfs epubs references editorial |
|
# 664: pdfs/*.pdf epubs/*.epub references/*.html editorial/*.html |
|
|
|
# scp build files |
|
# source credentials/from password manager/… |
|
#~ . ~/ds-web.sh |
|
|
|
#~ scp -i $key pdfs/ds102.pdf $user@$host:$webdir/pdfs/ |
|
#~ scp -i $key covers/ds102.* $user@$host:$webdir/covers/ |
|
#~ scp -i $key epubs/ds102.epub $user@$host:${webdir}/epubs/ |
|
#~ scp -i $key editorial/ds102.html $user@$host:${webdir}/editorial/ |
|
#~ scp -i $key references/ds102.html $user@$host:${webdir}/references/ |
|
#~ scp -i $key references.html $user@$host:$webdir/ |
|
#~ scp -i $key download.html $user@$host:$webdir/ |
|
#~ scp -i $key index.html $user@$host:$webdir/ |
|
#~ scp -i $key ds-feed.xml $user@$host:$webdir/ |
|
|
|
# change to local copy |
|
#~ set pwd=$(pwd) |
|
#~ cd ../ds.ccc.de/ |
|
# copy files |
|
#~ cp ../ds-web/data.xml ./ |
|
#~ cp ../ds-web/ds-feed.xml ./ |
|
#~ cp ../ds-web/index.html ./ |
|
#~ cp ../ds-web/download.html ./ |
|
#~ cp ../ds-web/references.html ./ |
|
#~ cp ../ds-web/references/*.html ./references/ |
|
#~ cp ../ds-web/editorial/*.html ./editorial/ |
|
#~ cp ../ds-web/epubs/*.epub ./epubs/ |
|
#~ cp ../ds-web/pdfs/*.pdf ./pdfs/ |
|
#~ cp ../ds-web/vorab/*.pdf ./vorab/ |
|
# commit build files to git |
|
#~ git commit $filelist -m "upload "$(date ) |
|
#~ cd "${pwd}" |
|
} |
|
newinfoentry () { |
|
>&2 printf "not implemented yet.\n\nwill be similar to 'release'.\n" |
|
} |
|
help () { |
|
printf "commands:\n" |
|
printf "\tlinkcheck – check links for availability\n" |
|
printf "\tbuild – create HTML from XML\n" |
|
printf "\trelease – create XML for newly released issue\n" |
|
printf "\tinfo – to be: create XML for informational entry\n" |
|
printf "\tupload – to be: upload issue\n" |
|
printf "\tsortxml – sort items of the XML file\n" |
|
printf "\tmirror – create local mirror of the website\n" |
|
printf "\n" |
|
} |
|
|
|
|
|
#~ echo $# $@ |
|
|
|
if test $# -lt 1 ; then |
|
printf "missing parameter.\n" |
|
exit 1 |
|
else |
|
# a mirror is useful for local testing with files |
|
case "$1" in |
|
upload) |
|
shift |
|
upload |
|
;; |
|
build) |
|
shift |
|
f=$(assert_source $@) || { exit 1; } |
|
build "${f}" |
|
;; |
|
sortxml) |
|
shift |
|
f=$(assert_source $@) || { exit 1; } |
|
sortxml "${f}" |
|
;; |
|
mirror) |
|
>&2 printf "downloading the website may take quite some time.\n" |
|
mirror_website http://ds.ccc.de/ |
|
;; |
|
linkcheck) |
|
linkcheck http://ds.ccc.de/ |
|
;; |
|
release) |
|
shift |
|
newdataentry $@ |
|
;; |
|
info) |
|
shift |
|
newinfoentry $@ |
|
;; |
|
help) |
|
printf "to be extended, see README for exaples." |
|
help |
|
;; |
|
*) |
|
printf "this command does not exist.\n" |
|
help |
|
;; |
|
esac |
|
fi |
|
|
|
# generate small size version of pdf, might help e.g. for email attachments |
|
#gs -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -q -o ds100-smaller.pdf ds100.pdf
|
|
|