vv01f/ds-rss
vv01f
/
ds-rss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

enhance for parameters and future ISO date

This commit is contained in:
vv01f 2020-04-15 02:52:28 +02:00
parent f552bc36ad
commit 1da960f423
Signed by untrusted user who does not match committer: vv01f
GPG Key ID: 02625A16AC1D1FF6
1 changed files with 166 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
gen.sh
View File

@ -1,24 +1,44 @@
#!/usr/bin/env sh #!/usr/bin/env sh
#~ set -x
assert_tools () { assert_tools () {
err=0 err=0
while test $# -gt 0; do while test $# -gt 0; do
which $1 >/dev/null 2>/dev/null || { which $1 >/dev/null 2>/dev/null || {
>&2 echo "tool missing: "$1 >&2 printf "tool missing: $1\n"
err=$(( $err + 1 )) err=$(( $err + 1 ))
} }
shift shift
done done
test $err -eq 0 || exit $err test $err -eq 0 || exit $err
} }
test $# -eq 0 && { assert_source () {
dependencies="test printf"
assert_tools ${dependencies}
test $# -gt 0 && {
datafn="$1";
} || {
>&2 printf "option for data source missing.\n";
datafn="data.xml";
>&2 printf "assuming fallback '${datafn}'.\n";
}
test -f "$datafn" && { >&2 printf "from ${datafn} …\n"; exit 0; } || { >&2 printf "failed.\nfile '${datafn}' does not exist.\n"; exit 1; }
printf "$datafn"
}
build () {
datafn="$1"
dependencies="xsltproc" dependencies="xsltproc"
assert_tools ${dependencies} assert_tools ${dependencies}
xsltproc --encoding utf-8 -o ds-feed.xml rss.xsl data.xml >&2 printf "building ds-feed.xml … "
xsltproc --encoding iso-8859-1 -o download.html download.xsl data.xml xsltproc --encoding utf-8 -o ds-feed.xml rss.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on rss.xsl.\n"; exit 1; }
xsltproc --encoding utf-8 -o index.html index.xsl data.xml >&2 printf "building download.html … "
xsltproc --encoding iso-8859-1 -o download.html download.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on download.xsl.\n"; exit 1; }
>&2 printf "building index.html … "
xsltproc --encoding utf-8 -o index.html index.xsl $datafn && { >&2 printf "done.\n"; } || { >&2 printf "failed on index.xsl.\n"; exit 1; }
exit 0 exit 0
} }
test "$1" = "sort" && { sortxml () {
#~ test "$1" = "sortxml" && {
dependencies="xsltproc" dependencies="xsltproc"
assert_tools ${dependencies} assert_tools ${dependencies}
xsltproc --encoding utf-8 -o sorted.xml sort.xsl data.xml xsltproc --encoding utf-8 -o sorted.xml sort.xsl data.xml
@ -27,12 +47,12 @@ test "$1" = "sort" && {
linkcheck () { linkcheck () {
dependencies="curl uuid" dependencies="curl uuid"
assert_tools ${dependencies} assert_tools ${dependencies}
tmpfile="/tmp/"$(uuid); curl https://ds.ccc.de/download.html > $tmpfile tmpfile="/tmp/"$(uuid); curl -k https://ds.ccc.de/download.html > $tmpfile
for f in $(echo "download.html "$tmpfile); do for f in download.html ${tmpfile} ; do
test -e $f || continue test -e $f || continue
for e in $(cat ${f}|sed 's/<a /\n<a /g'|grep -i " href="|cut -d"=" -f2|cut -d'"' -f2); do for e in $(cat ${f}|sed 's/<a /\n<a /g'|grep -i " href="|cut -d"=" -f2|cut -d'"' -f2); do
status=$(curl --max-time 1 --head --silent --output /dev/null --write-out "%{http_code}" $e ) status=$(curl -k --max-time 1 --head --silent --output /dev/null --write-out "%{http_code}" $e )
test $status -eq 200 && >&2 echo ${e}" – "${status}" OK" || >&2 echo ${e}" – "${status}" FAILED" test $status -eq 200 && >&2 printf ${e}" – "${status}" OK\n" || >&2 printf ${e}" – "${status}" FAILED\n"
done done
done done
} }
@ -42,6 +62,7 @@ mirror_website () {
local url=$@ local url=$@
local domain=`expr "$url" : '^http[s]*://\([^/?]*\)'` local domain=`expr "$url" : '^http[s]*://\([^/?]*\)'`
wget \ wget \
--no-check-certificate \
--recursive \ --recursive \
--no-clobber \ --no-clobber \
--page-requisites \ --page-requisites \
@ -53,128 +74,166 @@ mirror_website () {
$url $url
} }
newdataentry () { newdataentry () {
dependencies="cat grep head cut date printf" dependencies="cat grep head cut rev date printf"
assert_tools ${dependencies} assert_tools ${dependencies}
#~ datafn=$(assert_source $1) && { shift; } || { >&2 printf "file '${datafn}' not found."; err=1; exit 1; }
datafn="data.xml"
err=0 err=0
customdir=$(pwd) customdir=$(pwd)
fl="" fl=""
test $# -eq 0 && {
>&2 echo "determining issue number based on data.xml.\n" >&2 printf "determining issue number based on '${datafn}'.\n"
lastentry=$(cat data.xml |grep -i "<schleuder id="|head -1|cut -d'"' -f2) lastentry=$(grep -i "<schleuder id=" ${datafn} | head -1 | cut -d'"' -f2)
newentry=$(( $lastentry + 1 )) #~ lastentry=$(grep "<schleuder id=" ${datafn} | cut -d'"' -f2 | sort | tail -1)
} || { newentry=$(( $lastentry + 1 ))
# todo: switches
#~ for o in "$@"; do # todo: switches
#~ on=$(echo $o|cut -d"=" -f1) #~ for o in "$@"; do
#~ ov=$(echo $o|cut -d"=" -f2-) #~ on=$(printf $o|cut -d"=" -f1)
#~ case $on in #~ ov=$(printf $o|cut -d"=" -f2-)
# custom data directory for availability test and filesize, default pwd #~ case $on in
#~ c) # custom data directory for availability test and filesize, default pwd
#~ customdir) ;; #~ c)
# custom issue, default next after top entry in data.xml #~ customdir) ;;
#~ i) # custom issue, default next after top entry in ${datafn}
#~ issue) ;; #~ i)
# custom timestamp, default: current time #~ issue) ;;
#~ d) # custom timestamp, default: current time
#~ datetime) ;; #~ d)
# custom teaser xml-text, default: none #~ datetime) ;;
#~ t) # custom teaser xml-text, default: none
#~ teaser) ;; #~ t)
# custom fs for files #~ teaser) ;;
#~ p) # custom fs for files
#~ pdffs) ;; #~ p)
#~ e) #~ pdffs) ;;
#~ epubfs) ;; #~ e)
# help text #~ epubfs) ;;
#~ h) # help text
#~ help) #~ h)
#~ *) ;; #~ help)
#~ esac #~ *) ;;
#~ done #~ esac
test -d $1 && { #~ done
customdir=$1; shift test -d $1 && {
>&2 echo "custom base directory for files such as jpg, pdf, epub: "${customdir}".\n" customdir=$1; shift
} >&2 printf "custom base directory for files such as jpg, pdf, epub: "${customdir}".\n\n"
test "$1" = "help" && {
echo "prints a new data entry for a release of datenschleuder\n"
echo "release [options]\n"
echo "option\tdescription\n"
echo "directory\tcustom data directory for availability test and filesize, default pwd"
echo "number\tprint entry for release with custom number, default is the increment of the top entry in data.xml\n"
echo "date\tuse a unix timestamp for a custom date\n"
echo "teaser\tinclude teaser xml-message as last argument(s)\n"
echo "full example: `./gen.sh release 99 1568505600 Die Ausgabe 99 ist fertig.`"
exit 1
}
test $(( $1 * 1 )) -eq $1 && {
>&2 echo "custom entry for data.xml.\n"
newentry=$1; shift;
} || { echo "argument not a number."; exit 1; }
test $(( $1 * 1 )) -eq $1 && {
>&2 echo "custom datetime.\n"
datetime=$1; shift;
} || { datetime=0; } # remaining arguments should be teaser xml-message.
} }
echo "<schleuder id=\"$newentry\">" test "$1" = "help" && {
test "${datetime}" = "" || test $datetime -eq 0 && { printf "prints a new data entry for a release of datenschleuder\n\n"
#~ datetime=$(LANG=en_US.utf8 date -u +"%a %d %b %Y %H:%M") # manual copy of known entries printf "release [options]\n\n"
datetime=$(LANG=en_US.utf8 date -u -R |rev|cut -d":" -f2-|rev) # RFC-5322-Format without seconds and TZ printf "option\tdescription\n\n"
} || { printf "directory\tcustom data directory for availability test and filesize, default pwd\n\n"
datetime=$(LANG=en_US.utf8 date -d @${datetime} -u -R|rev|cut -d":" -f2-|rev) printf "number\tprint entry for release with custom number, default is the increment of the top entry in ${datafn}\n\n"
printf "date\tuse a unix timestamp for a custom date\n\n"
printf "teaser\tinclude teaser xml-message as last argument(s)\n\n"
printf "full example: './gen.sh release 99 1568505600 Die Ausgabe 99 ist fertig.'\n"
exit 1
} }
echo "\t<date>"${datetime}" UT</date>" # custom format universal time 2>/dev/null test $(( $1 * 1 )) -eq $1 && {
newentry=$1; shift;
>&2 printf "custom issue number '${newentry}' for '${datafn}'.\n"
} || {
printf "next argument not a number. assuming issue based on top entry.\n"
}
2>/dev/null test $(( $1 * 1 )) -eq $1 && {
>&2 printf "custom datetime.\n\n"
datetime="$1"; shift;
} || { printf "next argument not a number. using actual system time.\n"; datetime=0; } # remaining arguments should be teaser xml-message.
printf "\nschleuder id=\"$newentry\">\n"
# todo: switch for dtg, integrate custom datetime input early
# or simply implement xslt for handling ISO and nothing else for input
# RFC-5322-Format without seconds and TZ
dtg="TZ=:UTC LANG=en_US.utf8 date -u -R |rev|cut -d\":\" -f2-|rev"; tz=" UT"
# ISO dtg as issues may provide datetime-data on months but mostly not on days …
#~ dtg="TZ=:UTC LANG=en_US.utf8 date +%Y-%m-%d %H:%M %Z"; tz=""
test $datetime -eq 0 && {
datetime=$(TZ=:UTC LANG=en_US.utf8 date -u -R |rev|cut -d":" -f2-|rev) #
} || {
#~ dtg="TZ=:UTC LANG=en_US.utf8 date -d @${datetime} +\"%Y-%m-%d %H:%M %Z\""; tz=""
dtg="TZ=:UTC LANG=en_US.utf8 date -d @${datetime} -u -R |rev|cut -d\":\" -f2-|rev"; tz=" UT"
#~ datetime="$(TZ=:UTC LANG=en_US.utf8 date -d @${datetime} -u -R|rev|cut -d":" -f2-|rev)"
# TZ=:UTC LANG=en_US.utf8 date -d @${datetime} +"%Y-%m-%d %H:%M %Z"
}
datetime="$(sh -c "${dtg}")"
printf "\t<date>${datetime}${tz}</date>\n" # custom format universal time
fn="ds"$(printf "%03d" $newentry) fn="ds"$(printf "%03d" $newentry)
for d in $(echo "covers pdfs epubs"); do for d in covers pdfs epubs ; do
case "$d" in case "$d" in
covers) ffn=${d}"/"${fn}".jpg"; cover=$ffn ;; covers) ffn=${d}"/"${fn}".jpg"; cover=$ffn ;;
pdfs) ffn=${d}"/"${fn}".pdf" ;; pdfs) ffn=${d}"/"${fn}".pdf" ;;
epubs) ffn=${d}"/"${fn}".epub" ;; epubs) ffn=${d}"/"${fn}".epub" ;;
#~ *) echo "file "${ffn}" not found, in "${d}"." ;; #~ *) printf "file "${ffn}" not found, in "${d}".\n" ;;
esac esac
#~ echo $ffn #~ printf "$ffn\n"
test -e ${customdir}"/"${ffn} && { test -e ${customdir}"/"${ffn} && {
test "$d" = "covers" && { test "$d" = "covers" && {
echo "\t<image>"${cover}"</image>" printf "\t<image>"${cover}"</image>\n"
} || { } || {
fs=$(stat -c %s ${customdir}"/"${ffn}) fs=$(stat -c %s ${customdir}"/"${ffn})
echo "\t<link filesize=\""${fs}"\">"${ffn}"</link>" printf "\t<link filesize=\""${fs}"\">"${ffn}"</link>\n"
} }
} || { } || {
mfl=${mfl}" "${ffn}"\n" mfl=${mfl}" "${ffn}"\n"
#~ echo "file "${ffn}" not found." #~ printf "file "${ffn}" not found.\n"
err=3 err=3
} }
done done
#~ echo "\t<preface></preface>" # not used on server yet #~ printf "\t<preface></preface>\n" # not used on server yet
test $# -eq 0 && { teaser="" ; } || { test $# -eq 0 && { teaser="" ; } || {
teaser=$@ ; teaser="$@";
echo "\t<teaser>"${teaser}"</teaser>" printf "\t<teaser>"${teaser}"</teaser>\n"
} }
echo "</schleuder>" printf "</schleuder>\n"
test "$teaser" = "" && { >&2 echo "\nmissing teaser message."; } test "$teaser" = "" && { >&2 printf "\nmissing teaser message.\n"; }
test -e ${customdir}"/"$cover || { >&2 echo "\nmissing cover file: "${cover}; } test -e ${customdir}"/"$cover || { >&2 printf "\nmissing cover file: "${cover}"\n"; }
test $err -gt 0 && { >&2 echo "\nmissing files:\n"${mfl}; } test $err -gt 0 && { >&2 printf "\nmissing files:\n"${mfl}"\n"; }
} }
newinfoentry () { newinfoentry () {
echo "not implemented yet.\n\nwill be similar to `release`." >&2 printf "not implemented yet.\n\nwill be similar to 'release'.\n"
}
# a mirror is useful for local testing with files
test "$1" = "mirror" && {
shift
>&2 echo "downloading the website may take quite some time."
mirror_website http://ds.ccc.de/
}
test "$1" = "linkcheck" && {
shift
linkcheck http://ds.ccc.de/
}
test "$1" = "release" && {
shift
newdataentry $@
}
test "$1" = "info" && {
shift
newinfoentry $@
} }
#~ echo $# $@
if test $# -lt 1 ; then
printf "missing parameter.\n"
exit 1
else
# a mirror is useful for local testing with files
case "$1" in
build)
shift
f=$(assert_source $@) && { shift; } || { exit 1; }
build "${f}"
;;
sortxml)
shift
f=$(assert_source $@) && { shift; } || { exit 1; }
sortxml "${f}"
;;
mirror)
>&2 printf "downloading the website may take quite some time.\n"
mirror_website http://ds.ccc.de/
;;
linkcheck)
linkcheck http://ds.ccc.de/
;;
release)
shift
newdataentry $@
;;
info)
shift
newinfoentry $@
;;
*)
printf "this is not implemented yet.\n"
;;
esac
fi
# generate small size version of pdf, might help e.g. for email attachments
#gs -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -q -o ds100-smaller.pdf ds100.pdf #gs -sDEVICE=pdfwrite -dPDFSETTINGS=/ebook -q -o ds100-smaller.pdf ds100.pdf