Compare commits
12 Commits
Author | SHA1 | Date |
---|---|---|
vv01f | fda4d6b31e | |
vv01f | 827654219e | |
vv01f | 32452c4596 | |
vv01f | 39f8a983ca | |
vv01f | 91b6523ff6 | |
vv01f | 90ca41549f | |
vv01f | 7cf298d9a1 | |
vv01f | a6e23b72ab | |
vv01f | 8fd1b9e246 | |
vv01f | faaa86a910 | |
vv01f | 8162ca0d37 | |
vv01f | fc0951e63d |
|
@ -1 +1,2 @@
|
|||
baerer
|
||||
|
||||
members.*
|
||||
|
|
|
@ -1,72 +1,5 @@
|
|||
|
||||
# ccc maps
|
||||
# VEB IT map
|
||||
|
||||
the [svg map] used on [ccc de] is at least potentially outdated
|
||||
and thus in future should be updated automatically
|
||||
|
||||
[svg map]: https://chaos.expert/telegnom/erfakarte
|
||||
[ccc de]: https://www.ccc.de/regional
|
||||
|
||||
|
||||
## my target process
|
||||
|
||||
* [x] fetch addressed of a convenient source
|
||||
* [x] resolve geo coordinates for addresses, e. g. [geopy]
|
||||
* [x] visualize on a map, e. g. via [leafletjs] or [QGIS]
|
||||
* [ ] calculate coordinates to place POI on a SVG with a known projection
|
||||
* [ ] manipulate SVG so that the POI are shown accordingly even without any application
|
||||
* [ ] change view port of SVG so it frames the POIs properly
|
||||
* [x] create PNG from the SVG and deploy
|
||||
* [ ] document for others to use
|
||||
|
||||
|
||||
## data
|
||||
|
||||
### POI
|
||||
|
||||
the address data source is the semantic [documentation wiki], queried with [result format]
|
||||
|
||||
[documentation wiki]: https://doku.ccc.de/Liste_der_Erfa-Kreise_und_Chaostreffs
|
||||
[result format]: https://www.semantic-mediawiki.org/wiki/Help:CSV_format
|
||||
|
||||
To create GeoJSON from recent address data:
|
||||
|
||||
1. `prepare.sh` pulls CSV from the Wiki
|
||||
2. `create-geojson.sh` calls `lookup.py` for each CSV set
|
||||
3. The resulting [data](https://gitea.c3d2.de/vv01f/ccc-map/src/branch/data) in [GeoJSON] format can be used with e.g. [leafletjs] or [QGIS]
|
||||
|
||||
|
||||
### Map
|
||||
|
||||
The old [svg map] was nice for Germany only.
|
||||
Over time more European spaces joined and the map could not display all of them.
|
||||
|
||||
Another point is that the projection parameters of the SVG are not known.
|
||||
New material can be produced choosing the projection.
|
||||
One recommended program to do this is [QGIS].
|
||||
Freely useable data is available on e. g. [Natural Earth].
|
||||
|
||||
To create a new SVG Map:
|
||||
|
||||
1. Start QGIS, KBS setting recommendation: EPSG:3857 / Pseudo Mercator
|
||||
2. Drop a Shape-File, e. g. sqlite format (and optionally a GeoJSON-File)
|
||||
3. Adjust colors in the layers styles
|
||||
4. Print as SVG (Label for POI data is lost in version 2.18)
|
||||
|
||||
[QGIS]: https://qgis.org/ "QGIS"
|
||||
[Natural Earth]: http://naturalearthdata.com/ "Natural Earth"
|
||||
|
||||
[geopy]: https://geopy.readthedocs.io/
|
||||
[leafletjs]: https://leafletjs.com/
|
||||
[GeoJSON]: https://geojson.org/ "Website for RFC 7946"
|
||||
|
||||
<!--
|
||||
|
||||
## Thanks go to …
|
||||
|
||||
* [telegnom](https://chaos.expert/telegnom/erfakarte) for patiently describing the problems and needs for the map
|
||||
* [ax3l](https://github.com/ax3l) for his [spontanious lightning talk](https://media.ccc.de/v/DS2016-7782-lightning_talks#t=5112) on [„Visualisierung Flüchtlingsfeindlicher Vorfälle in Deutschland“](https://github.com/ax3l/chronik-vorfaelle) at [Datenspuren](http://datenspuren.de/) in 2016 getting me started years later when I saw it again
|
||||
* The [Dresden OSM Meeting](https://wiki.openstreetmap.org/wiki/DresdnerOSMStammtisch) for hinting me on QGIS and an introduction how to use it
|
||||
* and quite some people helping me to finally polish some python
|
||||
|
||||
-->
|
||||
this GeoJSON generator simply produces markers for a map of members.
|
||||
the resolution is set to a 11km grid for not identifying people easily.
|
||||
|
|
|
@ -1,11 +1,23 @@
|
|||
#!/usr/bin/env sh
|
||||
echo "this takes some time depending on the amount of addresses in your lists …"
|
||||
for file in *-csv ; do
|
||||
out=$(echo "${file}"|cut -d- -f1)".geojson"
|
||||
if test -e "${out}" ; do
|
||||
for file in *.csv ; do
|
||||
out=$(echo "${file}"|cut -d"." -f1)".geojson"
|
||||
if test -e "${out}" ; then
|
||||
echo "file exists: ${out}, skipping."
|
||||
continue
|
||||
fi
|
||||
echo ./lookup.py "${file}"
|
||||
./lookup.py "${file}" 2>/dev/null > "${out}"
|
||||
./lookup.py "${file}" > "${out}"
|
||||
|
||||
#~ fs=$(stat -t "${out}" |cut -d" " -f2)
|
||||
#~ data="vebit-geojson/"
|
||||
|
||||
#~ if test "$fs" -gt 0 ; then
|
||||
#~ if test -d "$data" ; then
|
||||
#~ cp -f "${out}" "$data"
|
||||
#~ cd "$data"
|
||||
#~ git commit "${out}" -m "update file" && git push
|
||||
#~ fi
|
||||
#~ fi
|
||||
|
||||
done
|
||||
|
|
34
lookup.py
34
lookup.py
|
@ -5,13 +5,19 @@ from geopy import Nominatim
|
|||
# ~ import json
|
||||
import sys
|
||||
|
||||
import sys
|
||||
#disable ssl verification
|
||||
import ssl
|
||||
import geopy.geocoders
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
geopy.geocoders.options.default_ssl_context = ctx
|
||||
#/
|
||||
|
||||
# debug print to stderr, https://stackoverflow.com/questions/5574702/how-to-print-to-stderr-in-python
|
||||
def eprint(*args, **kwargs):
|
||||
print(*args, file=sys.stderr, **kwargs)
|
||||
|
||||
# ~ fn="ct-csv"
|
||||
if len(sys.argv) == 2:
|
||||
fn = sys.argv[1]
|
||||
else:
|
||||
|
@ -21,9 +27,9 @@ else:
|
|||
sys.exit(1) # exit with error
|
||||
|
||||
# for retreiving geo coordinates from addresses
|
||||
geolocator = Nominatim(user_agent="my-mapper")
|
||||
# lat: 0=111,1km,1=11km,2=1km… lng: 0=70km,1=7km,2=.7km…
|
||||
precision = 2
|
||||
geolocator = Nominatim(user_agent="vebit-mapper")
|
||||
# lat: 0=111,1km; 1=11km; 2=1km… lng: 0=70km; 1=7km; 2=0,7km…
|
||||
precision = 1
|
||||
|
||||
# convert csv to geojson
|
||||
|
||||
|
@ -31,18 +37,24 @@ firstline = True
|
|||
# start json list
|
||||
print ( '{"type":"FeatureCollection","features":[' )
|
||||
# get data from file
|
||||
# ~ eprint( "opening: "+fn )
|
||||
with open( fn, 'r' ) as fp:
|
||||
for place in fp:
|
||||
#error handling: expect
|
||||
arrAddress = place.split(",")
|
||||
strAddress = ",".join(arrAddress[2:6])
|
||||
arrAddress = place.split(";")
|
||||
strAddress1 = arrAddress[1]+" "+arrAddress[2]
|
||||
strAddress2 = ",".join(arrAddress[3:5])
|
||||
strAddress = strAddress1+","+strAddress2
|
||||
eprint( "looking up: "+strAddress )
|
||||
location = geolocator.geocode( strAddress )
|
||||
if location is not None: # its a class
|
||||
# todo: ceil coords to hide true location in format string
|
||||
# ~ strCoordPlace = '{:s},{:.6f},{:.6f}'.format( arrAddress[3], location.latitude, location.longitude )
|
||||
geojson = '{{"type":"Feature","geometry":{{"type":"Point","coordinates":[{:.'+str(precision)+'f},{:.'+str(precision)+'f}]}},"properties":{{"name":"{:s}","marker":"{:s}"}}}}';
|
||||
strCoordPlace = geojson.format( location.longitude, location.latitude, arrAddress[1], fn.split("-")[0] )
|
||||
# done: ceil coords to hide true location in format string
|
||||
# ~ strCoordPlace = '{:s},{:.6f},{:.6f}'.format( arrAddress[3], location.latitude, location.longitude )
|
||||
# todo: auto deduplicate coords
|
||||
# ~ strFormatGeoCoord = '{:.'+str(precision)+'f},{:.'+str(precision)+'f}';
|
||||
# ~ strGeoCoord = strFormatGeoCoord.format( location.longitude, location.latitude )
|
||||
geojson = '{{"type":"Feature","geometry":{{"type":"Point","coordinates":[{:.'+str(precision)+'f},{:.'+str(precision)+'f}]}},"properties":{{"name":"anonentry","marker":"veb-it"}}}}';
|
||||
strCoordPlace = geojson.format( location.longitude, location.latitude )
|
||||
if firstline == False :
|
||||
print (",")
|
||||
else:
|
||||
|
|
38
prepare.sh
38
prepare.sh
|
@ -1,33 +1,15 @@
|
|||
#!/usr/bin/env sh
|
||||
if="../non-public/eV/members.fods"
|
||||
ofn=$(echo $if|rev|cut -d"/" -f1|cut -d"." -f2-|rev)
|
||||
|
||||
# baerer is the `authority` as in RFC 3986, here login for doku.ccc.de
|
||||
fae="file already exists."
|
||||
# recv data
|
||||
if test -e baerer ; then
|
||||
auth=$(head -1 baerer)
|
||||
else
|
||||
echo "error: authentication information not found."
|
||||
exit
|
||||
fi
|
||||
#~ test -e ct-json && echo "$fae" || curl -o ct-json "https://"${auth}"doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::falsch-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DErfa-2DCity/offset%3D0/format%3Djson/headers%3Dshow/searchlabel%3DJSON"
|
||||
#~ test -e erfa-json && echo "$fae" || curl -o erfa-json "https://"${auth}"doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::wahr-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DErfa-2DCity/offset%3D0/format%3Djson/headers%3Dshow/searchlabel%3DJSON"
|
||||
if test -e ct-csv ; then
|
||||
echo "$fae"
|
||||
else
|
||||
curl -o ct-csv "https://${auth}doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::falsch-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity%3DLabel/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DChaostreff-2DCity/offset%3D0/format%3Dcsv/headers%3Dshow/searchlabel%3DCSV/sep%3D,/filename%3Dct-2Dbesuchsadressen.csv"
|
||||
fi
|
||||
if test -e erfa-csv ; then
|
||||
echo "$fae"
|
||||
else
|
||||
curl -o erfa-csv "https://${auth}doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Erfa-2DKreise-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity%3DLabel/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DChaostreff-2DCity/offset%3D0/format%3Dcsv/headers%3Dshow/searchlabel%3DCSV/sep%3D,/filename%3Derfa-2Dbesuchsadressen.csv"
|
||||
fi
|
||||
# preprocess csv data
|
||||
for file in ct-csv erfa-csv ; do
|
||||
sed -e 's/"//g' ${file} > tmpfile && mv tmpfile ${file}
|
||||
#~ sed -e 's/\([0-9]\{4,5\}\),/\1 /g' ${file} > tmpfile && mv tmpfile ${file}
|
||||
sed '1d' ${file} > tmpfile && mv tmpfile ${file}
|
||||
done
|
||||
# dependencies
|
||||
|
||||
#~ pip3 install geopy # looking up addresses/coordinates
|
||||
#~ pip3 install folium # leaflet.js with python
|
||||
if test -e $ofn".csv" ; then
|
||||
echo "$fae"
|
||||
else
|
||||
libreoffice --headless --convert-to csv:"Text - txt - csv (StarCalc)":"59,,76,3,1/1/2/2/2" $if || { echo "errors during export to csv"; exit 3; }
|
||||
tail -n +2 ${ofn}".csv" | cut -d";" -f2,13-17|grep -e "^[v|g|n]" > ${ofn}".tmp"
|
||||
mv ${ofn}".tmp" ${ofn}".csv" || { echo "error on removing first line."; exit 6; }
|
||||
fi
|
||||
|
||||
|
|
Loading…
Reference in New Issue