add note

.gitignore

@@ -1 +1,2 @@
-baerer
+
+members.*

README.markdown

@@ -1,72 +1,5 @@
 
-# ccc maps
+# VEB IT map
 
-the [svg map] used on [ccc de] is at least potentially outdated
-and thus in future should be updated automatically
-
-[svg map]: https://chaos.expert/telegnom/erfakarte 
-[ccc de]: https://www.ccc.de/regional
-
-
-## my target process
-
-* [x] fetch addressed of a convenient source
-* [x] resolve geo coordinates for addresses, e. g. [geopy]
-* [x] visualize on a map, e. g. via [leafletjs] or [QGIS]
-* [ ] calculate coordinates to place POI on a SVG with a known projection
-* [ ] manipulate SVG so that the POI are shown accordingly even without any application
-* [ ] change view port of SVG so it frames the POIs properly
-* [x] create PNG from the SVG and deploy
-* [ ] document for others to use
-
-
-## data
-
-### POI
-
-the address data source is the semantic [documentation wiki], queried with [result format]
-
-[documentation wiki]: https://doku.ccc.de/Liste_der_Erfa-Kreise_und_Chaostreffs
-[result format]: https://www.semantic-mediawiki.org/wiki/Help:CSV_format
-
-To create GeoJSON from recent address data:
-
-1. `prepare.sh` pulls CSV from the Wiki
-2. `create-geojson.sh` calls `lookup.py` for each CSV set 
-3. The resulting [data](https://gitea.c3d2.de/vv01f/ccc-map/src/branch/data) in [GeoJSON] format can be used with e.g. [leafletjs] or [QGIS] 
-
-
-### Map
-
-The old [svg map] was nice for Germany only.
-Over time more European spaces joined and the map could not display all of them.
-
-Another point is that the projection parameters of the SVG are not known.
-New material can be produced choosing the projection.
-One recommended program to do this is [QGIS].
-Freely useable data is available on e. g. [Natural Earth].
-
-To create a new SVG Map:
-
-1. Start QGIS, KBS setting recommendation: EPSG:3857 / Pseudo Mercator
-2. Drop a Shape-File, e. g. sqlite format (and optionally a GeoJSON-File)
-3. Adjust colors in the layers styles
-4. Print as SVG (Label for POI data is lost in version 2.18)
-
-[QGIS]: https://qgis.org/ "QGIS"
-[Natural Earth]: http://naturalearthdata.com/ "Natural Earth"
-
-[geopy]: https://geopy.readthedocs.io/
-[leafletjs]: https://leafletjs.com/
-[GeoJSON]: https://geojson.org/ "Website for RFC 7946"
-
-<!--
-
-## Thanks go to …
-
-* [telegnom](https://chaos.expert/telegnom/erfakarte) for patiently describing the problems and needs for the map
-* [ax3l](https://github.com/ax3l) for his [spontanious lightning talk](https://media.ccc.de/v/DS2016-7782-lightning_talks#t=5112) on [„Visualisierung Flüchtlingsfeindlicher Vorfälle in Deutschland“](https://github.com/ax3l/chronik-vorfaelle) at [Datenspuren](http://datenspuren.de/) in 2016 getting me started years later when I saw it again
-* The [Dresden OSM Meeting](https://wiki.openstreetmap.org/wiki/DresdnerOSMStammtisch) for hinting me on QGIS and an introduction how to use it
-* and quite some people helping me to finally polish some python
-
--->
+this GeoJSON generator simply produces markers for a map of members.
+the resolution is set to a 11km grid for not identifying people easily.

create-geojson.sh

@@ -1,11 +1,23 @@
 #!/usr/bin/env sh
 echo "this takes some time depending on the amount of addresses in your lists …"
-for file in *-csv ; do
-	out=$(echo "${file}"|cut -d- -f1)".geojson"
-	if test -e "${out}" ; do
+for file in *.csv ; do
+	out=$(echo "${file}"|cut -d"." -f1)".geojson"
+	if test -e "${out}" ; then
 		echo "file exists: ${out}, skipping."
 		continue
 	fi
 	echo ./lookup.py "${file}"
-	./lookup.py "${file}" 2>/dev/null > "${out}"
+	./lookup.py "${file}" > "${out}"
+
+	#~ fs=$(stat -t "${out}" |cut -d" " -f2)
+	#~ data="vebit-geojson/"
+
+	#~ if test "$fs" -gt 0 ; then
+		#~ if test -d "$data" ; then 
+			#~ cp -f "${out}" "$data"
+			#~ cd "$data"
+			#~ git commit "${out}" -m "update file" && git push
+		#~ fi
+	#~ fi
+
 done

lookup.py

@@ -5,13 +5,19 @@ from geopy import Nominatim
 # ~ import json
 import sys
 
-import sys
+#disable ssl verification
+import ssl
+import geopy.geocoders
+ctx = ssl.create_default_context()
+ctx.check_hostname = False
+ctx.verify_mode = ssl.CERT_NONE
+geopy.geocoders.options.default_ssl_context = ctx
+#/
 
 # debug print to stderr, https://stackoverflow.com/questions/5574702/how-to-print-to-stderr-in-python
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
-# ~ fn="ct-csv"
 if len(sys.argv) == 2:
     fn = sys.argv[1]
 else:
@@ -21,9 +27,9 @@ else:
     sys.exit(1) # exit with error
 
 # for retreiving geo coordinates from addresses
-geolocator = Nominatim(user_agent="my-mapper")
-# lat: 0=111,1km,1=11km,2=1km… lng: 0=70km,1=7km,2=.7km…
-precision = 2
+geolocator = Nominatim(user_agent="vebit-mapper")
+# lat: 0=111,1km; 1=11km; 2=1km… lng: 0=70km; 1=7km; 2=0,7km…
+precision = 1
 
 # convert csv to geojson
 
@@ -31,18 +37,24 @@ firstline = True
 # start json list
 print ( '{"type":"FeatureCollection","features":[' )
 # get data from file
+# ~ eprint( "opening: "+fn )
 with open( fn, 'r' ) as fp:
     for place in fp:
         #error handling: expect
-        arrAddress = place.split(",")
-        strAddress = ",".join(arrAddress[2:6])
+        arrAddress = place.split(";")
+        strAddress1 = arrAddress[1]+" "+arrAddress[2]
+        strAddress2 = ",".join(arrAddress[3:5])
+        strAddress = strAddress1+","+strAddress2
         eprint( "looking up: "+strAddress )
         location = geolocator.geocode( strAddress )
         if location is not None: # its a class
-            # todo: ceil coords to hide true location in format string
-            # ~ strCoordPlace = '{:s},{:.6f},{:.6f}'.format( arrAddress[3], location.latitude, location.longitude ) 
-            geojson = '{{"type":"Feature","geometry":{{"type":"Point","coordinates":[{:.'+str(precision)+'f},{:.'+str(precision)+'f}]}},"properties":{{"name":"{:s}","marker":"{:s}"}}}}';
-            strCoordPlace = geojson.format( location.longitude, location.latitude, arrAddress[1], fn.split("-")[0] ) 
+            # done: ceil coords to hide true location in format string
+            # ~ strCoordPlace = '{:s},{:.6f},{:.6f}'.format( arrAddress[3], location.latitude, location.longitude )
+            # todo: auto deduplicate coords
+            # ~ strFormatGeoCoord = '{:.'+str(precision)+'f},{:.'+str(precision)+'f}';
+            # ~ strGeoCoord = strFormatGeoCoord.format( location.longitude, location.latitude )
+            geojson = '{{"type":"Feature","geometry":{{"type":"Point","coordinates":[{:.'+str(precision)+'f},{:.'+str(precision)+'f}]}},"properties":{{"name":"anonentry","marker":"veb-it"}}}}';
+            strCoordPlace = geojson.format( location.longitude, location.latitude ) 
             if firstline == False :
                 print (",")
             else:

prepare.sh

@@ -1,33 +1,15 @@
 #!/usr/bin/env sh
+if="../non-public/eV/members.fods"
+ofn=$(echo $if|rev|cut -d"/" -f1|cut -d"." -f2-|rev)
 
-# baerer is the `authority` as in RFC 3986, here login for doku.ccc.de
 fae="file already exists."
 # recv data
-if test -e baerer ; then
-	auth=$(head -1 baerer)
-else
-	echo "error: authentication information not found."
-	exit
-fi
-#~ test -e ct-json && echo "$fae" || curl -o ct-json "https://"${auth}"doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::falsch-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DErfa-2DCity/offset%3D0/format%3Djson/headers%3Dshow/searchlabel%3DJSON"
-#~ test -e erfa-json && echo "$fae" || curl -o erfa-json "https://"${auth}"doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::wahr-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DErfa-2DCity/offset%3D0/format%3Djson/headers%3Dshow/searchlabel%3DJSON"
-if test -e ct-csv ; then
-	echo "$fae"
-else
-	curl -o ct-csv "https://${auth}doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Chaostreffs-5D-5D-20-5B-5BChaostreff-2DIs-2DErfa::falsch-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity%3DLabel/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DChaostreff-2DCity/offset%3D0/format%3Dcsv/headers%3Dshow/searchlabel%3DCSV/sep%3D,/filename%3Dct-2Dbesuchsadressen.csv"
-fi
-if test -e erfa-csv ; then
-	echo "$fae"
-else
-	curl -o erfa-csv "https://${auth}doku.ccc.de/Spezial:Semantische_Suche/-5B-5BKategorie:Erfa-2DKreise-5D-5D-20-5B-5BChaostreff-2DActive::wahr-5D-5D/-3FChaostreff-2DCity%3DLabel/-3FChaostreff-2DPhysical-2DAddress%3DAdresse/-3FChaostreff-2DPhysical-2DHousenumber%3DHausnummer/-3FChaostreff-2DPhysical-2DPostcode%3DPLZ/-3FChaostreff-2DPhysical-2DCity%3DStadt/-3FChaostreff-2DCountry%3DLand/mainlabel%3D/limit%3D100/order%3DASC/sort%3DChaostreff-2DCity/offset%3D0/format%3Dcsv/headers%3Dshow/searchlabel%3DCSV/sep%3D,/filename%3Derfa-2Dbesuchsadressen.csv"
-fi
-# preprocess csv data
-for file in ct-csv erfa-csv ; do
-	sed -e 's/"//g' ${file} > tmpfile && mv tmpfile ${file}
-	#~ sed -e 's/\([0-9]\{4,5\}\),/\1 /g' ${file} > tmpfile && mv tmpfile ${file}
-	sed '1d' ${file} > tmpfile && mv tmpfile ${file}
-done
-# dependencies
 
-#~ pip3 install geopy # looking up addresses/coordinates
-#~ pip3 install folium # leaflet.js with python
+if test -e $ofn".csv" ; then
+	echo "$fae"
+else
+	libreoffice --headless --convert-to csv:"Text - txt - csv (StarCalc)":"59,,76,3,1/1/2/2/2" $if || { echo "errors during export to csv"; exit 3; }
+	tail -n +2 ${ofn}".csv" | cut -d";" -f2,13-17|grep -e "^[v|g|n]" > ${ofn}".tmp"
+	mv ${ofn}".tmp" ${ofn}".csv" || { echo "error on removing first line."; exit 6; }
+fi
+