backend: import first warhelp sample datasets

This commit is contained in:
Johannes Lötzsch 2022-03-16 17:52:15 +01:00
parent f7460f320e
commit 78a340a1d0
8 changed files with 53 additions and 9 deletions

View File

@ -2,7 +2,8 @@
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.import.offer.helper :refer [update-offers]]
[beherbergung.db.import.offer.ngo.random :as random]
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]))
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]
[beherbergung.db.import.offer.ngo.warhelp :as warhelp]))
(defn import! []
(if-not (:import-ngo env)
@ -12,6 +13,8 @@
(case (:import-ngo env)
"lifeline_beherbergung"
(lifeline/importfile->table (:import-file env))
"warhelp_beherbergung"
(warhelp/importfile->table (:import-file env))
(random/importfile->table)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
@ -20,11 +23,12 @@
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
_ (require '[beherbergung.db.state :refer [db_ctx]])
(require '[beherbergung.db.state :refer [db_ctx]])
(let [ngo:id "warhelp_beherbergung"
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))
(->> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id)
#_(map :id_tmp))))

View File

@ -0,0 +1,8 @@
(ns beherbergung.db.import.offer.ngo.warhelp
(:require [clojure.edn]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.warhelp]))
(defn importfile->table [file]
(take 5 (unify (clojure.edn/read-string (slurp file)) ;; TODO all
beherbergung.model.offer-mapping.warhelp/mapping)))

View File

@ -6,7 +6,7 @@
(defn JaNein->bool [JaNein]
({"Ja" true "Nein" false} JaNein))
(def mapping {:id_tmp #(or (get % "E-Mail") (get % "Telefonnummer"))
(def mapping {:id_tmp #(or (not-empty (get % "E-Mail")) (get % "Telefonnummer"))
:time_from_str "frühestes Einzugsdatum"
:time_duration_str "Möglicher Aufenthalt (Dauer)" ;; TODO: the duration is not parsed till now

View File

@ -0,0 +1,32 @@
(ns beherbergung.model.offer-mapping.warhelp
(:require [clojure.spec.alpha :as s]
[beherbergung.model.offer :refer [int_string]]
[clojure.string :refer [split]]))
(defn split_user_string
"TODO handle common separators entered by users like `,` or `and`"
[s]
(split s #" "))
(def mapping {:id_tmp #(or (not-empty (get % "E-Mail ")) (get % "Phone"))
:time_from_str (constantly nil)
:time_duration_str "Available from- , to / Verfügbar von- bis"
:beds "How many people can you host? / Wievielen Menschen können sie Unterkunft bieten?"
:languages ["The language you speak / Gesprochene Sprachen" split_user_string]
:place_country "Country / Land"
:place_city (constantly nil)
:place_zip (constantly nil)
:place_street (constantly nil)
:place_street_number (constantly nil)
:accessible (constantly nil)
:animals_allowed (constantly nil)
:animals_present (constantly nil)
:contact_name_full "Name "
:contact_phone "Phone"
:contact_email "E-Mail "
:note "Anything else to keep in mind? Animals? Allergies? / Gibt es sonst noch etwas zu bedenken? Tiere? Allergien?"})