From 78a340a1d0e0e6fb682a3a726ae92fa952ff4b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20L=C3=B6tzsch?= Date: Wed, 16 Mar 2022 17:52:15 +0100 Subject: [PATCH] backend: import first warhelp sample datasets --- .../src/beherbergung/db/import/offer/core.clj | 20 +++++++----- .../db/import/offer/ngo/warhelp.clj | 8 +++++ .../model/offer_mapping/lifeline.clj | 2 +- .../model/offer_mapping/warhelp.clj | 32 +++++++++++++++++++ .../csv2edn/.gitignore | 0 .../csv2edn/README.md | 0 .../csv2edn/project.clj | 0 .../csv2edn/src/csv2edn/core.clj | 0 8 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 backend/src/beherbergung/db/import/offer/ngo/warhelp.clj create mode 100644 backend/src/beherbergung/model/offer_mapping/warhelp.clj rename import/{api/wpforms-crawler => converter}/csv2edn/.gitignore (100%) rename import/{api/wpforms-crawler => converter}/csv2edn/README.md (100%) rename import/{api/wpforms-crawler => converter}/csv2edn/project.clj (100%) rename import/{api/wpforms-crawler => converter}/csv2edn/src/csv2edn/core.clj (100%) diff --git a/backend/src/beherbergung/db/import/offer/core.clj b/backend/src/beherbergung/db/import/offer/core.clj index 32749c6..81571fa 100644 --- a/backend/src/beherbergung/db/import/offer/core.clj +++ b/backend/src/beherbergung/db/import/offer/core.clj @@ -2,7 +2,8 @@ (:require [beherbergung.config.state :refer [env]] [beherbergung.db.import.offer.helper :refer [update-offers]] [beherbergung.db.import.offer.ngo.random :as random] - [beherbergung.db.import.offer.ngo.lifeline :as lifeline])) + [beherbergung.db.import.offer.ngo.lifeline :as lifeline] + [beherbergung.db.import.offer.ngo.warhelp :as warhelp])) (defn import! [] (if-not (:import-ngo env) @@ -12,6 +13,8 @@ (case (:import-ngo env) "lifeline_beherbergung" (lifeline/importfile->table (:import-file env)) + "warhelp_beherbergung" + (warhelp/importfile->table (:import-file env)) (random/importfile->table)))] (println "Records to be imported:" (count table)) (update-offers (:import-ngo env) table) @@ -20,11 +23,12 @@ (comment (import!) - (let [ngo:id "lifeline_beherbergung" - _ (require '[beherbergung.db.state :refer [db_ctx]]) + (require '[beherbergung.db.state :refer [db_ctx]]) + (let [ngo:id "warhelp_beherbergung" {:keys [q_unary]} db_ctx] - (-> (q_unary '{:find [(pull ?e [*])] - :where [[?e :xt/spec :beherbergung.model.offer/record] - [?e :beherbergung.model.ngo/id ngo:id]] - :in [ngo:id]} - ngo:id)))) + (->> (q_unary '{:find [(pull ?e [*])] + :where [[?e :xt/spec :beherbergung.model.offer/record] + [?e :beherbergung.model.ngo/id ngo:id]] + :in [ngo:id]} + ngo:id) + #_(map :id_tmp)))) diff --git a/backend/src/beherbergung/db/import/offer/ngo/warhelp.clj b/backend/src/beherbergung/db/import/offer/ngo/warhelp.clj new file mode 100644 index 0000000..7477c65 --- /dev/null +++ b/backend/src/beherbergung/db/import/offer/ngo/warhelp.clj @@ -0,0 +1,8 @@ +(ns beherbergung.db.import.offer.ngo.warhelp + (:require [clojure.edn] + [beherbergung.model.offer-mapping.core :refer [unify]] + [beherbergung.model.offer-mapping.warhelp])) + +(defn importfile->table [file] + (take 5 (unify (clojure.edn/read-string (slurp file)) ;; TODO all + beherbergung.model.offer-mapping.warhelp/mapping))) diff --git a/backend/src/beherbergung/model/offer_mapping/lifeline.clj b/backend/src/beherbergung/model/offer_mapping/lifeline.clj index 49a1945..6806559 100644 --- a/backend/src/beherbergung/model/offer_mapping/lifeline.clj +++ b/backend/src/beherbergung/model/offer_mapping/lifeline.clj @@ -6,7 +6,7 @@ (defn JaNein->bool [JaNein] ({"Ja" true "Nein" false} JaNein)) -(def mapping {:id_tmp #(or (get % "E-Mail") (get % "Telefonnummer")) +(def mapping {:id_tmp #(or (not-empty (get % "E-Mail")) (get % "Telefonnummer")) :time_from_str "frühestes Einzugsdatum" :time_duration_str "Möglicher Aufenthalt (Dauer)" ;; TODO: the duration is not parsed till now diff --git a/backend/src/beherbergung/model/offer_mapping/warhelp.clj b/backend/src/beherbergung/model/offer_mapping/warhelp.clj new file mode 100644 index 0000000..c5d9f9b --- /dev/null +++ b/backend/src/beherbergung/model/offer_mapping/warhelp.clj @@ -0,0 +1,32 @@ +(ns beherbergung.model.offer-mapping.warhelp + (:require [clojure.spec.alpha :as s] + [beherbergung.model.offer :refer [int_string]] + [clojure.string :refer [split]])) + +(defn split_user_string + "TODO handle common separators entered by users like `,` or `and`" + [s] + (split s #" ")) + +(def mapping {:id_tmp #(or (not-empty (get % "E-Mail ")) (get % "Phone")) + + :time_from_str (constantly nil) + :time_duration_str "Available from- , to / Verfügbar von- bis" + + :beds "How many people can you host? / Wievielen Menschen können sie Unterkunft bieten?" + :languages ["The language you speak / Gesprochene Sprachen" split_user_string] + + :place_country "Country / Land" + :place_city (constantly nil) + :place_zip (constantly nil) + :place_street (constantly nil) + :place_street_number (constantly nil) + + :accessible (constantly nil) + :animals_allowed (constantly nil) + :animals_present (constantly nil) + + :contact_name_full "Name " + :contact_phone "Phone" + :contact_email "E-Mail " + :note "Anything else to keep in mind? Animals? Allergies? / Gibt es sonst noch etwas zu bedenken? Tiere? Allergien?"}) diff --git a/import/api/wpforms-crawler/csv2edn/.gitignore b/import/converter/csv2edn/.gitignore similarity index 100% rename from import/api/wpforms-crawler/csv2edn/.gitignore rename to import/converter/csv2edn/.gitignore diff --git a/import/api/wpforms-crawler/csv2edn/README.md b/import/converter/csv2edn/README.md similarity index 100% rename from import/api/wpforms-crawler/csv2edn/README.md rename to import/converter/csv2edn/README.md diff --git a/import/api/wpforms-crawler/csv2edn/project.clj b/import/converter/csv2edn/project.clj similarity index 100% rename from import/api/wpforms-crawler/csv2edn/project.clj rename to import/converter/csv2edn/project.clj diff --git a/import/api/wpforms-crawler/csv2edn/src/csv2edn/core.clj b/import/converter/csv2edn/src/csv2edn/core.clj similarity index 100% rename from import/api/wpforms-crawler/csv2edn/src/csv2edn/core.clj rename to import/converter/csv2edn/src/csv2edn/core.clj