Browse Source

backend: import first warhelp sample datasets

main
Johannes Lötzsch 5 months ago
parent
commit
78a340a1d0
  1. 20
      backend/src/beherbergung/db/import/offer/core.clj
  2. 8
      backend/src/beherbergung/db/import/offer/ngo/warhelp.clj
  3. 2
      backend/src/beherbergung/model/offer_mapping/lifeline.clj
  4. 32
      backend/src/beherbergung/model/offer_mapping/warhelp.clj
  5. 0
      import/converter/csv2edn/.gitignore
  6. 0
      import/converter/csv2edn/README.md
  7. 0
      import/converter/csv2edn/project.clj
  8. 0
      import/converter/csv2edn/src/csv2edn/core.clj

20
backend/src/beherbergung/db/import/offer/core.clj

@ -2,7 +2,8 @@
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.import.offer.helper :refer [update-offers]]
[beherbergung.db.import.offer.ngo.random :as random]
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]))
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]
[beherbergung.db.import.offer.ngo.warhelp :as warhelp]))
(defn import! []
(if-not (:import-ngo env)
@ -12,6 +13,8 @@
(case (:import-ngo env)
"lifeline_beherbergung"
(lifeline/importfile->table (:import-file env))
"warhelp_beherbergung"
(warhelp/importfile->table (:import-file env))
(random/importfile->table)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
@ -20,11 +23,12 @@
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
_ (require '[beherbergung.db.state :refer [db_ctx]])
(require '[beherbergung.db.state :refer [db_ctx]])
(let [ngo:id "warhelp_beherbergung"
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))
(->> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id)
#_(map :id_tmp))))

8
backend/src/beherbergung/db/import/offer/ngo/warhelp.clj

@ -0,0 +1,8 @@
(ns beherbergung.db.import.offer.ngo.warhelp
(:require [clojure.edn]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.warhelp]))
(defn importfile->table [file]
(take 5 (unify (clojure.edn/read-string (slurp file)) ;; TODO all
beherbergung.model.offer-mapping.warhelp/mapping)))

2
backend/src/beherbergung/model/offer_mapping/lifeline.clj

@ -6,7 +6,7 @@
(defn JaNein->bool [JaNein]
({"Ja" true "Nein" false} JaNein))
(def mapping {:id_tmp #(or (get % "E-Mail") (get % "Telefonnummer"))
(def mapping {:id_tmp #(or (not-empty (get % "E-Mail")) (get % "Telefonnummer"))
:time_from_str "frühestes Einzugsdatum"
:time_duration_str "Möglicher Aufenthalt (Dauer)" ;; TODO: the duration is not parsed till now

32
backend/src/beherbergung/model/offer_mapping/warhelp.clj

@ -0,0 +1,32 @@
(ns beherbergung.model.offer-mapping.warhelp
(:require [clojure.spec.alpha :as s]
[beherbergung.model.offer :refer [int_string]]
[clojure.string :refer [split]]))
(defn split_user_string
"TODO handle common separators entered by users like `,` or `and`"
[s]
(split s #" "))
(def mapping {:id_tmp #(or (not-empty (get % "E-Mail ")) (get % "Phone"))
:time_from_str (constantly nil)
:time_duration_str "Available from- , to / Verfügbar von- bis"
:beds "How many people can you host? / Wievielen Menschen können sie Unterkunft bieten?"
:languages ["The language you speak / Gesprochene Sprachen" split_user_string]
:place_country "Country / Land"
:place_city (constantly nil)
:place_zip (constantly nil)
:place_street (constantly nil)
:place_street_number (constantly nil)
:accessible (constantly nil)
:animals_allowed (constantly nil)
:animals_present (constantly nil)
:contact_name_full "Name "
:contact_phone "Phone"
:contact_email "E-Mail "
:note "Anything else to keep in mind? Animals? Allergies? / Gibt es sonst noch etwas zu bedenken? Tiere? Allergien?"})

0
import/api/wpforms-crawler/csv2edn/.gitignore → import/converter/csv2edn/.gitignore vendored

0
import/api/wpforms-crawler/csv2edn/README.md → import/converter/csv2edn/README.md

0
import/api/wpforms-crawler/csv2edn/project.clj → import/converter/csv2edn/project.clj

0
import/api/wpforms-crawler/csv2edn/src/csv2edn/core.clj → import/converter/csv2edn/src/csv2edn/core.clj

Loading…
Cancel
Save