backend: refactored import

This commit is contained in:
Johannes Lötzsch 2022-03-16 15:16:35 +01:00
parent bd56ea9487
commit f7460f320e
5 changed files with 51 additions and 33 deletions

View File

@ -0,0 +1,30 @@
(ns beherbergung.db.import.offer.core
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.import.offer.helper :refer [update-offers]]
[beherbergung.db.import.offer.ngo.random :as random]
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]))
(defn import! []
(if-not (:import-ngo env)
(println "No IMPORT_NGO defined")
(let [table (if-not (:import-file env)
(random/importfile->table)
(case (:import-ngo env)
"lifeline_beherbergung"
(lifeline/importfile->table (:import-file env))
(random/importfile->table)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
(println "import finished :)"))))
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
_ (require '[beherbergung.db.state :refer [db_ctx]])
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))

View File

@ -1,13 +1,9 @@
(ns beherbergung.db.import.offer.lifeline
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.state :refer [db_ctx]]
[beherbergung.auth.uuid.core :refer [uuid]]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]
[beherbergung.model.offer :as offer]
[beherbergung.model.ngo :as ngo]
(ns beherbergung.db.import.offer.helper
(:require [beherbergung.db.state :refer [db_ctx]]
[clj-http.client :as client]
[clojure.edn]))
[beherbergung.auth.uuid.core :refer [uuid]]
[beherbergung.model.offer :as offer]
[beherbergung.model.ngo :as ngo]))
(defn geocode [record]
(let [params {:city (:place_city record)}
@ -38,26 +34,3 @@
(tx-fn-call :update-offer (or existingId (uuid))
(geocode record)
ngo:id))))
(defn import! []
(if-not (:import-ngo env)
(println "No IMPORT_NGO defined")
(let [table (if (:import-file env)
(unify (clojure.edn/read-string (slurp (:import-file env)))
beherbergung.model.offer-mapping.lifeline/mapping)
(clojure.edn/read-string (slurp "./data/import/example.edn")) ;; till conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
(println "import finished :)"))))
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec ::offer/record]
[?e ::ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))

View File

@ -0,0 +1,8 @@
(ns beherbergung.db.import.offer.ngo.lifeline
(:require [clojure.edn]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]))
(defn importfile->table [file]
(unify (clojure.edn/read-string (slurp file))
beherbergung.model.offer-mapping.lifeline/mapping))

View File

@ -0,0 +1,7 @@
(ns beherbergung.db.import.offer.ngo.random
(:require [clojure.edn]))
(defn importfile->table []
(clojure.edn/read-string (slurp "./data/import/example.edn"))
;; once conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))

View File

@ -3,7 +3,7 @@
(:require [ring.adapter.jetty]
[ring.middleware.reload]
[beherbergung.webserver.handler]
[beherbergung.db.import.offer.lifeline :refer [import!]]
[beherbergung.db.import.offer.core :refer [import!]]
[mount.core :as mount :refer [defstate]]
[beherbergung.config.state]
[signal.handler :refer [with-handler]]))