backend: refactored import

main
Johannes Lötzsch 7 months ago
parent bd56ea9487
commit f7460f320e
  1. 30
      backend/src/beherbergung/db/import/offer/core.clj
  2. 35
      backend/src/beherbergung/db/import/offer/helper.clj
  3. 8
      backend/src/beherbergung/db/import/offer/ngo/lifeline.clj
  4. 7
      backend/src/beherbergung/db/import/offer/ngo/random.clj
  5. 2
      backend/src/beherbergung/webserver/state.clj

@ -0,0 +1,30 @@
(ns beherbergung.db.import.offer.core
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.import.offer.helper :refer [update-offers]]
[beherbergung.db.import.offer.ngo.random :as random]
[beherbergung.db.import.offer.ngo.lifeline :as lifeline]))
(defn import! []
(if-not (:import-ngo env)
(println "No IMPORT_NGO defined")
(let [table (if-not (:import-file env)
(random/importfile->table)
(case (:import-ngo env)
"lifeline_beherbergung"
(lifeline/importfile->table (:import-file env))
(random/importfile->table)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
(println "import finished :)"))))
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
_ (require '[beherbergung.db.state :refer [db_ctx]])
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec :beherbergung.model.offer/record]
[?e :beherbergung.model.ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))

@ -1,13 +1,9 @@
(ns beherbergung.db.import.offer.lifeline
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.state :refer [db_ctx]]
(ns beherbergung.db.import.offer.helper
(:require [beherbergung.db.state :refer [db_ctx]]
[clj-http.client :as client]
[beherbergung.auth.uuid.core :refer [uuid]]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]
[beherbergung.model.offer :as offer]
[beherbergung.model.ngo :as ngo]
[clj-http.client :as client]
[clojure.edn]))
[beherbergung.model.ngo :as ngo]))
(defn geocode [record]
(let [params {:city (:place_city record)}
@ -38,26 +34,3 @@
(tx-fn-call :update-offer (or existingId (uuid))
(geocode record)
ngo:id))))
(defn import! []
(if-not (:import-ngo env)
(println "No IMPORT_NGO defined")
(let [table (if (:import-file env)
(unify (clojure.edn/read-string (slurp (:import-file env)))
beherbergung.model.offer-mapping.lifeline/mapping)
(clojure.edn/read-string (slurp "./data/import/example.edn")) ;; till conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))]
(println "Records to be imported:" (count table))
(update-offers (:import-ngo env) table)
(println "import finished :)"))))
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec ::offer/record]
[?e ::ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))

@ -0,0 +1,8 @@
(ns beherbergung.db.import.offer.ngo.lifeline
(:require [clojure.edn]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]))
(defn importfile->table [file]
(unify (clojure.edn/read-string (slurp file))
beherbergung.model.offer-mapping.lifeline/mapping))

@ -0,0 +1,7 @@
(ns beherbergung.db.import.offer.ngo.random
(:require [clojure.edn]))
(defn importfile->table []
(clojure.edn/read-string (slurp "./data/import/example.edn"))
;; once conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))

@ -3,7 +3,7 @@
(:require [ring.adapter.jetty]
[ring.middleware.reload]
[beherbergung.webserver.handler]
[beherbergung.db.import.offer.lifeline :refer [import!]]
[beherbergung.db.import.offer.core :refer [import!]]
[mount.core :as mount :refer [defstate]]
[beherbergung.config.state]
[signal.handler :refer [with-handler]]))

Loading…
Cancel
Save