From f7460f320e3703a3751a81b0c2732dc7c60d352e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20L=C3=B6tzsch?= Date: Wed, 16 Mar 2022 15:16:35 +0100 Subject: [PATCH] backend: refactored import --- .../src/beherbergung/db/import/offer/core.clj | 30 +++++++++++++++ .../import/offer/{lifeline.clj => helper.clj} | 37 +++---------------- .../db/import/offer/ngo/lifeline.clj | 8 ++++ .../db/import/offer/ngo/random.clj | 7 ++++ backend/src/beherbergung/webserver/state.clj | 2 +- 5 files changed, 51 insertions(+), 33 deletions(-) create mode 100644 backend/src/beherbergung/db/import/offer/core.clj rename backend/src/beherbergung/db/import/offer/{lifeline.clj => helper.clj} (55%) create mode 100644 backend/src/beherbergung/db/import/offer/ngo/lifeline.clj create mode 100644 backend/src/beherbergung/db/import/offer/ngo/random.clj diff --git a/backend/src/beherbergung/db/import/offer/core.clj b/backend/src/beherbergung/db/import/offer/core.clj new file mode 100644 index 0000000..32749c6 --- /dev/null +++ b/backend/src/beherbergung/db/import/offer/core.clj @@ -0,0 +1,30 @@ +(ns beherbergung.db.import.offer.core + (:require [beherbergung.config.state :refer [env]] + [beherbergung.db.import.offer.helper :refer [update-offers]] + [beherbergung.db.import.offer.ngo.random :as random] + [beherbergung.db.import.offer.ngo.lifeline :as lifeline])) + +(defn import! [] + (if-not (:import-ngo env) + (println "No IMPORT_NGO defined") + (let [table (if-not (:import-file env) + (random/importfile->table) + (case (:import-ngo env) + "lifeline_beherbergung" + (lifeline/importfile->table (:import-file env)) + (random/importfile->table)))] + (println "Records to be imported:" (count table)) + (update-offers (:import-ngo env) table) + (println "import finished :)")))) + +(comment + (import!) + + (let [ngo:id "lifeline_beherbergung" + _ (require '[beherbergung.db.state :refer [db_ctx]]) + {:keys [q_unary]} db_ctx] + (-> (q_unary '{:find [(pull ?e [*])] + :where [[?e :xt/spec :beherbergung.model.offer/record] + [?e :beherbergung.model.ngo/id ngo:id]] + :in [ngo:id]} + ngo:id)))) diff --git a/backend/src/beherbergung/db/import/offer/lifeline.clj b/backend/src/beherbergung/db/import/offer/helper.clj similarity index 55% rename from backend/src/beherbergung/db/import/offer/lifeline.clj rename to backend/src/beherbergung/db/import/offer/helper.clj index d537c14..dd47126 100644 --- a/backend/src/beherbergung/db/import/offer/lifeline.clj +++ b/backend/src/beherbergung/db/import/offer/helper.clj @@ -1,13 +1,9 @@ -(ns beherbergung.db.import.offer.lifeline - (:require [beherbergung.config.state :refer [env]] - [beherbergung.db.state :refer [db_ctx]] - [beherbergung.auth.uuid.core :refer [uuid]] - [beherbergung.model.offer-mapping.core :refer [unify]] - [beherbergung.model.offer-mapping.lifeline] - [beherbergung.model.offer :as offer] - [beherbergung.model.ngo :as ngo] +(ns beherbergung.db.import.offer.helper + (:require [beherbergung.db.state :refer [db_ctx]] [clj-http.client :as client] - [clojure.edn])) + [beherbergung.auth.uuid.core :refer [uuid]] + [beherbergung.model.offer :as offer] + [beherbergung.model.ngo :as ngo])) (defn geocode [record] (let [params {:city (:place_city record)} @@ -38,26 +34,3 @@ (tx-fn-call :update-offer (or existingId (uuid)) (geocode record) ngo:id)))) - -(defn import! [] - (if-not (:import-ngo env) - (println "No IMPORT_NGO defined") - (let [table (if (:import-file env) - (unify (clojure.edn/read-string (slurp (:import-file env))) - beherbergung.model.offer-mapping.lifeline/mapping) - (clojure.edn/read-string (slurp "./data/import/example.edn")) ;; till conflict between `specialist-server.type` and `with-gen` is fixed - #_(gen/sample (s/gen ::offer)))] - (println "Records to be imported:" (count table)) - (update-offers (:import-ngo env) table) - (println "import finished :)")))) - -(comment - (import!) - - (let [ngo:id "lifeline_beherbergung" - {:keys [q_unary]} db_ctx] - (-> (q_unary '{:find [(pull ?e [*])] - :where [[?e :xt/spec ::offer/record] - [?e ::ngo/id ngo:id]] - :in [ngo:id]} - ngo:id)))) diff --git a/backend/src/beherbergung/db/import/offer/ngo/lifeline.clj b/backend/src/beherbergung/db/import/offer/ngo/lifeline.clj new file mode 100644 index 0000000..cbcde4e --- /dev/null +++ b/backend/src/beherbergung/db/import/offer/ngo/lifeline.clj @@ -0,0 +1,8 @@ +(ns beherbergung.db.import.offer.ngo.lifeline + (:require [clojure.edn] + [beherbergung.model.offer-mapping.core :refer [unify]] + [beherbergung.model.offer-mapping.lifeline])) + +(defn importfile->table [file] + (unify (clojure.edn/read-string (slurp file)) + beherbergung.model.offer-mapping.lifeline/mapping)) diff --git a/backend/src/beherbergung/db/import/offer/ngo/random.clj b/backend/src/beherbergung/db/import/offer/ngo/random.clj new file mode 100644 index 0000000..a3b3867 --- /dev/null +++ b/backend/src/beherbergung/db/import/offer/ngo/random.clj @@ -0,0 +1,7 @@ +(ns beherbergung.db.import.offer.ngo.random + (:require [clojure.edn])) + +(defn importfile->table [] + (clojure.edn/read-string (slurp "./data/import/example.edn")) + ;; once conflict between `specialist-server.type` and `with-gen` is fixed + #_(gen/sample (s/gen ::offer))) diff --git a/backend/src/beherbergung/webserver/state.clj b/backend/src/beherbergung/webserver/state.clj index ad609bc..288af79 100644 --- a/backend/src/beherbergung/webserver/state.clj +++ b/backend/src/beherbergung/webserver/state.clj @@ -3,7 +3,7 @@ (:require [ring.adapter.jetty] [ring.middleware.reload] [beherbergung.webserver.handler] - [beherbergung.db.import.offer.lifeline :refer [import!]] + [beherbergung.db.import.offer.core :refer [import!]] [mount.core :as mount :refer [defstate]] [beherbergung.config.state] [signal.handler :refer [with-handler]]))