backend: import + serve offers from db

This commit is contained in:
Johannes Lötzsch 2022-03-15 02:27:32 +01:00
parent 3e18b45cd8
commit e381675a39
11 changed files with 112 additions and 48 deletions

View File

@ -1,4 +1,5 @@
({:id "0" ({:id "0"
:id_tmp "0"
:beds 0, :beds 0,
:place_street "", :place_street "",
:contact_email nil, :contact_email nil,
@ -6,16 +7,17 @@
:time_duration_str "", :time_duration_str "",
:note "", :note "",
:place_street_number "", :place_street_number "",
:place_city "", :place_city "Dresden",
:contact_phone nil, :contact_phone nil,
:place_zip "", :place_zip "",
:time_from_str "10/04/2022", :time_from_str "10/04/2022",
:place_country "", :place_country "Deutschland",
:animals_present false, :animals_present false,
:languages (), :languages (),
:accessible false, :accessible false,
:animals_allowed false} :animals_allowed false}
{:id "1" {:id "1"
:id_tmp "1"
:beds nil, :beds nil,
:place_street "g", :place_street "g",
:contact_email "8", :contact_email "8",
@ -23,7 +25,7 @@
:time_duration_str "5", :time_duration_str "5",
:note "F", :note "F",
:place_street_number "", :place_street_number "",
:place_city "", :place_city "Leipzig",
:contact_phone nil, :contact_phone nil,
:place_zip "", :place_zip "",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
@ -33,6 +35,7 @@
:accessible nil, :accessible nil,
:animals_allowed false} :animals_allowed false}
{:id "2" {:id "2"
:id_tmp "2"
:beds 1, :beds 1,
:place_street "p8", :place_street "p8",
:contact_email "i", :contact_email "i",
@ -40,16 +43,17 @@
:time_duration_str "D0", :time_duration_str "D0",
:note "0", :note "0",
:place_street_number "", :place_street_number "",
:place_city nil, :place_city "Chemnitz",
:contact_phone "Q", :contact_phone "Q",
:place_zip "RD", :place_zip "RD",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
:place_country nil, :place_country "Germany",
:animals_present true, :animals_present true,
:languages (), :languages (),
:accessible nil, :accessible nil,
:animals_allowed true} :animals_allowed true}
{:id "3" {:id "3"
:id_tmp "3"
:beds 0, :beds 0,
:place_street "w4", :place_street "w4",
:contact_email "5", :contact_email "5",
@ -57,16 +61,17 @@
:time_duration_str "6GW", :time_duration_str "6GW",
:note nil, :note nil,
:place_street_number "", :place_street_number "",
:place_city "1bh", :place_city "Mittweida",
:contact_phone nil, :contact_phone nil,
:place_zip "PG4", :place_zip "PG4",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
:place_country "I7", :place_country "Deutschland",
:animals_present false, :animals_present false,
:languages ("x" "ni"), :languages ("x" "ni"),
:accessible false, :accessible false,
:animals_allowed nil} :animals_allowed nil}
{:id "4" {:id "4"
:id_tmp "4"
:beds 2, :beds 2,
:place_street nil, :place_street nil,
:contact_email nil, :contact_email nil,
@ -74,16 +79,17 @@
:time_duration_str "wES5", :time_duration_str "wES5",
:note "", :note "",
:place_street_number "z7g", :place_street_number "z7g",
:place_city "x", :place_city "Freiberg",
:contact_phone "w8l", :contact_phone "w8l",
:place_zip "1E", :place_zip "1E",
:time_from_str "01/01/2022", :time_from_str "01/01/2022",
:place_country "8", :place_country "Deutschland",
:animals_present nil, :animals_present nil,
:languages (), :languages (),
:accessible true, :accessible true,
:animals_allowed true} :animals_allowed true}
{:id "5" {:id "5"
:id_tmp "5"
:beds 1, :beds 1,
:place_street "4N2", :place_street "4N2",
:contact_email "u1", :contact_email "u1",
@ -91,7 +97,7 @@
:time_duration_str "P0E5", :time_duration_str "P0E5",
:note "Ue7F1", :note "Ue7F1",
:place_street_number "9Ms7", :place_street_number "9Ms7",
:place_city "6", :place_city "Radeberg",
:contact_phone "OI", :contact_phone "OI",
:place_zip "1", :place_zip "1",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
@ -101,6 +107,7 @@
:accessible false, :accessible false,
:animals_allowed false} :animals_allowed false}
{:id "6" {:id "6"
:id_tmp "6"
:beds 0, :beds 0,
:place_street "T", :place_street "T",
:contact_email "W1wL", :contact_email "W1wL",
@ -108,16 +115,17 @@
:time_duration_str "", :time_duration_str "",
:note nil, :note nil,
:place_street_number "yPkLR", :place_street_number "yPkLR",
:place_city "cj1A2", :place_city "Radebeul",
:contact_phone "r", :contact_phone "r",
:place_zip "cJ9z7", :place_zip "cJ9z7",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
:place_country "5", :place_country "Deutschland",
:animals_present true, :animals_present true,
:languages nil, :languages nil,
:accessible false, :accessible false,
:animals_allowed true} :animals_allowed true}
{:id "7" {:id "7"
:id_tmp "7"
:beds nil, :beds nil,
:place_street "0IDd", :place_street "0IDd",
:contact_email "2511JLD", :contact_email "2511JLD",
@ -125,16 +133,17 @@
:time_duration_str "kP1S8pE", :time_duration_str "kP1S8pE",
:note "P3", :note "P3",
:place_street_number nil, :place_street_number nil,
:place_city "4ML4yG", :place_city "Meißen",
:contact_phone "7uQ3Px", :contact_phone "7uQ3Px",
:place_zip "kp5IS", :place_zip "kp5IS",
:time_from_str "03/04/2022", :time_from_str "03/04/2022",
:place_country nil, :place_country "Deutschland",
:animals_present true, :animals_present true,
:languages (), :languages (),
:accessible false, :accessible false,
:animals_allowed false} :animals_allowed false}
{:id "8" {:id "8"
:id_tmp "8"
:beds 4, :beds 4,
:place_street "Wy", :place_street "Wy",
:contact_email "35e6Rnj", :contact_email "35e6Rnj",
@ -142,16 +151,17 @@
:time_duration_str "rh9n6q4", :time_duration_str "rh9n6q4",
:note "89Rxzv", :note "89Rxzv",
:place_street_number "", :place_street_number "",
:place_city "u6k41v", :place_city "Pirna",
:contact_phone "354VW", :contact_phone "354VW",
:place_zip "DI19", :place_zip "DI19",
:time_from_str "09/04/2022", :time_from_str "09/04/2022",
:place_country nil, :place_country "Deutschland",
:animals_present true, :animals_present true,
:languages nil, :languages nil,
:accessible true, :accessible true,
:animals_allowed false} :animals_allowed false}
{:id "9" {:id "9"
:id_tmp "9"
:beds 27, :beds 27,
:place_street "wIbUsu", :place_street "wIbUsu",
:contact_email "x0Vl9", :contact_email "x0Vl9",
@ -159,11 +169,11 @@
:time_duration_str "33", :time_duration_str "33",
:note "Z", :note "Z",
:place_street_number "VB3", :place_street_number "VB3",
:place_city "f", :place_city "Freital",
:contact_phone "sQdB", :contact_phone "sQdB",
:place_zip "cid1", :place_zip "cid1",
:time_from_str "03/05/2022", :time_from_str "03/05/2022",
:place_country "toTz5B", :place_country "Deutschland",
:animals_present false, :animals_present false,
:languages ("vMR0zyECr"), :languages ("vMR0zyECr"),
:accessible false, :accessible false,

View File

@ -0,0 +1,54 @@
(ns beherbergung.db.import.offer.lifeline
(:require [beherbergung.config.state :refer [env]]
[beherbergung.db.state :refer [db_ctx]]
[beherbergung.auth.uuid.core :refer [uuid]]
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]
[beherbergung.model.offer :as offer]
[beherbergung.model.ngo :as ngo]
[clojure.edn]))
(defn geocode
[record]
(assoc record
:place_lon 12.34
:place_lat 51.34))
(defn update-offers [table]
(let [ngo:id "lifeline_beherbergung" ;; TODO
{:keys [tx-fn-put tx-fn-call]} db_ctx]
(tx-fn-put :update-offer
'(fn [ctx eid doc ngo:id]
(let [db (xtdb.api/db ctx)
entity (xtdb.api/entity db eid)]
[[:xtdb.api/put (assoc (merge entity doc)
:xt/id eid
:xt/spec ::offer/record
::ngo/id ngo:id)]])))
(doseq [record table
:let [existingId (when (:id_tmp record)
(str "offer_" (:id_tmp record)))]]
(tx-fn-call :update-offer (or existingId (uuid))
(geocode record)
ngo:id))))
(defn import! []
(let [table (if (:import-file env)
(unify (clojure.edn/read-string (slurp (:import-file env)))
beherbergung.model.offer-mapping.lifeline/mapping)
(clojure.edn/read-string (slurp "./data/import/example.edn")) ;; till conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))]
(println "Records to be imported:" (count table))
(update-offers table))
(println "imported :)"))
(comment
(import!)
(let [ngo:id "lifeline_beherbergung"
{:keys [q_unary]} db_ctx]
(-> (q_unary '{:find [(pull ?e [*])]
:where [[?e :xt/spec ::offer/record]
[?e ::ngo/id ngo:id]]
:in [ngo:id]}
ngo:id))))

View File

@ -2,9 +2,10 @@
(:require [clojure.spec.alpha :as s] (:require [clojure.spec.alpha :as s]
[specialist-server.type :as t])) [specialist-server.type :as t]))
(s/def ::id t/string)
(s/def ::ngo (s/keys :req-un [::id ::name])) (s/def ::ngo (s/keys :req-un [::id ::name]))
(s/def ::ngo:id t/id) (s/def ::ngo:id t/id) ;; TODO remove
(t/defscalar NgoRefs (t/defscalar NgoRefs
{:name "NgoRefs" :description "Either a collection of ngo-ids or `any`"} {:name "NgoRefs" :description "Either a collection of ngo-ids or `any`"}

View File

@ -23,7 +23,8 @@
(s/def ::t_int_string int_string #_ (s/with-gen t/int #(s/gen int?))) (s/def ::t_int_string int_string #_ (s/with-gen t/int #(s/gen int?)))
(s/def ::t_float t/float) (s/def ::t_float t/float)
(s/def :xtdb.api/id (s/nilable ::t_string)) ;; TODO: in future not nilable (s/def :xt/id ::t_string)
(s/def ::id_tmp (s/nilable ::t_string))
(s/def ::time_from_str (s/nilable ::t_string)) (s/def ::time_from_str (s/nilable ::t_string))
(s/def ::time_duration_str (s/nilable ::t_string)) (s/def ::time_duration_str (s/nilable ::t_string))
(s/def ::beds (s/nilable ::t_int_string)) (s/def ::beds (s/nilable ::t_int_string))
@ -42,7 +43,7 @@
(s/def ::contact_phone (s/nilable ::t_string)) (s/def ::contact_phone (s/nilable ::t_string))
(s/def ::contact_email (s/nilable ::t_string)) (s/def ::contact_email (s/nilable ::t_string))
(s/def ::note (s/nilable ::t_string)) (s/def ::note (s/nilable ::t_string))
(s/def ::offer (s/keys :req-un [:xtdb.api/id (s/def ::offer (s/keys :req-un [:xtdb.api/id ::id_tmp
::time_from_str ::time_duration_str ::beds ::languages ::time_from_str ::time_duration_str ::beds ::languages
::place_country ::place_city ::place_zip ::place_street ::place_street_number ::place_country ::place_city ::place_zip ::place_street ::place_street_number
::place_lon ::place_lat ::place_lon ::place_lat
@ -53,3 +54,9 @@
(comment (comment
(write-edn "./data/sample-data/example.edn" (write-edn "./data/sample-data/example.edn"
(gen/sample (s/gen ::offer)))) (gen/sample (s/gen ::offer))))
(s/def ::record ::offer)
(defn db->graphql [record]
(some-> record
(assoc :id (:xt/id record))))

View File

@ -6,7 +6,7 @@
(defn JaNein->bool [JaNein] (defn JaNein->bool [JaNein]
({"Ja" true "Nein" false} JaNein)) ({"Ja" true "Nein" false} JaNein))
(def mapping {:id #(or (get % "E-Mail") (get % "Telefonnummer")) ;; TODO: uuid will be generated when record is written to db (def mapping {:id_tmp #(or (get % "E-Mail") (get % "Telefonnummer"))
:time_from_str "frühestes Einzugsdatum" :time_from_str "frühestes Einzugsdatum"
:time_duration_str "Möglicher Aufenthalt (Dauer)" ;; TODO: the duration is not parsed till now :time_duration_str "Möglicher Aufenthalt (Dauer)" ;; TODO: the duration is not parsed till now

View File

@ -2,40 +2,25 @@
(:require [clojure.spec.alpha :as s] (:require [clojure.spec.alpha :as s]
[specialist-server.type :as t] [specialist-server.type :as t]
[beherbergung.auth.core :refer [auth+role->entity]] [beherbergung.auth.core :refer [auth+role->entity]]
[beherbergung.config.state :refer [env]]
[beherbergung.model.auth :as auth] [beherbergung.model.auth :as auth]
[beherbergung.model.ngo :as ngo] [beherbergung.model.ngo :as ngo]
[beherbergung.model.offer :as offer] [beherbergung.model.offer :as offer :refer [db->graphql]]))
[beherbergung.model.offer-mapping.core :refer [unify]]
[beherbergung.model.offer-mapping.lifeline]
[clojure.edn]))
(defn mock_geocoding
"just to provide sample data while developing the frontend"
[table]
(map #(-> %
(assoc :place_lon 12.34
:place_lat 51.34))
table))
(s/fdef get_offers (s/fdef get_offers
:args (s/tuple map? (s/keys :req-un [::auth/auth]) map? map?) :args (s/tuple map? (s/keys :req-un [::auth/auth]) map? map?)
:ret (s/nilable (s/* ::offer/offer))) :ret (s/nilable (s/* ::offer/offer)))
(defn get_offers (defn get_offers
"The offers that are visible for the ngo, belonging to the login" "The offers that are visible for the ngo, belonging to the login"
[_node opt ctx _info] [_node opt ctx _info]
(let [{:keys [_TODO]} (:db_ctx ctx) (let [{:keys [q_unary]} (:db_ctx ctx)
[ngo:id] (auth+role->entity ctx (:auth opt) ::ngo/record)] [ngo:id] (auth+role->entity ctx (:auth opt) ::ngo/record)]
(when ngo:id (when ngo:id
;; TODO: take it from the db and filter it by visibility to the ngo (map db->graphql
;; When importing, we want define to which ngo the imported dataset is visible (q_unary '{:find [(pull ?e [*])]
(mock_geocoding ;; TODO :where [[?e :xt/spec ::offer/record]
(if (:import-file env) [?e ::ngo/id ngo:id]]
(unify (clojure.edn/read-string (slurp (:import-file env))) :in [ngo:id]}
beherbergung.model.offer-mapping.lifeline/mapping) ngo:id)))))
(clojure.edn/read-string (slurp "./data/sample-data/example.edn")) ;; till conflict between `specialist-server.type` and `with-gen` is fixed
#_(gen/sample (s/gen ::offer)))))))
(s/def ::get_offers (t/resolver #'get_offers)) (s/def ::get_offers (t/resolver #'get_offers))

View File

@ -28,7 +28,7 @@
[[:xtdb.api/put (assoc (merge entity doc) [[:xtdb.api/put (assoc (merge entity doc)
:xt/id eid :xt/id eid
:xt/spec ::offer-rw/record)]]))) :xt/spec ::offer-rw/record)]])))
(tx-fn-call :write_rw rowId doc)))] (tx-fn-call :write_rw (str "rw_" rowId) doc)))]
(sync) (sync)
(boolean (:xtdb.api/tx-id tx_result)))) (boolean (:xtdb.api/tx-id tx_result))))

View File

@ -3,6 +3,7 @@
(:require [ring.adapter.jetty] (:require [ring.adapter.jetty]
[ring.middleware.reload] [ring.middleware.reload]
[beherbergung.webserver.handler] [beherbergung.webserver.handler]
[beherbergung.db.import.offer.lifeline :refer [import!]]
[mount.core :as mount :refer [defstate]] [mount.core :as mount :refer [defstate]]
[beherbergung.config.state] [beherbergung.config.state]
[signal.handler :refer [with-handler]])) [signal.handler :refer [with-handler]]))
@ -17,6 +18,8 @@
(defn -main [& _args] (defn -main [& _args]
(mount/start) (mount/start)
(import!) ;; This is not the seeding, but import from external formats
(let [finaly (fn [] (mount/stop) ;; Export the database (let [finaly (fn [] (mount/stop) ;; Export the database
(System/exit 0))] (System/exit 0))]
(with-handler :term (finaly)) ;; kill (with-handler :term (finaly)) ;; kill

View File

@ -105,6 +105,7 @@ export type Get_Offers = {
contact_name_full?: Maybe<Scalars['String']>; contact_name_full?: Maybe<Scalars['String']>;
contact_phone?: Maybe<Scalars['String']>; contact_phone?: Maybe<Scalars['String']>;
id?: Maybe<Scalars['String']>; id?: Maybe<Scalars['String']>;
id_tmp?: Maybe<Scalars['String']>;
languages?: Maybe<Array<Scalars['String']>>; languages?: Maybe<Array<Scalars['String']>>;
note?: Maybe<Scalars['String']>; note?: Maybe<Scalars['String']>;
place_city?: Maybe<Scalars['String']>; place_city?: Maybe<Scalars['String']>;
@ -146,7 +147,7 @@ export type GetOffersQueryVariables = Exact<{
}>; }>;
export type GetOffersQuery = { __typename?: 'QueryType', get_offers?: Array<{ __typename?: 'get_offers', id?: string | null, time_from_str?: string | null, time_duration_str?: string | null, beds?: number | null, languages?: Array<string> | null, place_country?: string | null, place_city?: string | null, place_zip?: string | null, place_street?: string | null, place_street_number?: string | null, place_lon?: number | null, place_lat?: number | null, accessible?: boolean | null, animals_allowed?: boolean | null, animals_present?: boolean | null, contact_name_full?: string | null, contact_phone?: string | null, contact_email?: string | null, note?: string | null }> | null }; export type GetOffersQuery = { __typename?: 'QueryType', get_offers?: Array<{ __typename?: 'get_offers', id?: string | null, id_tmp?: string | null, time_from_str?: string | null, time_duration_str?: string | null, beds?: number | null, languages?: Array<string> | null, place_country?: string | null, place_city?: string | null, place_zip?: string | null, place_street?: string | null, place_street_number?: string | null, place_lon?: number | null, place_lat?: number | null, accessible?: boolean | null, animals_allowed?: boolean | null, animals_present?: boolean | null, contact_name_full?: string | null, contact_phone?: string | null, contact_email?: string | null, note?: string | null }> | null };
export type GetRwQueryVariables = Exact<{ export type GetRwQueryVariables = Exact<{
auth: Auth; auth: Auth;
@ -179,6 +180,7 @@ export const GetOffersDocument = `
query GetOffers($auth: Auth!) { query GetOffers($auth: Auth!) {
get_offers(auth: $auth) { get_offers(auth: $auth) {
id id
id_tmp
time_from_str time_from_str
time_duration_str time_duration_str
beds beds

View File

@ -9,6 +9,7 @@ export const get_offers = gql`
query GetOffers($auth: Auth!) { query GetOffers($auth: Auth!) {
get_offers(auth: $auth) { get_offers(auth: $auth) {
id id
id_tmp
time_from_str time_from_str
time_duration_str time_duration_str
beds beds

View File

@ -145,7 +145,8 @@ const HostOfferLookupTable = ({data_ro, data_rw, refetch_rw, onFilteredDataChang
const data = filterUndefOrNull( data_ro const data = filterUndefOrNull( data_ro
?.map( e_ro => ({ ?.map( e_ro => ({
...e_ro, ...e_ro,
...((data_rw?.find((e_rw) => e_rw.id === e_ro.id) || rw_default))}) ) || []) ...((data_rw?.find((e_rw) => e_ro.id_tmp === e_rw.id || `rw_${e_ro.id}` === e_rw.id
) || rw_default))}) ) || [])
// @ts-ignore // @ts-ignore
data && setDataSource(data) data && setDataSource(data)