Introducing Redis for caching

.clj-kondo/config.edn

@@ -11,6 +11,8 @@
                                            prompt-fmt])]}
   :unused-namespace {:exclude ["game.core.*"]}
   :refer-all {:level :off}
+  :unused-referred-var {:level :off}
+  :unresolved-var {:exclude [taoensso.carmine]}
   }
  :lint-as {game.test-framework/before-each clojure.core/let
            reagent.core/with-let clojure.core/let
@@ -24,6 +26,8 @@
            game.core/expose-vars potemkin/import-vars
            monger.operators/defoperator clojure.core/def
            monger.query/with-collection clojure.core/->
+           taoensso.msgpack.common/extend-packable clj-kondo.lint-as/def-catch-all
+           org.httpkit.client/defreq clj-kondo.lint-as/def-catch-all
            }
  :hooks {:analyze-call {jinteki.i18n/simple-msg hooks.netrunner/effect
                         jinteki.i18n/msg-with-cost hooks.netrunner/effect}}

project.clj

@@ -66,6 +66,7 @@
                  [time-literals "0.1.5"]
                  [metosin/reitit "0.7.2"]
                  [org.flatland/ordered "1.15.12"]
+                 [com.taoensso/carmine "3.5.0"]
                  [io.github.noahtheduke/fluent-clj "0.3.0"]]
 
   :test-selectors {:default (fn [m] (not (:kaocha/pending m)))}

resources/dev.edn

@@ -4,7 +4,10 @@
                       :name "netrunner"
                       ; Monger requires a db name in the URI string, be sure to specify
                       :connection-string #or [#env MONGO_CONNECTION_URI nil]}
- :logging/timbre {:appenders
+ :redis/connection {:pool {}
+                    :spec {}}
+ :logging/timbre {:min-level :info
+                  :appenders
                   ;; set appender args:
                   ;; `:enabled?` sets whether an appender should run
                   ;; `:log-path` defaults to `~/logs/jinteki-*.log`
@@ -23,9 +26,16 @@
                      :same-site :lax
                      ; 60 days
                      :max-age 5184000}}
- :web/lobby {:interval 1000
+ :web/ws {:packer :edn
+          :buffer-size 500
+          :buffer-clear-timer-ms 40}
+ :web/lobby {:time-inactive 600
+             ;; :interval and :clear-out-frequency are passed to enc/ms
+             :interval {:secs 1}
+             :clear-out-frequency {:mins 5}
              :mongo #ig/ref :mongodb/connection
-             :time-inactive 600}
+             :ws #ig/ref :web/ws
+             :redis #ig/ref :redis/connection}
  :web/chat {:max-length 144
             ;; sliding window size in seconds
             :rate-window 60
@@ -38,9 +48,13 @@
              :from "support@jinteki.net"
              :reset-subject "Jinteki Password Reset"
              :confirm-reset-subject "Your password has been changed"}
- :web/ws {:packer :edn}
+ :web/telemetry {:web/ws #ig/ref :web/ws
+                 :web/redis #ig/ref :redis/connection
+                 ;; :frequency is passed to enc/ms
+                 :frequency {:secs 30}}
  :web/app {:server-mode #ig/ref :server/mode
            :mongodb/connection #ig/ref :mongodb/connection
+           :redis/connection #ig/ref :redis/connection
            :web/auth #ig/ref :web/auth
            :web/chat #ig/ref :web/chat
            :web/email #ig/ref :web/email
@@ -49,9 +63,11 @@
                   :mongo #ig/ref :mongodb/connection}
  :web/i18n nil
  :frontend/version {:initial "1"
+                    :redis #ig/ref :redis/connection
                     :mongo #ig/ref :mongodb/connection}
- :sente/router {:ws #ig/ref :web/ws}
  :game/quotes nil
  :jinteki/cards-version {:initial "1"
+                         :redis #ig/ref :redis/connection
                          :mongo #ig/ref :mongodb/connection}
- :jinteki/cards {:mongo #ig/ref :mongodb/connection}}
+ :jinteki/cards {:redis #ig/ref :redis/connection
+                 :mongo #ig/ref :mongodb/connection}}

src/clj/web/admin.clj

@@ -1,15 +1,16 @@
 (ns web.admin
   (:require
    [cljc.java-time.instant :as inst]
+   [clojure.string :as str]
    [jinteki.utils :refer [superuser?]]
    [monger.collection :as mc]
    [monger.operators :refer :all]
    [monger.result :refer [acknowledged? updated-existing?]]
+   [taoensso.carmine :as car :refer [wcar]]
    [web.app-state :as app-state]
    [web.mongodb :refer [->object-id]]
    [web.user :refer [active-user?]]
    [web.utils :refer [response]]
-   [web.versions :refer [frontend-version banned-msg]]
    [web.ws :as ws]))
 
 (defn- last-ip-address
@@ -19,20 +20,20 @@
 
 (defmethod ws/-msg-handler :admin/announce
   admin--announce
-  [{{user :user} :ring-req
+  [{{:system/keys [ws] user :user} :ring-req
     {message :message} :?data
     reply-fn :?reply-fn}]
   (cond
     (not (superuser? user)) (reply-fn 403)
     (empty? message) (reply-fn 400)
     :else
     (do
-      (doseq [uid (ws/connected-uids)]
-        (ws/chsk-send! uid [:lobby/toast {:message message
-                                          :type "warning"}]))
+      (doseq [uid (ws/connected-uids ws)]
+        (ws/chsk-send! ws uid [:lobby/toast {:message message
+                                             :type "warning"}]))
       (reply-fn 200))))
 
-(defn news-create-handler [{db :system/db
+(defn news-create-handler [{:system/keys [db]
                             {item :item} :body}]
   (if-not (empty? item)
     (do
@@ -43,7 +44,7 @@
     (response 400 {:message "Missing news item"})))
 
 (defn news-delete-handler
-  [{db :system/db
+  [{:system/keys [db]
     {id :id} :path-params}]
   (try
     (if id
@@ -54,33 +55,34 @@
     (catch Exception _
       (response 409 {:message "Unknown news item id"}))))
 
-(defn version-handler [{db :system/db}]
+(defn version-handler [{:system/keys [db]}]
   (let [config (mc/find-one-as-map db "config" nil)
         version (:version config "0.0")]
     (response 200 {:message "ok" :version version})))
 
-(defn version-update-handler [{db :system/db
+(defn version-update-handler [{:system/keys [db redis]
                                {version :version} :body}]
   (if-not (empty? version)
     (do
-      (reset! frontend-version version)
+      (wcar redis (car/set :frontend/version version))
       (mc/update db "config" {} {$set {:version version}})
       (response 200 {:message "ok" :version version}))
     (response 400 {:message "Missing version item"})))
 
-(defn banned-message-handler [{db :system/db}]
-  (let [config (mc/find-one-as-map db "config" nil)
-        banned (:banned-msg config "Account is locked")]
-    (response 200 {:message "ok" :banned banned})))
+(defn banned-message-handler [{:system/keys [db redis]}]
+  (let [msg (or (wcar redis (car/get :config/banned-msg))
+              (:banned-msg (mc/find-one-as-map db "config" nil))
+              "Account is locked")]
+    (response 200 {:message "ok" :banned msg})))
 
-(defn banned-message-update-handler [{db :system/db
+(defn banned-message-update-handler [{:system/keys [db redis]
                                       {banned :banned} :body}]
-  (if-not (empty? banned)
+  (if (str/blank? banned)
+    (response 400 {:message "Missing banned message item"})
     (do
-      (reset! banned-msg banned)
+      (wcar redis (car/set :config/banned-msg banned))
       (mc/update db "config" {} {$set {:banned-msg banned}})
-      (response 200 {:message "ok" :banned banned}))
-    (response 400 {:message "Missing banned message item"})))
+      (response 200 {:message "ok" :banned banned}))))
 
 (def user-collection "users")
 
@@ -92,7 +94,7 @@
 
 (defmethod ws/-msg-handler :admin/edit-user
   admin--edit-user
-  [{{db :system/db user :user} :ring-req
+  [{{:system/keys [db ws] user :user} :ring-req
     {:keys [action user-type username] :as data} :?data
     uid :uid}]
   (if (and (active-user? user)
@@ -114,18 +116,18 @@
                      (update :_id str)))]
       (if user
         (do
-          (ws/broadcast-to! [uid] :admin/user-edit {:success (assoc data :user user)})
+          (ws/broadcast-to! ws [uid] :admin/user-edit {:success (assoc data :user user)})
           (when (= user-type :banned)
             (when-let [connected-user ((:users @app-state/app-state) username)]
-              (ws/broadcast-to! [(:uid connected-user)] :system/force-disconnect {}))))
-        (ws/broadcast-to! [uid] :admin/user-edit {:error "Not found"})))
-    (ws/broadcast-to! [uid] :admin/user-edit {:error "Not allowed"})))
+              (ws/broadcast-to! ws [(:uid connected-user)] :system/force-disconnect {}))))
+        (ws/broadcast-to! ws [uid] :admin/user-edit {:error "Not found"})))
+    (ws/broadcast-to! ws [uid] :admin/user-edit {:error "Not allowed"})))
 
 (def ip-ban-collection "ip-bans")
 
 (defmethod ws/-msg-handler :admin/look-up-ip
   admin--look-up-ip
-  [{{db :system/db user :user} :ring-req
+  [{{:system/keys [db ws] user :user} :ring-req
     {:keys [username] :as data} :?data
     uid :uid}]
   (if (and (active-user? user)
@@ -135,28 +137,28 @@
                                           :lastIpAddress 1
                                           :last-ip-address 1
                                           :_id 0})]
-      (ws/broadcast-to! [uid] :admin/look-up-ip {:success (-> res
-                                                              (dissoc :lastIpAddress)
-                                                              (assoc :last-ip-address (last-ip-address res)))})
-      (ws/broadcast-to! [uid] :admin/look-up-ip {:error "Not found"}))
-    (ws/broadcast-to! [uid] :admin/look-up-ip {:error "Not allowed"})))
+      (ws/broadcast-to! ws [uid] :admin/look-up-ip {:success (-> res
+                                                               (dissoc :lastIpAddress)
+                                                               (assoc :last-ip-address (last-ip-address res)))})
+      (ws/broadcast-to! ws [uid] :admin/look-up-ip {:error "Not found"}))
+    (ws/broadcast-to! ws [uid] :admin/look-up-ip {:error "Not allowed"})))
 
 (defmethod ws/-msg-handler :admin/fetch-ip-bans
   admin--fetch-ip-bans
-  [{{db :system/db user :user} :ring-req
+  [{{:system/keys [db ws] user :user} :ring-req
     uid :uid}]
   (if (and (active-user? user)
            (or (:ismoderator user) (:isadmin user)))
     (let [ip-bans (mc/find-maps db ip-ban-collection {}
                                 {:username 1
                                  :ip-address 1
                                  :_id 0})]
-      (ws/broadcast-to! [uid] :admin/fetch-ip-bans {:success ip-bans}))
-    (ws/broadcast-to! [uid] :admin/fetch-ip-bans {:error "Not allowed"})))
+      (ws/broadcast-to! ws [uid] :admin/fetch-ip-bans {:success ip-bans}))
+    (ws/broadcast-to! ws [uid] :admin/fetch-ip-bans {:error "Not allowed"})))
 
 (defmethod ws/-msg-handler :admin/ip-ban-user
   admin--ip-ban-user
-  [{{db :system/db user :user} :ring-req
+  [{{:system/keys [db ws] user :user} :ring-req
     {:keys [username] :as data} :?data
     uid :uid}]
   (if (and (active-user? user)
@@ -171,29 +173,29 @@
           (prn "res: " res)
           (prn "ip: " ip)
           (mc/insert db ip-ban-collection {:username username :ip-address ip})
-          (ws/broadcast-to! [uid] :admin/ip-ban-user {:success {:username username
+          (ws/broadcast-to! ws [uid] :admin/ip-ban-user {:success {:username username
                                                                 :ip-address ip}}))
-        (ws/broadcast-to! [uid] :admin/ip-ban-user {:error "Legacy user? No IP Address on record"}))
-      (ws/broadcast-to! [uid] :admin/ip-ban-user {:error "Not found"}))
-    (ws/broadcast-to! [uid] :admin/ip-ban-user {:error "Not allowed"})))
+        (ws/broadcast-to! ws [uid] :admin/ip-ban-user {:error "Legacy user? No IP Address on record"}))
+      (ws/broadcast-to! ws [uid] :admin/ip-ban-user {:error "Not found"}))
+    (ws/broadcast-to! ws [uid] :admin/ip-ban-user {:error "Not allowed"})))
 
 (defmethod ws/-msg-handler :admin/ip-unban-user
   admin--ip-unban-user
-    [{{db :system/db user :user} :ring-req
+    [{{:system/keys [db ws] user :user} :ring-req
     {:keys [username] :as data} :?data
       uid :uid}]
   (if (and (active-user? user)
            (or (:ismoderator user) (:isadmin user)))
     (let [result (mc/remove db ip-ban-collection {:username username})
           n (.getN result)]
       (if (pos? n)
-        (ws/broadcast-to! [uid] :admin/ip-unban-user {:success username})
-        (ws/broadcast-to! [uid] :admin/ip-unban-user {:error "Not found"})))
-    (ws/broadcast-to! [uid] :admin/ip-unban-user {:error "Not allowed"})))
+        (ws/broadcast-to! ws [uid] :admin/ip-unban-user {:success username})
+        (ws/broadcast-to! ws [uid] :admin/ip-unban-user {:error "Not found"})))
+    (ws/broadcast-to! ws [uid] :admin/ip-unban-user {:error "Not allowed"})))
 
 (defmethod ws/-msg-handler :admin/fetch-users
   admin--fetch-users
-  [{{db :system/db user :user} :ring-req
+  [{{:system/keys [db ws] user :user} :ring-req
     uid :uid}]
   (if (and (active-user? user)
            (or (:ismoderator user) (:isadmin user)))
@@ -203,8 +205,8 @@
                                                             {:banned true}]}
                                    [:_id :username :ismoderator :special :tournament-organizer :banned])
                      (map #(update % :_id str)))]
-      (ws/broadcast-to! [uid] :admin/fetch-users {:success users}))
-    (ws/broadcast-to! [uid] :admin/fetch-users {:error "Not allowed"})))
+      (ws/broadcast-to! ws [uid] :admin/fetch-users {:success users}))
+    (ws/broadcast-to! ws [uid] :admin/fetch-users {:error "Not allowed"})))
 
 (defmethod ws/-msg-handler :admin/block-game-creation
   admin--block-game-creation

src/clj/web/analytics.clj

@@ -1,13 +1,9 @@
 (ns web.analytics
   (:require
-   [clojure.set :as set]
-   [monger.collection :as mc]
-   [clojure.java.io :as io]
-   [taoensso.encore :as enc]
    [clojure.core.async :refer [<! go timeout]]
-   [cljc.java-time.temporal.chrono-unit :as chrono]
-   [cljc.java-time.duration :as duration]
-   [cljc.java-time.instant :as inst]))
+   [clojure.java.io :as io]
+   [clojure.set :as set]
+   [taoensso.encore :as enc]))
 
 (defonce daily-analytics
   (atom {}))

src/clj/web/api.clj

@@ -172,16 +172,22 @@
   (middleware/map->Middleware
     {:name ::wrap-system
      :description "Adds the relevant integrant system pieces to requests"
-     :wrap (fn [handler system]
-             (fn [request]
-               (-> request
-                   (assoc :system/db (-> system :mongodb/connection :db))
-                   (assoc :system/server-mode (:server-mode system))
-                   (assoc :system/auth (:web/auth system))
-                   (assoc :system/chat (:web/chat system))
-                   (assoc :system/email (:web/email system))
-                   (assoc :system/ws-config (:web/ws system))
-                   (handler))))}))
+     :wrap (fn [handler {:web/keys [auth chat email ws]
+                         :keys [server-mode]
+                         redis :redis/connection
+                         {db :db} :mongodb/connection}]
+             (let [system {:system/auth auth
+                           :system/chat chat
+                           :system/db db
+                           :system/email email
+                           :system/redis redis
+                           :system/server-mode server-mode
+                           :system/ws ws}]
+               (fn [request]
+                 (-> request
+                     (assoc :system/whole system)
+                     (merge system)
+                     (handler)))))}))
 
 (defn make-middleware [system]
   {:middleware [wrap-return-favicon