diff --git a/Cargo.lock b/Cargo.lock index 4ad71265..7537743d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,6 +299,15 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.6.1" @@ -516,6 +525,9 @@ name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +dependencies = [ + "serde_core", +] [[package]] name = "blake3" @@ -539,6 +551,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -582,6 +603,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -667,6 +694,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "coarsetime" version = "0.1.38" @@ -790,6 +823,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.3" @@ -823,6 +871,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" @@ -851,6 +908,24 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -860,7 +935,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -950,12 +1025,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -977,6 +1063,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.5" @@ -990,7 +1082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der 0.7.10", - "digest", + "digest 0.10.7", "elliptic-curve", "rfc6979", "signature", @@ -1016,7 +1108,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -1026,6 +1118,9 @@ name = "either" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +dependencies = [ + "serde", +] [[package]] name = "elegant-departure" @@ -1047,11 +1142,11 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", - "hkdf", + "hkdf 0.12.4", "pem-rfc7468 0.7.0", "pkcs8", "rand_core 0.6.4", @@ -1111,6 +1206,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + [[package]] name = "fancy-regex" version = "0.16.2" @@ -1206,6 +1321,17 @@ dependencies = [ "serde", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1306,6 +1432,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -1560,6 +1697,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "headers" version = "0.4.2" @@ -1572,7 +1718,7 @@ dependencies = [ "http 1.5.0", "httpdate", "mime", - "sha1", + "sha1 0.10.7", ] [[package]] @@ -1678,7 +1824,16 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", ] [[package]] @@ -1687,7 +1842,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1773,6 +1937,15 @@ dependencies = [ "serde", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.12.0" @@ -2208,7 +2381,7 @@ dependencies = [ "base64 0.22.1", "ed25519-dalek", "getrandom 0.2.17", - "hmac", + "hmac 0.12.1", "js-sys", "p256", "p384", @@ -2217,7 +2390,7 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "signature", "simple_asn1", "zeroize", @@ -2250,6 +2423,17 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.31" @@ -2338,6 +2522,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "mediatype" version = "0.23.0" @@ -2992,6 +3186,7 @@ dependencies = [ "sentry", "serde", "serde_json", + "sqlx", "tempfile", "thiserror 2.0.21", "tokio", @@ -3154,7 +3349,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -3166,7 +3361,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -3179,6 +3374,12 @@ dependencies = [ "seize", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -3963,7 +4164,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -3988,7 +4189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", - "digest", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -4318,7 +4519,7 @@ dependencies = [ "sentry-options-validation", "serde", "serde_json", - "sha1", + "sha1 0.10.7", "tempfile", "thiserror 2.0.21", "tracing", @@ -4518,7 +4719,18 @@ checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4529,7 +4741,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -4563,7 +4786,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -4630,6 +4853,9 @@ name = "smallvec" version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -4646,6 +4872,9 @@ name = "spin" version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -4657,6 +4886,183 @@ dependencies = [ "der 0.7.10", ] +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink", + "indexmap", + "log", + "memchr", + "percent-encoding", + "rustls", + "rustls-native-certs", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "thiserror 2.0.21", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1 0.11.0", + "sha2 0.11.0", + "sqlx-core", + "thiserror 2.0.21", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf 0.13.0", + "hmac 0.13.0", + "itoa", + "log", + "md-5", + "memchr", + "rand 0.10.3", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "chrono", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror 2.0.21", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4688,6 +5094,17 @@ dependencies = [ "yansi", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -5266,6 +5683,12 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -5278,6 +5701,21 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -5569,6 +6007,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "widestring" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index cc0fd4f6..ccf51554 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,10 @@ async-trait = "0.1.92" axum = "0.8.9" axum-extra = { version = "0.12.6", default-features = false } base64 = { version = "0.23.1", default-features = false, features = ["std"] } -bigtable_rs = { git = "https://github.com/getsentry/bigtable_rs.git", rev = "072cb5871766b39c08c9a4d1f43c8d3895eeb96e", default-features = false, features = ["gcp-auth-ring", "tonic-tls-ring"] } +bigtable_rs = { git = "https://github.com/getsentry/bigtable_rs.git", rev = "072cb5871766b39c08c9a4d1f43c8d3895eeb96e", default-features = false, features = [ + "gcp-auth-ring", + "tonic-tls-ring", +] } blake3 = "1.8.5" bytes = "1.12.0" bytesize = "2.4.0" @@ -104,12 +107,16 @@ serde = { version = "1.0.228", features = ["derive"] } serde-vars = "0.4.0" serde_json = "1.0.150" serde_yaml = "0.9.34-deprecated" +sqlx = "0.9.0" sketches-ddsketch = "0.4.2" tempfile = "3.27.0" thiserror = "2.0.18" thread_local = "1.1.9" jemalloc_pprof = "0.9.0" -tikv-jemallocator = { version = "0.7.0", features = ["background_threads", "override_allocator_on_supported_platforms"] } +tikv-jemallocator = { version = "0.7.0", features = [ + "background_threads", + "override_allocator_on_supported_platforms", +] } tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] } tokio = "1.52.3" tokio-util = "0.7.18" diff --git a/migrations/sqlite/0001_garbage_collector.sql b/migrations/sqlite/0001_garbage_collector.sql new file mode 100644 index 00000000..bd2d3a98 --- /dev/null +++ b/migrations/sqlite/0001_garbage_collector.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS garbage_collector ( + object_id TEXT NOT NULL PRIMARY KEY, + expires_at INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_garbage_collector_expires_at + ON garbage_collector (expires_at); diff --git a/objectstore-service/Cargo.toml b/objectstore-service/Cargo.toml index a03d7e25..8c982420 100644 --- a/objectstore-service/Cargo.toml +++ b/objectstore-service/Cargo.toml @@ -19,18 +19,35 @@ chrono = { workspace = true } futures-util = { workspace = true } gcp_auth = { workspace = true } humantime-serde = { workspace = true } -objectstore-inventory-tracker = { workspace = true, features = ["kafka"], optional = true } +objectstore-inventory-tracker = { workspace = true, features = [ + "kafka", +], optional = true } objectstore-log = { workspace = true } objectstore-metrics = { workspace = true } objectstore-types = { workspace = true } papaya = { workspace = true } quick-xml = { workspace = true, features = ["serialize"] } regex = { workspace = true } -reqwest = { workspace = true, features = ["charset", "http2", "hickory-dns", "json", "multipart", "native-tls-no-alpn", "stream", "system-proxy"] } +reqwest = { workspace = true, features = [ + "charset", + "http2", + "hickory-dns", + "json", + "multipart", + "native-tls-no-alpn", + "stream", + "system-proxy", +] } ring = { workspace = true } sentry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sqlx = { workspace = true, features = [ + "runtime-tokio", + "tls-rustls-ring-native-roots", + "chrono", + "sqlite", +] } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/objectstore-service/build.rs b/objectstore-service/build.rs new file mode 100644 index 00000000..3a8149ef --- /dev/null +++ b/objectstore-service/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/objectstore-service/src/garbage_collector/mod.rs b/objectstore-service/src/garbage_collector/mod.rs new file mode 100644 index 00000000..5e28034b --- /dev/null +++ b/objectstore-service/src/garbage_collector/mod.rs @@ -0,0 +1,51 @@ +//! Persistence of object expiration metadata, read by the garbage collector to +//! find objects whose lifetime has elapsed. +//! +//! [`GcStorage`] is the backend-agnostic interface. Implementations record the +//! expiration of each object when it is written and hand back the expired ids +//! that the collector is allowed to delete. + +/// [`GcStorage`] implementation backed by SQLite. +pub mod sqlite; + +use std::fmt::Debug; + +use objectstore_types::time::Timestamp; + +use crate::change_stream::ChangeStreamError; +use crate::id::ObjectId; + +/// Stores object expiration metadata for the garbage collector. +/// +/// Implementations are called on the write path (via [`upsert`](Self::upsert) +/// and [`remove`](Self::remove)) and by the collector itself (via +/// [`list_expired`](Self::list_expired)), so they must be safe to share across +/// tasks. +#[async_trait::async_trait] +pub trait GcStorage: Send + Sync + Debug { + /// Records that `id` expires at `expires_at`, replacing any previous deadline. + /// + /// A later deadline replaces an earlier one; a shorter deadline is ignored, so concurrent + /// writers cannot pull an expiration forward. `None` clears the recorded deadline and marks + /// the object as never expiring. + async fn upsert( + &self, + id: &ObjectId, + expires_at: Option, + ) -> Result<(), ChangeStreamError>; + + /// Forgets `id` entirely. + /// + /// Best-effort: failures are not reported to the caller. + async fn remove(&self, id: &ObjectId); + + /// Returns up to `limit` ids whose expiration is at or before `now`, + /// ordered by earliest expiry first. + /// + /// Objects without an expiration are never returned. + async fn list_expired( + &self, + now: Timestamp, + limit: u64, + ) -> Result, ChangeStreamError>; +} diff --git a/objectstore-service/src/garbage_collector/sqlite.rs b/objectstore-service/src/garbage_collector/sqlite.rs new file mode 100644 index 00000000..b6e8a516 --- /dev/null +++ b/objectstore-service/src/garbage_collector/sqlite.rs @@ -0,0 +1,405 @@ +use std::str::FromStr; + +use async_trait::async_trait; +use objectstore_log::Level; +use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode}; + +use crate::change_stream::ChangeStreamError; +use crate::garbage_collector::GcStorage; +use crate::id::ObjectId; + +/// Configuration for [`SqliteGarbageCollector`]. +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +#[allow(dead_code)] +pub struct SqliteGarbageCollectorConfig { + /// Path to the SQLite database file, created if it does not exist. + pub path: String, +} + +impl Default for SqliteGarbageCollectorConfig { + fn default() -> Self { + Self { + path: "./objectstore.db".to_owned(), + } + } +} + +/// [`GcStorage`] backed by a SQLite database. +#[derive(Debug, Clone)] +pub struct SqliteGarbageCollector { + pool: SqlitePool, +} + +#[allow(dead_code)] +impl SqliteGarbageCollector { + /// Opens (creating if needed) the database at [`config.path`](SqliteGarbageCollectorConfig::path) + /// in WAL mode and runs the schema migrations. + pub async fn new(config: &SqliteGarbageCollectorConfig) -> Result { + let opts = SqliteConnectOptions::from_str(&config.path)? + .journal_mode(SqliteJournalMode::Wal) + .create_if_missing(true); + let pool = SqlitePool::connect_with(opts).await?; + + sqlx::migrate!("./../migrations/sqlite").run(&pool).await?; + + Ok(Self { pool }) + } +} + +#[async_trait] +impl GcStorage for SqliteGarbageCollector { + async fn upsert( + &self, + id: &ObjectId, + expires_at: Option, + ) -> Result<(), ChangeStreamError> { + let _ = sqlx::query( + r#"INSERT INTO garbage_collector (object_id, expires_at) + VALUES (?, ?) + ON CONFLICT (object_id) + DO UPDATE + SET expires_at = excluded.expires_at + WHERE garbage_collector.expires_at IS NOT NULL + AND garbage_collector.expires_at < excluded.expires_at"#, + ) + .bind(id.as_storage_path().to_string()) + .bind(expires_at.map(|t| i64::try_from(t.as_secs()).ok())) + .execute(&self.pool) + .await + .map_err(|e| ChangeStreamError::Failure(Box::new(e)))?; + + Ok(()) + } + + async fn remove(&self, id: &ObjectId) { + let _ = sqlx::query("DELETE FROM garbage_collector WHERE object_id = ?") + .bind(id.as_storage_path().to_string()) + .execute(&self.pool) + .await; + } + + async fn list_expired( + &self, + now: objectstore_types::time::Timestamp, + limit: u64, + ) -> Result, ChangeStreamError> { + let query_result: Result, sqlx::Error> = sqlx::query_scalar::<_, String>( + r#"SELECT + object_id + FROM + garbage_collector + WHERE + expires_at IS NOT NULL + AND expires_at <= ? + ORDER BY + expires_at + LIMIT ?"#, + ) + .bind(i64::try_from(now.as_secs()).ok()) + .bind(i64::try_from(limit).unwrap_or(i64::MAX)) + .fetch_all(&self.pool) + .await; + + match query_result { + Ok(rows) => { + let mut ids = Vec::new(); + for row in rows { + let id = row.clone(); + if let Some(object_id) = ObjectId::from_storage_path(&id) { + ids.push(object_id); + } else { + objectstore_log::event!( + Level::WARN, + object_id = id, + "cannot resolve object id from storage path" + ); + } + } + Ok(ids) + } + Err(error) => Err(ChangeStreamError::Failure(Box::new(error))), + } + } +} + +#[cfg(test)] +mod tests { + use objectstore_types::scope::{Scope, Scopes}; + + use super::*; + use crate::id::ObjectContext; + + /// The latest representable [`Timestamp`](objectstore_types::time::Timestamp), used as + /// "infinitely far in the future" for expiration checks. + const MAX_SECS: u64 = 253_402_300_799; + + fn ts(secs: u64) -> objectstore_types::time::Timestamp { + objectstore_types::time::Timestamp::from_unix_secs(secs).unwrap() + } + + fn id_for(key: &str) -> ObjectId { + ObjectId::new( + ObjectContext { + usecase: "testing".parse().unwrap(), + scopes: Scopes::from_iter([ + Scope::create("organization", "17").unwrap(), + Scope::create("project", "42").unwrap(), + ]), + }, + key.to_owned(), + ) + } + + fn config_in(dir: &tempfile::TempDir) -> SqliteGarbageCollectorConfig { + SqliteGarbageCollectorConfig { + path: dir + .path() + .join("objectstore.db") + .to_string_lossy() + .into_owned(), + } + } + + async fn collector_in(dir: &tempfile::TempDir) -> SqliteGarbageCollector { + SqliteGarbageCollector::new(&config_in(dir)).await.unwrap() + } + + /// Creates a collector backed by a database in a fresh temporary directory. The directory is + /// returned so that it outlives the collector. + async fn make_collector() -> (tempfile::TempDir, SqliteGarbageCollector) { + let dir = tempfile::tempdir().unwrap(); + let collector = collector_in(&dir).await; + (dir, collector) + } + + #[tokio::test] + async fn new_runs_migrations_on_empty_database() { + let (_dir, collector) = make_collector().await; + + assert!( + collector + .list_expired(ts(MAX_SECS), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn upsert_and_list_expired_roundtrip_the_full_id() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(100))).await.unwrap(); + + // Equality covers usecase, scopes, and key, so this also proves the storage path + // roundtrips through `from_storage_path`. + assert_eq!(collector.list_expired(ts(100), 10).await.unwrap(), vec![id]); + } + + #[tokio::test] + async fn list_expired_excludes_deadlines_after_now() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(200))).await.unwrap(); + + assert!( + collector + .list_expired(ts(199), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn list_expired_ignores_objects_without_expiration() { + let (_dir, collector) = make_collector().await; + + collector.upsert(&id_for("key"), None).await.unwrap(); + + assert!( + collector + .list_expired(ts(MAX_SECS), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn upsert_extends_an_earlier_deadline() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(100))).await.unwrap(); + collector.upsert(&id, Some(ts(300))).await.unwrap(); + + assert!( + collector + .list_expired(ts(100), 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!(collector.list_expired(ts(300), 10).await.unwrap(), vec![id]); + } + + #[tokio::test] + async fn upsert_does_not_shorten_an_existing_deadline() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(300))).await.unwrap(); + collector.upsert(&id, Some(ts(100))).await.unwrap(); + + assert_eq!(collector.list_expired(ts(300), 10).await.unwrap(), vec![id]); + assert!( + collector + .list_expired(ts(200), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn upsert_infinite_duration_does_not_update_expiry() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, None).await.unwrap(); + collector.upsert(&id, Some(ts(100))).await.unwrap(); + + assert!( + collector + .list_expired(ts(100), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn upserting_the_same_id_twice_does_not_duplicate_it() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(100))).await.unwrap(); + collector.upsert(&id, Some(ts(100))).await.unwrap(); + + assert_eq!(collector.list_expired(ts(100), 10).await.unwrap(), vec![id]); + } + + #[tokio::test] + async fn upserting_the_same_id_twice_does_not_duplicate_it_infinite_expiry() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, None).await.unwrap(); + collector.upsert(&id, None).await.unwrap(); + + assert!( + collector + .list_expired(ts(100), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn remove_forgets_the_object() { + let (_dir, collector) = make_collector().await; + let id = id_for("key"); + + collector.upsert(&id, Some(ts(100))).await.unwrap(); + collector.remove(&id).await; + + assert!( + collector + .list_expired(ts(MAX_SECS), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn remove_of_unknown_object_is_a_noop() { + let (_dir, collector) = make_collector().await; + + collector.remove(&id_for("missing")).await; + + assert!( + collector + .list_expired(ts(MAX_SECS), 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn list_expired_orders_by_earliest_deadline_first() { + let (_dir, collector) = make_collector().await; + let (late, early, middle) = (id_for("late"), id_for("early"), id_for("middle")); + + collector.upsert(&late, Some(ts(300))).await.unwrap(); + collector.upsert(&early, Some(ts(100))).await.unwrap(); + collector.upsert(&middle, Some(ts(200))).await.unwrap(); + + assert_eq!( + collector.list_expired(ts(300), 10).await.unwrap(), + vec![early, middle, late] + ); + } + + #[tokio::test] + async fn list_expired_respects_the_limit() { + let (_dir, collector) = make_collector().await; + let (first, second, third) = (id_for("first"), id_for("second"), id_for("third")); + + collector.upsert(&first, Some(ts(100))).await.unwrap(); + collector.upsert(&second, Some(ts(200))).await.unwrap(); + collector.upsert(&third, Some(ts(300))).await.unwrap(); + + assert_eq!( + collector.list_expired(ts(300), 2).await.unwrap(), + vec![first, second] + ); + assert!(collector.list_expired(ts(300), 0).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn list_expired_with_limit_above_row_count_returns_everything() { + let (_dir, collector) = make_collector().await; + let (first, second) = (id_for("first"), id_for("second")); + + collector.upsert(&first, Some(ts(100))).await.unwrap(); + collector.upsert(&second, Some(ts(200))).await.unwrap(); + + assert_eq!( + collector.list_expired(ts(300), 1_000).await.unwrap(), + vec![first, second] + ); + } + + #[tokio::test] + async fn reopening_an_existing_database_keeps_its_rows() { + let dir = tempfile::tempdir().unwrap(); + let id = id_for("key"); + + { + let collector = collector_in(&dir).await; + collector.upsert(&id, Some(ts(100))).await.unwrap(); + } + + // Re-running the migrations must be a no-op, and the row must still be there. + let collector = collector_in(&dir).await; + assert_eq!(collector.list_expired(ts(100), 10).await.unwrap(), vec![id]); + } +} diff --git a/objectstore-service/src/lib.rs b/objectstore-service/src/lib.rs index 06e5e8cf..40258921 100644 --- a/objectstore-service/src/lib.rs +++ b/objectstore-service/src/lib.rs @@ -8,6 +8,7 @@ pub mod change_stream; pub mod concurrency; pub mod encryption; pub mod error; +pub mod garbage_collector; mod gcp_auth; pub mod id; pub mod resumable;