mirror of
https://github.com/stoatchat/stoatchat.git
synced 2026-10-31 13:37:42 +00:00
feat: opentelemetry crate and files/autumn integration (#499)
Signed-off-by: izzy <me@insrt.uk> GitHub-Pull-Request: https://github.com/stoatchat/stoatchat/pull/499 Change-Id: If576743ccf6c7bea288a4ebb3fc485708cd21b51 Reviewed-on: https://gerrit.stoatinternal.com/c/stoatchat/+/201 Tested-by: Stoat Code Review <code-review@stoat.chat> Reviewed-by: insert <insert@stoat.chat>
This commit is contained in:
izzy
3
.gitignore
vendored
3
.gitignore
vendored
@@ -13,4 +13,5 @@ venv/
|
||||
|
||||
livekit.yml
|
||||
.idea
|
||||
.vscode/launch.json
|
||||
.vscode/launch.json
|
||||
.mise/config.local.toml
|
||||
|
||||
@@ -5,7 +5,7 @@ pnpm = "10.28.1"
|
||||
gh = "2.25.0"
|
||||
|
||||
rust = "1.92.0"
|
||||
"cargo:cargo-nextest" = "0.9.122"
|
||||
"github:nextest-rs/nextest" = { version = "0.9.122", version_prefix = "cargo-nextest-" }
|
||||
"cargo:cargo-edit" = "0.13.13"
|
||||
|
||||
"github:git-town/git-town" = "22.7.1"
|
||||
@@ -21,12 +21,11 @@ wait_for = ["docker:start", "build"]
|
||||
run = [{ task = "service:*" }]
|
||||
|
||||
[env]
|
||||
RUST_LOG = "debug,opentelemetry=info,hyper=info,reqwest=info,mobc=info"
|
||||
|
||||
BUILDER = "cargo"
|
||||
SYSTEM_DEPS_DAV1D_BUILD_INTERNAL = "auto" # enable building dav1d iff the system doesn't have a new enough version
|
||||
DOCKER_NETWORK_NAME = "stoatchat_default"
|
||||
DATABASE_PORT = "27017"
|
||||
RABBIT_PORT = "5672"
|
||||
REDIS_PORT = "6379"
|
||||
|
||||
_.file = { path = ".env", tools = true }
|
||||
|
||||
[tasks."release:bump"]
|
||||
|
||||
@@ -2,12 +2,5 @@
|
||||
#MISE description="Start Docker containers"
|
||||
set -e
|
||||
|
||||
docker compose up -d
|
||||
|
||||
docker run \
|
||||
--network=${DOCKER_NETWORK_NAME} \
|
||||
--name wait \
|
||||
--rm dokku/wait -c \
|
||||
rabbit:${RABBIT_PORT},\
|
||||
database:${DATABASE_PORT},\
|
||||
redis:${REDIS_PORT}
|
||||
docker compose up -d "$@"
|
||||
docker compose run --rm wait
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
#MISE description="Stop Docker containers"
|
||||
set -e
|
||||
|
||||
docker compose down
|
||||
docker compose down "$@"
|
||||
|
||||
182
Cargo.lock
generated
182
Cargo.lock
generated
@@ -2719,7 +2719,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3885,7 +3885,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"system-configuration 0.7.0",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
@@ -4214,7 +4214,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi 0.5.2",
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4851,7 +4851,7 @@ dependencies = [
|
||||
"os_info",
|
||||
"parking_lot",
|
||||
"pbjson-types",
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
"rand 0.9.4",
|
||||
"reqwest 0.12.28",
|
||||
"rustls-native-certs 0.8.4",
|
||||
@@ -4873,7 +4873,7 @@ checksum = "dd59f3759f1a14e60b6bc6d4d415414e1aed686c8e4d2c7862542d95301cdc70"
|
||||
dependencies = [
|
||||
"pbjson",
|
||||
"pbjson-types",
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -5437,7 +5437,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5806,6 +5806,89 @@ dependencies = [
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"js-sys",
|
||||
"pin-project-lite",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-appender-tracing"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2b304e8d8f11326ba366a99ca5c7aaf7cd397b67268a332437175aeb42a40db"
|
||||
dependencies = [
|
||||
"opentelemetry",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-http"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"opentelemetry",
|
||||
"reqwest 0.13.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-otlp"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344"
|
||||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"httpdate",
|
||||
"opentelemetry",
|
||||
"opentelemetry-http",
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry_sdk",
|
||||
"prost 0.14.4",
|
||||
"reqwest 0.13.4",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-proto"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
|
||||
dependencies = [
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"prost 0.14.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry_sdk"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-executor",
|
||||
"futures-util",
|
||||
"opentelemetry",
|
||||
"percent-encoding",
|
||||
"portable-atomic",
|
||||
"rand 0.9.4",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-multimap"
|
||||
version = "0.4.3"
|
||||
@@ -5950,7 +6033,7 @@ checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"itertools 0.11.0",
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
"prost-types",
|
||||
]
|
||||
|
||||
@@ -5964,7 +6047,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"pbjson",
|
||||
"pbjson-build",
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
"prost-build",
|
||||
"serde",
|
||||
]
|
||||
@@ -6641,7 +6724,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
"prost-derive 0.12.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive 0.14.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6658,7 +6751,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"petgraph",
|
||||
"prettyplease",
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
"prost-types",
|
||||
"regex",
|
||||
"syn 2.0.118",
|
||||
@@ -6678,13 +6771,26 @@ dependencies = [
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote 1.0.46",
|
||||
"syn 2.0.118",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.12.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost 0.12.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6794,7 +6900,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls 0.23.41",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -6832,7 +6938,7 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.4",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
@@ -7232,7 +7338,9 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2 0.4.15",
|
||||
"http 1.4.2",
|
||||
"http-body 1.0.1",
|
||||
@@ -7314,6 +7422,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"simdutf8",
|
||||
"stoat-otel",
|
||||
"strum_macros",
|
||||
"tempfile",
|
||||
"thumbhash",
|
||||
@@ -7457,6 +7566,7 @@ dependencies = [
|
||||
"sha1 0.10.6",
|
||||
"tokio",
|
||||
"totp-lite",
|
||||
"tracing",
|
||||
"ulid",
|
||||
"unicode-segmentation",
|
||||
"url-escape",
|
||||
@@ -7705,6 +7815,7 @@ dependencies = [
|
||||
"sentry",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum_macros",
|
||||
"utoipa",
|
||||
]
|
||||
|
||||
@@ -8130,7 +8241,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8260,7 +8371,7 @@ dependencies = [
|
||||
"security-framework 3.7.0",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9038,7 +9149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9106,6 +9217,23 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "stoat-otel"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"opentelemetry",
|
||||
"opentelemetry-appender-tracing",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry_sdk",
|
||||
"revolt-result",
|
||||
"serde",
|
||||
"tracing",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strict-num"
|
||||
version = "0.1.1"
|
||||
@@ -9388,7 +9516,7 @@ dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9819,12 +9947,14 @@ checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.4.2",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9911,6 +10041,22 @@ dependencies = [
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-opentelemetry"
|
||||
version = "0.34.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"opentelemetry",
|
||||
"smallvec",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
"tracing-subscriber",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-subscriber"
|
||||
version = "0.3.23"
|
||||
@@ -10619,7 +10765,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
13
Cargo.toml
13
Cargo.toml
@@ -7,6 +7,7 @@ members = [
|
||||
"crates/core/*",
|
||||
"crates/services/*",
|
||||
"crates/daemons/*",
|
||||
"crates/common/otel",
|
||||
]
|
||||
|
||||
[patch.crates-io]
|
||||
@@ -152,10 +153,11 @@ tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.22", features = [
|
||||
"env-filter",
|
||||
] } # consider https://crates.io/crates/better-tracing
|
||||
opentelemetry = { version = "0.31.0", features = ["logs"] }
|
||||
opentelemetry_sdk = { version = "0.31.0", features = ["logs"] }
|
||||
opentelemetry-otlp = { version = "0.31.0", features = ["logs"] }
|
||||
opentelemetry-appender-tracing = "0.31.1"
|
||||
opentelemetry = { version = "0.33.0", features = ["logs"] }
|
||||
opentelemetry_sdk = { version = "0.33.0", features = ["logs"] }
|
||||
opentelemetry-otlp = { version = "0.33.0", features = ["logs"] }
|
||||
opentelemetry-appender-tracing = "0.33.0"
|
||||
tracing-opentelemetry = "0.34.0"
|
||||
|
||||
# RabbitMQ
|
||||
lapin = "4.7.1"
|
||||
@@ -204,3 +206,6 @@ revolt-permissions = { version = "0.16.0-rc.0", path = "crates/core/permissions"
|
||||
revolt-presence = { version = "0.16.0-rc.0", path = "crates/core/presence" }
|
||||
revolt-ratelimits = { version = "0.16.0-rc.0", path = "crates/core/ratelimits" }
|
||||
revolt-result = { version = "0.16.0-rc.0", path = "crates/core/result" }
|
||||
|
||||
# Local packages (unpublished)
|
||||
stoat-otel = { path = "crates/common/otel" }
|
||||
|
||||
@@ -33,6 +33,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
|
||||
COPY crates/core/result/Cargo.toml ./crates/core/result/
|
||||
COPY crates/core/coalesced/Cargo.toml ./crates/core/coalesced/
|
||||
COPY crates/core/ratelimits/Cargo.toml ./crates/core/ratelimits/
|
||||
COPY crates/common/otel/Cargo.toml ./crates/common/otel/
|
||||
COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
|
||||
COPY crates/services/january/Cargo.toml ./crates/services/january/
|
||||
COPY crates/services/gifbox/Cargo.toml ./crates/services/gifbox/
|
||||
|
||||
@@ -29,6 +29,7 @@ COPY crates/core/presence/Cargo.toml ./crates/core/presence/
|
||||
COPY crates/core/result/Cargo.toml ./crates/core/result/
|
||||
COPY crates/core/coalesced/Cargo.toml ./crates/core/coalesced/
|
||||
COPY crates/core/ratelimits/Cargo.toml ./crates/core/ratelimits/
|
||||
COPY crates/common/otel/Cargo.toml ./crates/common/otel/
|
||||
COPY crates/services/autumn/Cargo.toml ./crates/services/autumn/
|
||||
COPY crates/services/january/Cargo.toml ./crates/services/january/
|
||||
COPY crates/services/gifbox/Cargo.toml ./crates/services/gifbox/
|
||||
|
||||
29
README.md
29
README.md
@@ -60,6 +60,7 @@ As a heads-up, the development environment uses the following ports:
|
||||
| Maildev | 14025<br>14080 |
|
||||
| Revolt Web App | 14701 |
|
||||
| RabbitMQ | 5672<br>15672 |
|
||||
| Grafana LGTM (OTLP) | 13000<br>4318 |
|
||||
| `crates/delta` | 14702 |
|
||||
| `crates/bonfire` | 14703 |
|
||||
| `crates/services/autumn` | 14704 |
|
||||
@@ -117,6 +118,11 @@ If you'd like to change anything, create a `Revolt.overrides.toml` file and spec
|
||||
> ports: !override
|
||||
> - "14072:5672"
|
||||
> - "14672:15672"
|
||||
>
|
||||
> otel-lgtm:
|
||||
> ports: !override
|
||||
> - "13000:3000"
|
||||
> - "14318:4318"
|
||||
> ```
|
||||
>
|
||||
> With the corresponding Revolt configuration:
|
||||
@@ -125,20 +131,18 @@ If you'd like to change anything, create a `Revolt.overrides.toml` file and spec
|
||||
> # Revolt.overrides.toml
|
||||
> # and Revolt.test-overrides.toml
|
||||
> [database]
|
||||
> mongodb = "mongodb://127.0.0.1:14017"
|
||||
> mongodb = "mongodb://127.0.0.1:14017/?directConnection=true"
|
||||
> redis = "redis://127.0.0.1:14079/"
|
||||
>
|
||||
> [rabbit]
|
||||
> port = 14072
|
||||
> ```
|
||||
>
|
||||
> And mise configuration
|
||||
> And the OTLP endpoint:
|
||||
>
|
||||
> ```bash
|
||||
> #.env
|
||||
> DATABASE_PORT = "14017"
|
||||
> RABBIT_PORT = "14072"
|
||||
> REDIS_PORT = "14079"
|
||||
> OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14318"
|
||||
> ```
|
||||
|
||||
Then continue:
|
||||
@@ -166,6 +170,21 @@ When signing up, go to http://localhost:14080 to find confirmation/password rese
|
||||
To stop all services, hit (CTRL + c) in the terminal you ran `mise start` and run `mise docker:stop`
|
||||
|
||||
|
||||
## Environment Variables
|
||||
|
||||
This list is non-exhaustive right now.
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
| ------------------------------------- | ----------------------- | --------------------------------------------------------------------- |
|
||||
| `RUST_LOG` | `info` | Log filter, e.g. `info,revolt_autumn=debug` |
|
||||
| `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | | OTLP/HTTP endpoint for logs |
|
||||
| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | | OTLP/HTTP endpoint for traces |
|
||||
| `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | | OTLP/HTTP endpoint for metrics |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318` | Base URL for a single collector; appends `/v1/{logs,traces,metrics}` |
|
||||
| `OTEL_RESOURCE_ATTRIBUTES` | | Extra resource attributes, e.g. `deployment.environment.name=staging` |
|
||||
| `OTEL_TRACES_SAMPLER` | `parentbased_always_on` | Trace sampler, e.g. `parentbased_traceidratio` |
|
||||
| `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Sampling ratio, e.g. `0.1` |
|
||||
|
||||
## Deployment Guide
|
||||
|
||||
### Cutting new crate releases
|
||||
|
||||
16
compose.yml
16
compose.yml
@@ -1,3 +1,5 @@
|
||||
name: stoat-backend
|
||||
|
||||
services:
|
||||
# Redis
|
||||
redis:
|
||||
@@ -80,9 +82,23 @@ services:
|
||||
MAILDEV_INCOMING_USER: smtp
|
||||
MAILDEV_INCOMING_PASS: smtp
|
||||
|
||||
# LiveKit
|
||||
livekit:
|
||||
image: ghcr.io/stoatchat/livekit-server:v1.9.13
|
||||
command: --config /etc/livekit.yml
|
||||
network_mode: "host"
|
||||
volumes:
|
||||
- ./livekit.yml:/etc/livekit.yml
|
||||
|
||||
# Grafana LGTM (OTel Collector + Loki + Tempo + Prometheus + Grafana)
|
||||
otel-lgtm:
|
||||
image: grafana/otel-lgtm:0.35.0
|
||||
ports:
|
||||
- "13000:3000" # Grafana UI
|
||||
- "4318:4318" # OTLP/HTTP
|
||||
|
||||
# Waits for dependencies, used by `mise docker:start`
|
||||
wait:
|
||||
image: dokku/wait
|
||||
command: -c rabbit:5672,database:27017,redis:6379
|
||||
profiles: [tools]
|
||||
|
||||
29
crates/common/otel/Cargo.toml
Normal file
29
crates/common/otel/Cargo.toml
Normal file
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "stoat-otel"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
authors = ["izzy <me@insrt.uk>"]
|
||||
description = "Common OpenTelemetry configuration"
|
||||
publish = false
|
||||
|
||||
[features]
|
||||
axum = ["dep:axum", "dep:uuid", "dep:revolt-result", "dep:serde"]
|
||||
|
||||
[dependencies]
|
||||
# Logging
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
# OpenTelemetry
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true }
|
||||
opentelemetry-appender-tracing = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
|
||||
# HTTP
|
||||
axum = { workspace = true, optional = true }
|
||||
uuid = { workspace = true, optional = true, features = ["v4"] }
|
||||
revolt-result = { workspace = true, optional = true, features = ["axum"] }
|
||||
serde = { workspace = true, optional = true }
|
||||
3
crates/common/otel/src/implementation/mod.rs
Normal file
3
crates/common/otel/src/implementation/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod otel_log_impl;
|
||||
|
||||
pub(crate) use otel_log_impl::WIDE_LOGGER;
|
||||
70
crates/common/otel/src/implementation/otel_log_impl.rs
Normal file
70
crates/common/otel/src/implementation/otel_log_impl.rs
Normal file
@@ -0,0 +1,70 @@
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use opentelemetry::{
|
||||
Key,
|
||||
logs::{AnyValue, LogRecord, Logger, Severity},
|
||||
};
|
||||
use opentelemetry_sdk::logs::SdkLogger;
|
||||
use tracing::Level;
|
||||
|
||||
use crate::models::HttpRequestEvent;
|
||||
|
||||
pub(crate) static WIDE_LOGGER: OnceLock<SdkLogger> = OnceLock::new();
|
||||
|
||||
impl HttpRequestEvent {
|
||||
pub(crate) fn emit(self, attributes: Vec<(Key, AnyValue)>) {
|
||||
let level = self.level();
|
||||
let message = self.summary();
|
||||
let outcome = if self.rejected() { "error" } else { "success" };
|
||||
let duration_ms = self.duration.as_millis() as i64;
|
||||
|
||||
let logger = WIDE_LOGGER
|
||||
.get()
|
||||
.filter(|_| self.sampled || level == Level::ERROR);
|
||||
|
||||
if let Some(logger) = logger {
|
||||
let (severity, severity_text) = match level {
|
||||
Level::ERROR => (Severity::Error, "ERROR"),
|
||||
Level::WARN => (Severity::Warn, "WARN"),
|
||||
_ => (Severity::Info, "INFO"),
|
||||
};
|
||||
|
||||
let mut record = logger.create_log_record();
|
||||
record.set_event_name("http.request");
|
||||
record.set_severity_number(severity);
|
||||
record.set_severity_text(severity_text);
|
||||
record.set_body(message.clone().into());
|
||||
record.add_attribute("request_id", self.request_id);
|
||||
record.add_attribute("method", self.method);
|
||||
record.add_attribute("path", self.path);
|
||||
record.add_attribute("route", self.route);
|
||||
record.add_attribute("status_code", self.status as i64);
|
||||
record.add_attribute("outcome", outcome);
|
||||
record.add_attribute("duration_ms", duration_ms);
|
||||
|
||||
if let Some(error) = self.error {
|
||||
let kind: &'static str = (&error.error_type).into();
|
||||
record.add_attribute("error.type", kind);
|
||||
record.add_attribute("error.message", format!("{:?}", error.error_type));
|
||||
|
||||
if let Some(location) = error.location {
|
||||
record.add_attribute("error.location", location);
|
||||
}
|
||||
}
|
||||
|
||||
record.add_attributes(attributes);
|
||||
logger.emit(record);
|
||||
}
|
||||
|
||||
let status = self.status;
|
||||
match level {
|
||||
Level::ERROR => {
|
||||
tracing::error!(target: "http_request", status, duration_ms, "{message}")
|
||||
}
|
||||
Level::WARN => {
|
||||
tracing::warn!(target: "http_request", status, duration_ms, "{message}")
|
||||
}
|
||||
_ => tracing::info!(target: "http_request", status, duration_ms, "{message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
16
crates/common/otel/src/lib.rs
Normal file
16
crates/common/otel/src/lib.rs
Normal file
@@ -0,0 +1,16 @@
|
||||
#[cfg(feature = "axum")]
|
||||
mod implementation;
|
||||
pub mod middleware;
|
||||
#[cfg(feature = "axum")]
|
||||
mod models;
|
||||
#[cfg(feature = "axum")]
|
||||
mod repositories;
|
||||
mod telemetry;
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
pub use repositories::*;
|
||||
pub use telemetry::Telemetry;
|
||||
|
||||
pub fn init(service: &'static str, version: &'static str) -> Telemetry {
|
||||
Telemetry::create(service, version)
|
||||
}
|
||||
18
crates/common/otel/src/middleware/axum/context.rs
Normal file
18
crates/common/otel/src/middleware/axum/context.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
use std::convert::Infallible;
|
||||
|
||||
use ::axum::{async_trait, extract::FromRequestParts, http::request::Parts};
|
||||
|
||||
use crate::WideContextRepository;
|
||||
|
||||
#[async_trait]
|
||||
impl<S: Send + Sync> FromRequestParts<S> for WideContextRepository {
|
||||
type Rejection = Infallible;
|
||||
|
||||
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
|
||||
Ok(parts
|
||||
.extensions
|
||||
.get::<WideContextRepository>()
|
||||
.cloned()
|
||||
.unwrap_or_default())
|
||||
}
|
||||
}
|
||||
102
crates/common/otel/src/middleware/axum/events.rs
Normal file
102
crates/common/otel/src/middleware/axum/events.rs
Normal file
@@ -0,0 +1,102 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use ::axum::{
|
||||
Json,
|
||||
extract::{MatchedPath, Request},
|
||||
http::{HeaderValue, header::CONTENT_LENGTH},
|
||||
middleware::Next,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use opentelemetry::trace::TraceContextExt;
|
||||
use serde::Serialize;
|
||||
use tracing::{Instrument, Span, field::Empty, info_span};
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{WideContextRepository, models::HttpRequestEvent};
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ErrorBody<'a> {
|
||||
#[serde(flatten)]
|
||||
error: &'a revolt_result::Error,
|
||||
request_id: &'a str,
|
||||
}
|
||||
|
||||
impl HttpRequestEvent {
|
||||
fn from_axum_request(request: &Request) -> Self {
|
||||
let path = request.uri().path().to_owned();
|
||||
|
||||
HttpRequestEvent {
|
||||
request_id: request
|
||||
.headers()
|
||||
.get("x-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| Uuid::new_v4().to_string()),
|
||||
method: request.method().to_string(),
|
||||
route: request
|
||||
.extensions()
|
||||
.get::<MatchedPath>()
|
||||
.map(|matched| matched.as_str().to_owned())
|
||||
.unwrap_or_else(|| path.clone()),
|
||||
path,
|
||||
start: Instant::now(),
|
||||
status: 0,
|
||||
duration: Duration::ZERO,
|
||||
sampled: false,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_axum(&mut self, response: &mut Response, span: &Span) {
|
||||
self.status = response.status().as_u16();
|
||||
self.duration = self.start.elapsed();
|
||||
self.sampled = span.context().span().span_context().is_sampled();
|
||||
self.error = response.extensions_mut().remove::<revolt_result::Error>();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wide_events(mut request: Request, next: Next) -> Response {
|
||||
let context = WideContextRepository::default();
|
||||
request.extensions_mut().insert(context.clone());
|
||||
|
||||
let mut event = HttpRequestEvent::from_axum_request(&request);
|
||||
let span = info_span!(
|
||||
"http.request",
|
||||
otel.name = format!("{} {}", event.method, event.route),
|
||||
otel.kind = "server",
|
||||
request_id = event.request_id,
|
||||
http.request.method = event.method,
|
||||
http.route = event.route,
|
||||
url.path = event.path,
|
||||
http.response.status_code = Empty,
|
||||
otel.status_code = Empty,
|
||||
);
|
||||
|
||||
let mut response = next.run(request).instrument(span.clone()).await;
|
||||
event.finish_axum(&mut response, &span);
|
||||
span.record("http.response.status_code", event.status);
|
||||
|
||||
if event.failed() {
|
||||
span.record("otel.status_code", "error");
|
||||
}
|
||||
|
||||
if let Some(error) = &event.error {
|
||||
let (mut parts, _) = response.into_parts();
|
||||
parts.headers.remove(CONTENT_LENGTH);
|
||||
|
||||
let body = Json(ErrorBody {
|
||||
error,
|
||||
request_id: &event.request_id,
|
||||
});
|
||||
|
||||
response = Response::from_parts(parts, body.into_response().into_body());
|
||||
}
|
||||
|
||||
if let Ok(request_id) = HeaderValue::from_str(&event.request_id) {
|
||||
response.headers_mut().insert("x-request-id", request_id);
|
||||
}
|
||||
|
||||
span.in_scope(|| event.emit(context.take()));
|
||||
response
|
||||
}
|
||||
6
crates/common/otel/src/middleware/axum/mod.rs
Normal file
6
crates/common/otel/src/middleware/axum/mod.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
mod context;
|
||||
mod events;
|
||||
mod panic;
|
||||
|
||||
pub use events::wide_events;
|
||||
pub use panic::panic_response;
|
||||
12
crates/common/otel/src/middleware/axum/panic.rs
Normal file
12
crates/common/otel/src/middleware/axum/panic.rs
Normal file
@@ -0,0 +1,12 @@
|
||||
use std::any::Any;
|
||||
|
||||
use ::axum::response::{IntoResponse, Response};
|
||||
use revolt_result::{Error, ErrorType};
|
||||
|
||||
pub fn panic_response(_panic: Box<dyn Any + Send + 'static>) -> Response {
|
||||
Error {
|
||||
error_type: ErrorType::InternalError,
|
||||
location: None,
|
||||
}
|
||||
.into_response()
|
||||
}
|
||||
2
crates/common/otel/src/middleware/mod.rs
Normal file
2
crates/common/otel/src/middleware/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
#[cfg(feature = "axum")]
|
||||
pub mod axum;
|
||||
57
crates/common/otel/src/models/http_request_event.rs
Normal file
57
crates/common/otel/src/models/http_request_event.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tracing::Level;
|
||||
|
||||
const SLOW_REQUEST_THRESHOLD: Duration = Duration::from_millis(500);
|
||||
|
||||
pub(crate) struct HttpRequestEvent {
|
||||
pub request_id: String,
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
pub route: String,
|
||||
pub start: Instant,
|
||||
pub status: u16,
|
||||
pub duration: Duration,
|
||||
pub sampled: bool,
|
||||
pub error: Option<revolt_result::Error>,
|
||||
}
|
||||
|
||||
impl HttpRequestEvent {
|
||||
pub(crate) fn failed(&self) -> bool {
|
||||
self.status >= 500
|
||||
}
|
||||
|
||||
pub(crate) fn rejected(&self) -> bool {
|
||||
self.status >= 400
|
||||
}
|
||||
|
||||
fn slow(&self) -> bool {
|
||||
self.duration > SLOW_REQUEST_THRESHOLD
|
||||
}
|
||||
|
||||
pub(crate) fn level(&self) -> Level {
|
||||
if self.failed() {
|
||||
Level::ERROR
|
||||
} else if self.rejected() || self.slow() {
|
||||
Level::WARN
|
||||
} else {
|
||||
Level::INFO
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn summary(&self) -> String {
|
||||
let outcome = match &self.error {
|
||||
Some(error) => {
|
||||
let kind: &str = (&error.error_type).into();
|
||||
format!("ERROR {kind}")
|
||||
}
|
||||
None if self.rejected() => "ERROR".to_owned(),
|
||||
None => "OK".to_owned(),
|
||||
};
|
||||
let slow = if self.slow() { "[SLOW] " } else { "" };
|
||||
|
||||
let Self { method, route, .. } = self;
|
||||
|
||||
format!("{slow}{method} {route} ({outcome})")
|
||||
}
|
||||
}
|
||||
3
crates/common/otel/src/models/mod.rs
Normal file
3
crates/common/otel/src/models/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod http_request_event;
|
||||
|
||||
pub(crate) use http_request_event::HttpRequestEvent;
|
||||
3
crates/common/otel/src/repositories/mod.rs
Normal file
3
crates/common/otel/src/repositories/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod wide_context_repository;
|
||||
|
||||
pub use wide_context_repository::WideContextRepository;
|
||||
@@ -0,0 +1,21 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use opentelemetry::{Key, logs::AnyValue};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct WideContextRepository {
|
||||
attributes: Arc<Mutex<Vec<(Key, AnyValue)>>>,
|
||||
}
|
||||
|
||||
impl WideContextRepository {
|
||||
pub fn add(&self, key: impl Into<Key>, value: impl Into<AnyValue>) {
|
||||
self.attributes
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((key.into(), value.into()));
|
||||
}
|
||||
|
||||
pub(crate) fn take(&self) -> Vec<(Key, AnyValue)> {
|
||||
std::mem::take(&mut *self.attributes.lock().unwrap())
|
||||
}
|
||||
}
|
||||
122
crates/common/otel/src/telemetry.rs
Normal file
122
crates/common/otel/src/telemetry.rs
Normal file
@@ -0,0 +1,122 @@
|
||||
use opentelemetry::KeyValue;
|
||||
#[cfg(feature = "axum")]
|
||||
use opentelemetry::logs::LoggerProvider;
|
||||
use opentelemetry::trace::TracerProvider;
|
||||
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
|
||||
use opentelemetry_otlp::{LogExporter, Protocol, SpanExporter, WithExportConfig};
|
||||
use opentelemetry_sdk::Resource;
|
||||
use opentelemetry_sdk::logs::SdkLoggerProvider;
|
||||
use opentelemetry_sdk::trace::SdkTracerProvider;
|
||||
use tracing::debug;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
use crate::implementation::WIDE_LOGGER;
|
||||
|
||||
pub struct Telemetry {
|
||||
logger_provider: SdkLoggerProvider,
|
||||
tracer_provider: SdkTracerProvider,
|
||||
}
|
||||
|
||||
impl Telemetry {
|
||||
pub fn create(service: &'static str, version: &'static str) -> Telemetry {
|
||||
let resource = Resource::builder()
|
||||
.with_service_name(service)
|
||||
.with_attribute(KeyValue::new("service.version", version))
|
||||
.build();
|
||||
|
||||
let log_exporter = LogExporter::builder()
|
||||
.with_http()
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.build()
|
||||
.expect("Failed to create log exporter");
|
||||
|
||||
let logger_provider = SdkLoggerProvider::builder()
|
||||
.with_resource(resource.clone())
|
||||
.with_batch_exporter(log_exporter)
|
||||
.build();
|
||||
|
||||
let span_exporter = SpanExporter::builder()
|
||||
.with_http()
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.build()
|
||||
.expect("Failed to create span exporter");
|
||||
|
||||
let tracer_provider = SdkTracerProvider::builder()
|
||||
.with_resource(resource)
|
||||
.with_batch_exporter(span_exporter)
|
||||
.build();
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
let _ = WIDE_LOGGER.set(logger_provider.logger("stoat_otel::wide"));
|
||||
|
||||
let otel_layer =
|
||||
OpenTelemetryTracingBridge::new(&logger_provider).with_filter(export_filter());
|
||||
|
||||
let trace_layer = tracing_opentelemetry::layer()
|
||||
.with_tracer(tracer_provider.tracer("stoat_otel"))
|
||||
.with_filter(export_filter());
|
||||
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_thread_names(true)
|
||||
.with_filter(env_filter());
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(otel_layer)
|
||||
.with(trace_layer)
|
||||
.with(fmt_layer)
|
||||
.init();
|
||||
|
||||
let default_hook = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(move |info| {
|
||||
let message = info.payload_as_str().unwrap_or_default();
|
||||
let location = info.location();
|
||||
|
||||
tracing::error!(
|
||||
exception.type = "panic",
|
||||
exception.message = message,
|
||||
code.file.path = location.map(|location| location.file()),
|
||||
code.line.number = location.map(|location| location.line()),
|
||||
"panicked: {message}"
|
||||
);
|
||||
|
||||
default_hook(info);
|
||||
}));
|
||||
|
||||
debug!("hai i'm a log :3 (initialised OTLP successfully)");
|
||||
|
||||
Telemetry {
|
||||
logger_provider,
|
||||
tracer_provider,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn env_filter() -> EnvFilter {
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))
|
||||
}
|
||||
|
||||
fn export_filter() -> EnvFilter {
|
||||
env_filter()
|
||||
.add_directive("hyper=off".parse().unwrap())
|
||||
.add_directive("tonic=off".parse().unwrap())
|
||||
.add_directive("h2=off".parse().unwrap())
|
||||
.add_directive("reqwest=off".parse().unwrap())
|
||||
.add_directive("aws_smithy_runtime=off".parse().unwrap())
|
||||
.add_directive("aws_sdk_s3=off".parse().unwrap())
|
||||
.add_directive("tower_http::catch_panic=off".parse().unwrap())
|
||||
.add_directive("http_request=off".parse().unwrap())
|
||||
}
|
||||
|
||||
impl Drop for Telemetry {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.logger_provider.shutdown() {
|
||||
eprintln!("otel logger shutdown failed: {e}");
|
||||
}
|
||||
|
||||
if let Err(e) = self.tracer_provider.shutdown() {
|
||||
eprintln!("otel tracer shutdown failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,613 +1,5 @@
|
||||
#[cfg(feature = "test")]
|
||||
use std::sync::OnceLock;
|
||||
use std::{collections::HashMap, path::Path, sync::LazyLock};
|
||||
mod sentry;
|
||||
mod settings;
|
||||
|
||||
use cached::proc_macro::cached;
|
||||
use config::{Config, Environment, File, FileFormat};
|
||||
use futures_locks::RwLock;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[cfg(feature = "sentry")]
|
||||
pub use sentry::{Level, capture_error, capture_message};
|
||||
#[cfg(feature = "anyhow")]
|
||||
pub use sentry_anyhow::capture_anyhow;
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! report_error {
|
||||
( $expr: expr, $error: ident $( $tt:tt )? ) => {
|
||||
$expr
|
||||
.inspect_err(|err| {
|
||||
$crate::capture_message(
|
||||
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
})
|
||||
.map_err(|_| ::revolt_result::create_error!($error))
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! capture_internal_error {
|
||||
( $expr: expr ) => {
|
||||
$crate::capture_message(
|
||||
&format!("{:?} ({}:{}:{})", $expr, file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! report_internal_error {
|
||||
( $expr: expr ) => {
|
||||
$expr
|
||||
.inspect_err(|err| {
|
||||
$crate::capture_message(
|
||||
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
})
|
||||
.map_err(|_| ::revolt_result::create_error!(InternalError))
|
||||
};
|
||||
}
|
||||
|
||||
/// Paths to search for configuration
|
||||
static CONFIG_SEARCH_PATHS: [&str; 3] = [
|
||||
// current working directory
|
||||
"Revolt.toml",
|
||||
// current working directory - overrides file
|
||||
"Revolt.overrides.toml",
|
||||
// root directory, for Docker containers
|
||||
"/Revolt.toml",
|
||||
];
|
||||
|
||||
/// Path to search for test overrides
|
||||
static TEST_OVERRIDE_PATH: &str = "Revolt.test-overrides.toml";
|
||||
|
||||
/// Configuration builder
|
||||
static CONFIG_BUILDER: LazyLock<RwLock<Config>> = LazyLock::new(|| {
|
||||
RwLock::new({
|
||||
let mut builder = Config::builder().add_source(File::from_str(
|
||||
include_str!("../Revolt.toml"),
|
||||
FileFormat::Toml,
|
||||
));
|
||||
|
||||
let cwd = std::env::current_dir().unwrap();
|
||||
let mut cwd: Option<&Path> = Some(&cwd);
|
||||
|
||||
while let Some(path) = cwd {
|
||||
for config_path in CONFIG_SEARCH_PATHS {
|
||||
let config_path = path.join(config_path);
|
||||
if config_path.exists() {
|
||||
builder = builder
|
||||
.add_source(File::new(config_path.to_str().unwrap(), FileFormat::Toml));
|
||||
}
|
||||
}
|
||||
|
||||
cwd = path.parent();
|
||||
}
|
||||
|
||||
if std::env::var("TEST_DB").is_ok() {
|
||||
builder = builder.add_source(File::from_str(
|
||||
include_str!("../Revolt.test.toml"),
|
||||
FileFormat::Toml,
|
||||
));
|
||||
|
||||
// recursively search upwards for an overrides file (if there is one)
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let mut path = Some(cwd.as_path());
|
||||
while let Some(current_path) = path {
|
||||
let target_path = current_path.join(TEST_OVERRIDE_PATH);
|
||||
if target_path.exists() {
|
||||
builder = builder
|
||||
.add_source(File::new(target_path.to_str().unwrap(), FileFormat::Toml));
|
||||
}
|
||||
|
||||
path = current_path.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builder = builder.add_source(Environment::with_prefix("REVOLT").separator("__"));
|
||||
|
||||
builder.build().unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Database {
|
||||
pub mongodb: String,
|
||||
pub redis: String,
|
||||
pub redis_pubsub: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct RabbitQueues {
|
||||
pub acks: String,
|
||||
pub events: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Rabbit {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub default_exchange: String,
|
||||
pub queues: RabbitQueues,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Hosts {
|
||||
pub app: String,
|
||||
pub api: String,
|
||||
pub events: String,
|
||||
pub autumn: String,
|
||||
pub january: String,
|
||||
pub livekit: HashMap<String, String>,
|
||||
pub assets: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiRegistration {
|
||||
pub invite_only: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSmtp {
|
||||
pub host: String,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub from_address: String,
|
||||
pub reply_to: Option<String>,
|
||||
pub port: Option<i32>,
|
||||
pub use_tls: Option<bool>,
|
||||
pub use_starttls: Option<bool>,
|
||||
pub expiry: EmailExpiry,
|
||||
}
|
||||
|
||||
/// Email expiration config
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct EmailExpiry {
|
||||
/// How long email verification codes should last for (in seconds)
|
||||
pub expire_verification: i64,
|
||||
/// How long password reset codes should last for (in seconds)
|
||||
pub expire_password_reset: i64,
|
||||
/// How long account deletion codes should last for (in seconds)
|
||||
pub expire_account_deletion: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushVapid {
|
||||
pub queue: String,
|
||||
pub private_key: String,
|
||||
pub public_key: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushFcm {
|
||||
pub queue: String,
|
||||
pub key_type: String,
|
||||
pub project_id: String,
|
||||
pub private_key_id: String,
|
||||
pub private_key: String,
|
||||
pub client_email: String,
|
||||
pub client_id: String,
|
||||
pub auth_uri: String,
|
||||
pub token_uri: String,
|
||||
pub auth_provider_x509_cert_url: String,
|
||||
pub client_x509_cert_url: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushApn {
|
||||
pub queue: String,
|
||||
pub sandbox: bool,
|
||||
pub pkcs8: String,
|
||||
pub key_id: String,
|
||||
pub team_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurityCaptcha {
|
||||
pub hcaptcha_key: String,
|
||||
pub hcaptcha_sitekey: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurityShield {
|
||||
pub host: String,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurity {
|
||||
pub shield: ApiSecurityShield,
|
||||
pub captcha: ApiSecurityCaptcha,
|
||||
pub trust_cloudflare: bool,
|
||||
pub easypwned: String,
|
||||
pub tenor_key: String,
|
||||
pub admin_keys: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiWorkers {
|
||||
pub max_concurrent_connections: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiLiveKit {
|
||||
pub call_ring_duration: usize,
|
||||
pub nodes: HashMap<String, LiveKitNode>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct LiveKitNode {
|
||||
pub url: String,
|
||||
pub lat: f64,
|
||||
pub lon: f64,
|
||||
pub key: String,
|
||||
pub secret: String,
|
||||
|
||||
// whether to hide the node in the nodes list
|
||||
#[serde(default)]
|
||||
pub private: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiUsers {
|
||||
pub early_adopter_cutoff: Option<u64>,
|
||||
pub min_username_length: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiAuditLogs {
|
||||
/// How long audit log entries last before being removed, in seconds
|
||||
pub expires_after: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Api {
|
||||
pub registration: ApiRegistration,
|
||||
pub smtp: ApiSmtp,
|
||||
pub security: ApiSecurity,
|
||||
pub workers: ApiWorkers,
|
||||
pub livekit: ApiLiveKit,
|
||||
pub users: ApiUsers,
|
||||
pub audit_logs: ApiAuditLogs,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Pushd {
|
||||
pub production: bool,
|
||||
pub exchange: String,
|
||||
pub mass_mention_chunk_size: usize,
|
||||
pub render_cache_time: usize,
|
||||
|
||||
// Queues
|
||||
pub message_queue: String,
|
||||
pub mass_mention_queue: String,
|
||||
pub dm_call_queue: String,
|
||||
pub fr_accepted_queue: String,
|
||||
pub fr_received_queue: String,
|
||||
pub generic_queue: String,
|
||||
pub ack_queue: String,
|
||||
|
||||
pub vapid: PushVapid,
|
||||
pub fcm: PushFcm,
|
||||
pub apn: PushApn,
|
||||
}
|
||||
|
||||
impl Pushd {
|
||||
fn get_routing_key(&self, key: String) -> String {
|
||||
match self.production {
|
||||
true => key + "-prd",
|
||||
false => key + "-tst",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_ack_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.ack_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_message_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.message_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_mass_mention_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.mass_mention_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_dm_call_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.dm_call_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_fr_accepted_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.fr_accepted_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_fr_received_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.fr_received_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_generic_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.generic_queue.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct January {
|
||||
pub blocked_domains: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FilesLimit {
|
||||
pub min_file_size: usize,
|
||||
pub min_resolution: [usize; 2],
|
||||
pub max_mega_pixels: usize,
|
||||
pub max_pixel_side: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FilesS3 {
|
||||
pub endpoint: String,
|
||||
pub path_style_buckets: bool,
|
||||
pub region: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: String,
|
||||
pub default_bucket: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Files {
|
||||
pub encryption_key: String,
|
||||
pub webp_quality: f32,
|
||||
pub blocked_mime_types: Vec<String>,
|
||||
pub clamd_host: String,
|
||||
pub scan_mime_types: Vec<String>,
|
||||
|
||||
pub limit: FilesLimit,
|
||||
pub preview: HashMap<String, [usize; 2]>,
|
||||
pub s3: FilesS3,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct GlobalLimits {
|
||||
pub group_size: usize,
|
||||
pub message_embeds: usize,
|
||||
pub message_replies: usize,
|
||||
pub message_reactions: usize,
|
||||
pub server_emoji: usize,
|
||||
pub server_roles: usize,
|
||||
pub server_channels: usize,
|
||||
|
||||
pub new_user_hours: usize,
|
||||
|
||||
pub body_limit_size: usize,
|
||||
|
||||
pub restrict_server_creation: Vec<String>,
|
||||
|
||||
pub max_invite_duration_days: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesLimits {
|
||||
pub outgoing_friend_requests: usize,
|
||||
|
||||
pub bots: usize,
|
||||
pub message_length: usize,
|
||||
pub message_attachments: usize,
|
||||
pub servers: usize,
|
||||
pub voice_quality: u32,
|
||||
pub video: bool,
|
||||
pub video_resolution: [u32; 2],
|
||||
pub video_aspect_ratio: [f32; 2],
|
||||
|
||||
pub file_upload_size_limit: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesLimitsCollection {
|
||||
pub global: GlobalLimits,
|
||||
|
||||
pub new_user: FeaturesLimits,
|
||||
pub default: FeaturesLimits,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub roles: HashMap<String, FeaturesLimits>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct LegalLinks {
|
||||
/// Terms of Service URL
|
||||
pub terms_of_service: String,
|
||||
/// Privacy Policy URL
|
||||
pub privacy_policy: String,
|
||||
/// Guidelines URL
|
||||
pub guidelines: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesAdvanced {
|
||||
#[serde(default)]
|
||||
pub process_message_delay_limit: u16,
|
||||
#[serde(default)]
|
||||
pub seen_events_cache_size: u32,
|
||||
}
|
||||
|
||||
impl Default for FeaturesAdvanced {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
process_message_delay_limit: 5,
|
||||
seen_events_cache_size: 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Features {
|
||||
pub limits: FeaturesLimitsCollection,
|
||||
pub legal_links: LegalLinks,
|
||||
pub webhooks_enabled: bool,
|
||||
pub mass_mentions_send_notifications: bool,
|
||||
pub mass_mentions_enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub advanced: FeaturesAdvanced,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Sentry {
|
||||
pub api: String,
|
||||
pub events: String,
|
||||
pub voice_ingress: String,
|
||||
pub files: String,
|
||||
pub proxy: String,
|
||||
pub pushd: String,
|
||||
pub crond: String,
|
||||
pub gifbox: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Settings {
|
||||
pub database: Database,
|
||||
pub rabbit: Rabbit,
|
||||
pub hosts: Hosts,
|
||||
pub api: Api,
|
||||
pub pushd: Pushd,
|
||||
pub january: January,
|
||||
pub files: Files,
|
||||
pub features: Features,
|
||||
pub sentry: Sentry,
|
||||
pub production: bool,
|
||||
pub environment: String,
|
||||
pub disable_events_dont_use: bool,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn preflight_checks(&self) {
|
||||
if self.api.smtp.host.is_empty() {
|
||||
log::warn!("No SMTP settings specified! Remember to configure email.");
|
||||
}
|
||||
|
||||
if self.api.security.captcha.hcaptcha_key.is_empty() {
|
||||
log::warn!("No Captcha key specified! Remember to add hCaptcha key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init() {
|
||||
println!(
|
||||
":: Revolt Configuration ::\n\x1b[32m{:?}\x1b[0m",
|
||||
config().await
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn read() -> Config {
|
||||
CONFIG_BUILDER.read().await.clone()
|
||||
}
|
||||
|
||||
pub async fn config_no_cache() -> Settings {
|
||||
let mut config = read().await.try_deserialize::<Settings>().unwrap();
|
||||
|
||||
// inject REDIS_URI for redis-kiss library
|
||||
if std::env::var("REDIS_URI").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("REDIS_URI", config.database.redis.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// auto-detect production nodes
|
||||
if config.hosts.api.contains("https")
|
||||
&& (config.hosts.api.contains("revolt.chat") || config.hosts.api.contains("stoat.chat"))
|
||||
{
|
||||
config.production = true;
|
||||
}
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
#[cached(time = 30)]
|
||||
pub async fn config() -> Settings {
|
||||
#[cfg(feature = "test")]
|
||||
if let Some(overwrites) = CONFIG_OVERWRITES.get() {
|
||||
return overwrites.clone();
|
||||
}
|
||||
|
||||
config_no_cache().await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test")]
|
||||
static CONFIG_OVERWRITES: OnceLock<Settings> = OnceLock::new();
|
||||
|
||||
/// Modify the config values for a test, this can only be called once
|
||||
///
|
||||
/// This will also fail if two or more tests are running in the same process and both try to modify the config,
|
||||
/// This could happen if tests where run under `cargo test` instead of `nextest`.
|
||||
#[cfg(feature = "test")]
|
||||
pub async fn overwrite_config(f: impl FnOnce(&mut Settings)) {
|
||||
let mut config = config_no_cache().await;
|
||||
|
||||
f(&mut config);
|
||||
|
||||
CONFIG_OVERWRITES.set(config).expect(
|
||||
"Cannot overwrite config multiple times, make sure you are running tests through nextest.",
|
||||
);
|
||||
}
|
||||
|
||||
/// Configure logging and common Rust variables
|
||||
#[cfg(feature = "sentry")]
|
||||
pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("RUST_LOG", "info");
|
||||
}
|
||||
}
|
||||
|
||||
if std::env::var("ROCKET_ADDRESS").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("ROCKET_ADDRESS", "0.0.0.0");
|
||||
}
|
||||
}
|
||||
|
||||
pretty_env_logger::init();
|
||||
log::info!("Starting {release}");
|
||||
|
||||
if dsn.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(sentry::init((
|
||||
dsn,
|
||||
sentry::ClientOptions {
|
||||
release: Some(release.into()),
|
||||
..Default::default()
|
||||
},
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "sentry")]
|
||||
#[macro_export]
|
||||
macro_rules! configure {
|
||||
($application: ident) => {
|
||||
let config = $crate::config().await;
|
||||
let _sentry = $crate::setup_logging(
|
||||
concat!(env!("CARGO_PKG_NAME"), "@", env!("CARGO_PKG_VERSION")),
|
||||
config.sentry.$application,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(feature = "test")]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::init;
|
||||
|
||||
#[tokio::test]
|
||||
async fn it_works() {
|
||||
init().await;
|
||||
}
|
||||
}
|
||||
pub use sentry::*;
|
||||
pub use settings::*;
|
||||
|
||||
89
crates/core/config/src/sentry.rs
Normal file
89
crates/core/config/src/sentry.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
#[cfg(feature = "sentry")]
|
||||
pub use sentry::{Level, capture_error, capture_message};
|
||||
#[cfg(feature = "anyhow")]
|
||||
pub use sentry_anyhow::capture_anyhow;
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! report_error {
|
||||
( $expr: expr, $error: ident $( $tt:tt )? ) => {
|
||||
$expr
|
||||
.inspect_err(|err| {
|
||||
$crate::capture_message(
|
||||
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
})
|
||||
.map_err(|_| ::revolt_result::create_error!($error))
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! capture_internal_error {
|
||||
( $expr: expr ) => {
|
||||
$crate::capture_message(
|
||||
&format!("{:?} ({}:{}:{})", $expr, file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "report-macros", feature = "sentry"))]
|
||||
#[macro_export]
|
||||
macro_rules! report_internal_error {
|
||||
( $expr: expr ) => {
|
||||
$expr
|
||||
.inspect_err(|err| {
|
||||
$crate::capture_message(
|
||||
&format!("{err:?} ({}:{}:{})", file!(), line!(), column!()),
|
||||
$crate::Level::Error,
|
||||
);
|
||||
})
|
||||
.map_err(|_| ::revolt_result::create_error!(InternalError))
|
||||
};
|
||||
}
|
||||
|
||||
/// Configure logging and common Rust variables
|
||||
#[cfg(feature = "sentry")]
|
||||
pub async fn setup_logging(release: &'static str, dsn: String) -> Option<sentry::ClientInitGuard> {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("RUST_LOG", "info");
|
||||
}
|
||||
}
|
||||
|
||||
if std::env::var("ROCKET_ADDRESS").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("ROCKET_ADDRESS", "0.0.0.0");
|
||||
}
|
||||
}
|
||||
|
||||
pretty_env_logger::init();
|
||||
log::info!("Starting {release}");
|
||||
|
||||
if dsn.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(sentry::init((
|
||||
dsn,
|
||||
sentry::ClientOptions {
|
||||
release: Some(release.into()),
|
||||
..Default::default()
|
||||
},
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "sentry")]
|
||||
#[macro_export]
|
||||
macro_rules! configure {
|
||||
($application: ident) => {
|
||||
let config = $crate::config().await;
|
||||
let _sentry = $crate::setup_logging(
|
||||
concat!(env!("CARGO_PKG_NAME"), "@", env!("CARGO_PKG_VERSION")),
|
||||
config.sentry.$application,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
}
|
||||
523
crates/core/config/src/settings.rs
Normal file
523
crates/core/config/src/settings.rs
Normal file
@@ -0,0 +1,523 @@
|
||||
#[cfg(feature = "test")]
|
||||
use std::sync::OnceLock;
|
||||
use std::{collections::HashMap, path::Path, sync::LazyLock};
|
||||
|
||||
use cached::proc_macro::cached;
|
||||
use config::{Config, Environment, File, FileFormat};
|
||||
use futures_locks::RwLock;
|
||||
use serde::Deserialize;
|
||||
|
||||
/// Paths to search for configuration
|
||||
static CONFIG_SEARCH_PATHS: [&str; 3] = [
|
||||
// current working directory
|
||||
"Revolt.toml",
|
||||
// current working directory - overrides file
|
||||
"Revolt.overrides.toml",
|
||||
// root directory, for Docker containers
|
||||
"/Revolt.toml",
|
||||
];
|
||||
|
||||
/// Path to search for test overrides
|
||||
static TEST_OVERRIDE_PATH: &str = "Revolt.test-overrides.toml";
|
||||
|
||||
/// Configuration builder
|
||||
static CONFIG_BUILDER: LazyLock<RwLock<Config>> = LazyLock::new(|| {
|
||||
RwLock::new({
|
||||
let mut builder = Config::builder().add_source(File::from_str(
|
||||
include_str!("../Revolt.toml"),
|
||||
FileFormat::Toml,
|
||||
));
|
||||
|
||||
let cwd = std::env::current_dir().unwrap();
|
||||
let mut cwd: Option<&Path> = Some(&cwd);
|
||||
|
||||
while let Some(path) = cwd {
|
||||
for config_path in CONFIG_SEARCH_PATHS {
|
||||
let config_path = path.join(config_path);
|
||||
if config_path.exists() {
|
||||
builder = builder
|
||||
.add_source(File::new(config_path.to_str().unwrap(), FileFormat::Toml));
|
||||
}
|
||||
}
|
||||
|
||||
cwd = path.parent();
|
||||
}
|
||||
|
||||
if std::env::var("TEST_DB").is_ok() {
|
||||
builder = builder.add_source(File::from_str(
|
||||
include_str!("../Revolt.test.toml"),
|
||||
FileFormat::Toml,
|
||||
));
|
||||
|
||||
// recursively search upwards for an overrides file (if there is one)
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let mut path = Some(cwd.as_path());
|
||||
while let Some(current_path) = path {
|
||||
let target_path = current_path.join(TEST_OVERRIDE_PATH);
|
||||
if target_path.exists() {
|
||||
builder = builder
|
||||
.add_source(File::new(target_path.to_str().unwrap(), FileFormat::Toml));
|
||||
}
|
||||
|
||||
path = current_path.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builder = builder.add_source(Environment::with_prefix("REVOLT").separator("__"));
|
||||
|
||||
builder.build().unwrap()
|
||||
})
|
||||
});
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Database {
|
||||
pub mongodb: String,
|
||||
pub redis: String,
|
||||
pub redis_pubsub: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct RabbitQueues {
|
||||
pub acks: String,
|
||||
pub events: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Rabbit {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub default_exchange: String,
|
||||
pub queues: RabbitQueues,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Hosts {
|
||||
pub app: String,
|
||||
pub api: String,
|
||||
pub events: String,
|
||||
pub autumn: String,
|
||||
pub january: String,
|
||||
pub livekit: HashMap<String, String>,
|
||||
pub assets: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiRegistration {
|
||||
pub invite_only: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSmtp {
|
||||
pub host: String,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub from_address: String,
|
||||
pub reply_to: Option<String>,
|
||||
pub port: Option<i32>,
|
||||
pub use_tls: Option<bool>,
|
||||
pub use_starttls: Option<bool>,
|
||||
pub expiry: EmailExpiry,
|
||||
}
|
||||
|
||||
/// Email expiration config
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct EmailExpiry {
|
||||
/// How long email verification codes should last for (in seconds)
|
||||
pub expire_verification: i64,
|
||||
/// How long password reset codes should last for (in seconds)
|
||||
pub expire_password_reset: i64,
|
||||
/// How long account deletion codes should last for (in seconds)
|
||||
pub expire_account_deletion: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushVapid {
|
||||
pub queue: String,
|
||||
pub private_key: String,
|
||||
pub public_key: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushFcm {
|
||||
pub queue: String,
|
||||
pub key_type: String,
|
||||
pub project_id: String,
|
||||
pub private_key_id: String,
|
||||
pub private_key: String,
|
||||
pub client_email: String,
|
||||
pub client_id: String,
|
||||
pub auth_uri: String,
|
||||
pub token_uri: String,
|
||||
pub auth_provider_x509_cert_url: String,
|
||||
pub client_x509_cert_url: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct PushApn {
|
||||
pub queue: String,
|
||||
pub sandbox: bool,
|
||||
pub pkcs8: String,
|
||||
pub key_id: String,
|
||||
pub team_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurityCaptcha {
|
||||
pub hcaptcha_key: String,
|
||||
pub hcaptcha_sitekey: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurityShield {
|
||||
pub host: String,
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiSecurity {
|
||||
pub shield: ApiSecurityShield,
|
||||
pub captcha: ApiSecurityCaptcha,
|
||||
pub trust_cloudflare: bool,
|
||||
pub easypwned: String,
|
||||
pub tenor_key: String,
|
||||
pub admin_keys: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiWorkers {
|
||||
pub max_concurrent_connections: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiLiveKit {
|
||||
pub call_ring_duration: usize,
|
||||
pub nodes: HashMap<String, LiveKitNode>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct LiveKitNode {
|
||||
pub url: String,
|
||||
pub lat: f64,
|
||||
pub lon: f64,
|
||||
pub key: String,
|
||||
pub secret: String,
|
||||
|
||||
// whether to hide the node in the nodes list
|
||||
#[serde(default)]
|
||||
pub private: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiUsers {
|
||||
pub early_adopter_cutoff: Option<u64>,
|
||||
pub min_username_length: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct ApiAuditLogs {
|
||||
/// How long audit log entries last before being removed, in seconds
|
||||
pub expires_after: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Api {
|
||||
pub registration: ApiRegistration,
|
||||
pub smtp: ApiSmtp,
|
||||
pub security: ApiSecurity,
|
||||
pub workers: ApiWorkers,
|
||||
pub livekit: ApiLiveKit,
|
||||
pub users: ApiUsers,
|
||||
pub audit_logs: ApiAuditLogs,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Pushd {
|
||||
pub production: bool,
|
||||
pub exchange: String,
|
||||
pub mass_mention_chunk_size: usize,
|
||||
pub render_cache_time: usize,
|
||||
|
||||
// Queues
|
||||
pub message_queue: String,
|
||||
pub mass_mention_queue: String,
|
||||
pub dm_call_queue: String,
|
||||
pub fr_accepted_queue: String,
|
||||
pub fr_received_queue: String,
|
||||
pub generic_queue: String,
|
||||
pub ack_queue: String,
|
||||
|
||||
pub vapid: PushVapid,
|
||||
pub fcm: PushFcm,
|
||||
pub apn: PushApn,
|
||||
}
|
||||
|
||||
impl Pushd {
|
||||
fn get_routing_key(&self, key: String) -> String {
|
||||
match self.production {
|
||||
true => key + "-prd",
|
||||
false => key + "-tst",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_ack_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.ack_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_message_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.message_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_mass_mention_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.mass_mention_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_dm_call_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.dm_call_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_fr_accepted_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.fr_accepted_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_fr_received_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.fr_received_queue.clone())
|
||||
}
|
||||
|
||||
pub fn get_generic_routing_key(&self) -> String {
|
||||
self.get_routing_key(self.generic_queue.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct January {
|
||||
pub blocked_domains: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FilesLimit {
|
||||
pub min_file_size: usize,
|
||||
pub min_resolution: [usize; 2],
|
||||
pub max_mega_pixels: usize,
|
||||
pub max_pixel_side: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FilesS3 {
|
||||
pub endpoint: String,
|
||||
pub path_style_buckets: bool,
|
||||
pub region: String,
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: String,
|
||||
pub default_bucket: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Files {
|
||||
pub encryption_key: String,
|
||||
pub webp_quality: f32,
|
||||
pub blocked_mime_types: Vec<String>,
|
||||
pub clamd_host: String,
|
||||
pub scan_mime_types: Vec<String>,
|
||||
|
||||
pub limit: FilesLimit,
|
||||
pub preview: HashMap<String, [usize; 2]>,
|
||||
pub s3: FilesS3,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct GlobalLimits {
|
||||
pub group_size: usize,
|
||||
pub message_embeds: usize,
|
||||
pub message_replies: usize,
|
||||
pub message_reactions: usize,
|
||||
pub server_emoji: usize,
|
||||
pub server_roles: usize,
|
||||
pub server_channels: usize,
|
||||
|
||||
pub new_user_hours: usize,
|
||||
|
||||
pub body_limit_size: usize,
|
||||
|
||||
pub restrict_server_creation: Vec<String>,
|
||||
|
||||
pub max_invite_duration_days: usize,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesLimits {
|
||||
pub outgoing_friend_requests: usize,
|
||||
|
||||
pub bots: usize,
|
||||
pub message_length: usize,
|
||||
pub message_attachments: usize,
|
||||
pub servers: usize,
|
||||
pub voice_quality: u32,
|
||||
pub video: bool,
|
||||
pub video_resolution: [u32; 2],
|
||||
pub video_aspect_ratio: [f32; 2],
|
||||
|
||||
pub file_upload_size_limit: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesLimitsCollection {
|
||||
pub global: GlobalLimits,
|
||||
|
||||
pub new_user: FeaturesLimits,
|
||||
pub default: FeaturesLimits,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub roles: HashMap<String, FeaturesLimits>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct LegalLinks {
|
||||
/// Terms of Service URL
|
||||
pub terms_of_service: String,
|
||||
/// Privacy Policy URL
|
||||
pub privacy_policy: String,
|
||||
/// Guidelines URL
|
||||
pub guidelines: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct FeaturesAdvanced {
|
||||
#[serde(default)]
|
||||
pub process_message_delay_limit: u16,
|
||||
#[serde(default)]
|
||||
pub seen_events_cache_size: u32,
|
||||
}
|
||||
|
||||
impl Default for FeaturesAdvanced {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
process_message_delay_limit: 5,
|
||||
seen_events_cache_size: 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Features {
|
||||
pub limits: FeaturesLimitsCollection,
|
||||
pub legal_links: LegalLinks,
|
||||
pub webhooks_enabled: bool,
|
||||
pub mass_mentions_send_notifications: bool,
|
||||
pub mass_mentions_enabled: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub advanced: FeaturesAdvanced,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Sentry {
|
||||
pub api: String,
|
||||
pub events: String,
|
||||
pub voice_ingress: String,
|
||||
pub files: String,
|
||||
pub proxy: String,
|
||||
pub pushd: String,
|
||||
pub crond: String,
|
||||
pub gifbox: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Settings {
|
||||
pub database: Database,
|
||||
pub rabbit: Rabbit,
|
||||
pub hosts: Hosts,
|
||||
pub api: Api,
|
||||
pub pushd: Pushd,
|
||||
pub january: January,
|
||||
pub files: Files,
|
||||
pub features: Features,
|
||||
pub sentry: Sentry,
|
||||
pub production: bool,
|
||||
pub environment: String,
|
||||
pub disable_events_dont_use: bool,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn preflight_checks(&self) {
|
||||
if self.api.smtp.host.is_empty() {
|
||||
log::warn!("No SMTP settings specified! Remember to configure email.");
|
||||
}
|
||||
|
||||
if self.api.security.captcha.hcaptcha_key.is_empty() {
|
||||
log::warn!("No Captcha key specified! Remember to add hCaptcha key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init() {
|
||||
println!(
|
||||
":: Revolt Configuration ::\n\x1b[32m{:?}\x1b[0m",
|
||||
config().await
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn read() -> Config {
|
||||
CONFIG_BUILDER.read().await.clone()
|
||||
}
|
||||
|
||||
pub async fn config_no_cache() -> Settings {
|
||||
let mut config = read().await.try_deserialize::<Settings>().unwrap();
|
||||
|
||||
// inject REDIS_URI for redis-kiss library
|
||||
if std::env::var("REDIS_URI").is_err() {
|
||||
unsafe {
|
||||
std::env::set_var("REDIS_URI", config.database.redis.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// auto-detect production nodes
|
||||
if config.hosts.api.contains("https")
|
||||
&& (config.hosts.api.contains("revolt.chat") || config.hosts.api.contains("stoat.chat"))
|
||||
{
|
||||
config.production = true;
|
||||
}
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
#[cached(time = 30)]
|
||||
pub async fn config() -> Settings {
|
||||
#[cfg(feature = "test")]
|
||||
if let Some(overwrites) = CONFIG_OVERWRITES.get() {
|
||||
return overwrites.clone();
|
||||
}
|
||||
|
||||
config_no_cache().await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test")]
|
||||
static CONFIG_OVERWRITES: OnceLock<Settings> = OnceLock::new();
|
||||
|
||||
/// Modify the config values for a test, this can only be called once
|
||||
///
|
||||
/// This will also fail if two or more tests are running in the same process and both try to modify the config,
|
||||
/// This could happen if tests where run under `cargo test` instead of `nextest`.
|
||||
#[cfg(feature = "test")]
|
||||
pub async fn overwrite_config(f: impl FnOnce(&mut Settings)) {
|
||||
let mut config = config_no_cache().await;
|
||||
|
||||
f(&mut config);
|
||||
|
||||
CONFIG_OVERWRITES.set(config).expect(
|
||||
"Cannot overwrite config multiple times, make sure you are running tests through nextest.",
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test")]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::init;
|
||||
|
||||
#[tokio::test]
|
||||
async fn it_works() {
|
||||
init().await;
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ revolt-coalesced = { workspace = true }
|
||||
|
||||
# Utility
|
||||
log = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
lru = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
ulid = { workspace = true }
|
||||
|
||||
@@ -28,11 +28,31 @@ extern crate bson;
|
||||
#[cfg(not(feature = "tokio-runtime"))]
|
||||
compile_error!("tokio-runtime feature must be enabled.");
|
||||
|
||||
#[macro_export]
|
||||
#[doc(hidden)]
|
||||
macro_rules! query_span {
|
||||
( $type: ident, $collection: expr ) => {
|
||||
::tracing::info_span!(
|
||||
"mongodb",
|
||||
otel.name = format!("{} {}", stringify!($type), $collection),
|
||||
otel.kind = "client",
|
||||
db.system = "mongodb",
|
||||
db.collection.name = $collection,
|
||||
db.operation.name = stringify!($type),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
#[cfg(debug_assertions)]
|
||||
macro_rules! query {
|
||||
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
|
||||
Ok($self.$type($collection, $($rest),+).await.unwrap())
|
||||
Ok(::tracing::Instrument::instrument(
|
||||
$self.$type($collection, $($rest),+),
|
||||
$crate::query_span!($type, $collection),
|
||||
)
|
||||
.await
|
||||
.unwrap())
|
||||
};
|
||||
}
|
||||
|
||||
@@ -40,11 +60,15 @@ macro_rules! query {
|
||||
#[cfg(not(debug_assertions))]
|
||||
macro_rules! query {
|
||||
( $self: ident, $type: ident, $collection: expr, $($rest:expr),+ ) => {
|
||||
$self.$type($collection, $($rest),+).await
|
||||
.map_err(|err| {
|
||||
revolt_config::capture_internal_error!(err);
|
||||
create_database_error!(stringify!($type), $collection)
|
||||
})
|
||||
::tracing::Instrument::instrument(
|
||||
$self.$type($collection, $($rest),+),
|
||||
$crate::query_span!($type, $collection),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
revolt_config::capture_internal_error!(err);
|
||||
create_database_error!(stringify!($type), $collection)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ impl EncryptionKey {
|
||||
}
|
||||
|
||||
impl EncryptionRepository for EncryptionKey {
|
||||
#[tracing::instrument(skip_all, fields(size = buf.len()))]
|
||||
fn decrypt_buffer(&self, mut buf: Vec<u8>, iv: &str) -> anyhow::Result<Vec<u8>> {
|
||||
let iv = &BASE64_STANDARD.decode(iv).unwrap()[..];
|
||||
let iv: &Nonce<typenum::consts::U12> = iv.into();
|
||||
@@ -43,6 +44,7 @@ impl EncryptionRepository for EncryptionKey {
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all, fields(size = buf.len()))]
|
||||
fn encrypt_buffer(&self, buf: &[u8]) -> anyhow::Result<(Vec<u8>, String)> {
|
||||
let iv = Aes256Gcm::generate_nonce(&mut OsRng);
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ impl MediaRepository for MediaImpl {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, v))]
|
||||
fn image_size_vec(&self, v: &[u8], mime: &str) -> Option<(usize, usize)> {
|
||||
match mime {
|
||||
"image/svg+xml" => {
|
||||
@@ -83,6 +84,7 @@ impl MediaRepository for MediaImpl {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, reader))]
|
||||
fn decode_image<R: Read + BufRead + Seek>(
|
||||
&self,
|
||||
reader: &mut R,
|
||||
@@ -146,6 +148,7 @@ impl MediaRepository for MediaImpl {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, reader))]
|
||||
fn is_valid_image<R: Read + BufRead + Seek>(&self, reader: &mut R, mime: &str) -> bool {
|
||||
match mime {
|
||||
"image/jxl" => jxl_oxide::JxlImage::builder()
|
||||
@@ -162,6 +165,7 @@ impl MediaRepository for MediaImpl {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, image))]
|
||||
fn create_thumbnail(&self, image: DynamicImage, tag: &str) -> Vec<u8> {
|
||||
let [w, h] = self.config.preview.get(tag).unwrap();
|
||||
|
||||
@@ -186,6 +190,7 @@ impl MediaRepository for MediaImpl {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
fn video_size(&self, f: &NamedTempFile) -> Option<(i64, i64)> {
|
||||
if let Ok(data) = ffprobe::ffprobe(f.path())
|
||||
.inspect_err(|err| tracing::error!("Failed to ffprobe file! {err:?}"))
|
||||
|
||||
@@ -45,6 +45,7 @@ impl<ER: EncryptionRepository> S3Storage<ER> {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl<ER: EncryptionRepository> FileStorageRepository for S3Storage<ER> {
|
||||
#[tracing::instrument(skip(self), err)]
|
||||
async fn create_bucket(&self, bucket_id: &str) -> anyhow::Result<()> {
|
||||
self.client
|
||||
.create_bucket()
|
||||
@@ -56,6 +57,7 @@ impl<ER: EncryptionRepository> FileStorageRepository for S3Storage<ER> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, iv), err)]
|
||||
async fn fetch_and_decrypt_file(
|
||||
&self,
|
||||
bucket_id: &str,
|
||||
@@ -84,6 +86,7 @@ impl<ER: EncryptionRepository> FileStorageRepository for S3Storage<ER> {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, buf), fields(size = buf.len()), err)]
|
||||
async fn encrypt_and_upload_file(
|
||||
&self,
|
||||
bucket_id: &str,
|
||||
@@ -104,6 +107,7 @@ impl<ER: EncryptionRepository> FileStorageRepository for S3Storage<ER> {
|
||||
Ok(iv)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), err)]
|
||||
async fn delete_file(&self, bucket_id: &str, path: &str) -> anyhow::Result<()> {
|
||||
self.client
|
||||
.delete_object()
|
||||
|
||||
@@ -36,6 +36,7 @@ revolt_okapi = { workspace = true, optional = true }
|
||||
|
||||
# utilities
|
||||
log = { workspace = true }
|
||||
strum_macros = { workspace = true }
|
||||
# Axum
|
||||
axum = { workspace = true, optional = true }
|
||||
|
||||
|
||||
@@ -125,6 +125,8 @@ impl IntoResponse for Error {
|
||||
ErrorType::IncorrectData { .. } => StatusCode::BAD_REQUEST,
|
||||
};
|
||||
|
||||
(status, Json(&self)).into_response()
|
||||
let mut response = (status, Json(&self)).into_response();
|
||||
response.extensions_mut().insert(self);
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use std::fmt::Display;
|
||||
use std::panic::Location;
|
||||
|
||||
use strum_macros::IntoStaticStr;
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
#[macro_use]
|
||||
extern crate serde;
|
||||
@@ -65,7 +67,7 @@ impl serde::Serialize for Error {
|
||||
| ErrorType::UnverifiedAccount
|
||||
| ErrorType::LockedOut
|
||||
| ErrorType::DisallowedMFAMethod => None,
|
||||
_ => Some(self.location.as_deref().unwrap()),
|
||||
_ => self.location.as_deref(),
|
||||
};
|
||||
|
||||
Body {
|
||||
@@ -81,7 +83,7 @@ impl serde::Serialize for Error {
|
||||
#[cfg_attr(feature = "serde", serde(tag = "type"))]
|
||||
#[cfg_attr(feature = "schemas", derive(JsonSchema))]
|
||||
#[cfg_attr(feature = "utoipa", derive(ToSchema))]
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, IntoStaticStr)]
|
||||
pub enum ErrorType {
|
||||
/// This error was not labeled :(
|
||||
LabelMe,
|
||||
|
||||
@@ -47,19 +47,22 @@ tokio = { workspace = true, features = ["full"] }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
# Core crates
|
||||
# Core
|
||||
revolt-files = { workspace = true }
|
||||
revolt-config = { workspace = true }
|
||||
revolt-database = { workspace = true, features = ["axum-impl"] }
|
||||
revolt-result = { workspace = true, features = ["utoipa", "axum"] }
|
||||
revolt-ratelimits = { workspace = true, features = ["axum"] }
|
||||
|
||||
# Common
|
||||
stoat-otel = { workspace = true, features = ["axum"] }
|
||||
|
||||
# Axum / web server
|
||||
tempfile = { workspace = true }
|
||||
axum-macros = { workspace = true }
|
||||
axum_typed_multipart = { workspace = true }
|
||||
axum = { workspace = true, features = ["multipart"] }
|
||||
tower-http = { workspace = true, features = ["cors"] }
|
||||
tower-http = { workspace = true, features = ["catch-panic", "cors"] }
|
||||
|
||||
# OpenAPI & documentation generation
|
||||
utoipa-scalar = { workspace = true, features = ["axum"] }
|
||||
|
||||
@@ -21,6 +21,7 @@ use revolt_files::{
|
||||
use revolt_result::{create_error, Error, Result, ToRevoltError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
use stoat_otel::WideContextRepository;
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::time::Instant;
|
||||
use tower_http::cors::{AllowHeaders, Any, CorsLayer};
|
||||
@@ -106,6 +107,7 @@ static CRATE_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/",
|
||||
tag = "Files",
|
||||
responses(
|
||||
(status = 200, description = "Echo response", body = RootResponse)
|
||||
)
|
||||
@@ -163,6 +165,7 @@ pub struct UploadResponse {
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/{tag}",
|
||||
tag = "Files",
|
||||
responses(
|
||||
(status = 200, description = "Upload was successful", body = UploadResponse)
|
||||
),
|
||||
@@ -177,10 +180,13 @@ pub struct UploadResponse {
|
||||
)]
|
||||
async fn upload_file(
|
||||
State(db): State<Database>,
|
||||
wide_context: WideContextRepository,
|
||||
user: User,
|
||||
Path(tag): Path<Tag>,
|
||||
TypedMultipart(UploadPayload { mut file }): TypedMultipart<UploadPayload>,
|
||||
) -> Result<Json<UploadResponse>> {
|
||||
wide_context.add("user_id", user.id.clone());
|
||||
|
||||
// Fetch configuration
|
||||
let config = config().await;
|
||||
|
||||
@@ -358,6 +364,7 @@ pub static CACHE_CONTROL: &str = "public, max-age=604800, must-revalidate";
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/{tag}/{file_id}",
|
||||
tag = "Files",
|
||||
responses(
|
||||
(status = 200, description = "Generated preview", body = Vec<u8>)
|
||||
),
|
||||
@@ -452,6 +459,7 @@ async fn fetch_preview(
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/{tag}/{file_id}/{file_name}",
|
||||
tag = "Files",
|
||||
responses(
|
||||
(status = 200, description = "Original file", body = Vec<u8>)
|
||||
),
|
||||
|
||||
@@ -3,7 +3,6 @@ use std::time::Duration;
|
||||
use revolt_config::{config, report_internal_error};
|
||||
use revolt_result::Result;
|
||||
|
||||
/// Initialise ClamAV
|
||||
pub async fn init() {
|
||||
let config = config().await;
|
||||
|
||||
@@ -32,7 +31,6 @@ pub async fn init() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan for malware
|
||||
pub async fn is_malware(buf: &[u8]) -> Result<bool> {
|
||||
let config = config().await;
|
||||
if config.files.clamd_host.is_empty() {
|
||||
|
||||
@@ -1,16 +1,24 @@
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
|
||||
use axum::{middleware::from_fn_with_state, Router};
|
||||
use axum::{
|
||||
middleware::{from_fn, from_fn_with_state},
|
||||
Router,
|
||||
};
|
||||
|
||||
use axum_macros::FromRef;
|
||||
use revolt_database::{Database, DatabaseInfo};
|
||||
use revolt_ratelimits::axum as ratelimiter;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::{
|
||||
net::TcpListener,
|
||||
signal::unix::{signal, SignalKind},
|
||||
};
|
||||
use tower_http::catch_panic::CatchPanicLayer;
|
||||
use utoipa::{
|
||||
openapi::security::{ApiKey, ApiKeyValue, SecurityScheme},
|
||||
Modify, OpenApi,
|
||||
};
|
||||
use utoipa_scalar::{Scalar, Servable as ScalarServable};
|
||||
use tracing::info;
|
||||
|
||||
mod api;
|
||||
pub mod clamav;
|
||||
@@ -28,13 +36,12 @@ struct AppState {
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), std::io::Error> {
|
||||
// Configure logging and environment
|
||||
revolt_config::configure!(files);
|
||||
let _telemetry = stoat_otel::init(env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
|
||||
info!(version = env!("CARGO_PKG_VERSION"), awawa = true, "Starting the Stoat file server!");
|
||||
|
||||
// Wait for ClamAV
|
||||
revolt_config::config().await;
|
||||
clamav::init().await;
|
||||
|
||||
// Configure API schema
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
modifiers(&SecurityAddon),
|
||||
@@ -55,7 +62,7 @@ async fn main() -> Result<(), std::io::Error> {
|
||||
)
|
||||
),
|
||||
tags(
|
||||
// (name = "Files", description = "File uploads API")
|
||||
(name = "Files", description = "File uploads API")
|
||||
)
|
||||
)]
|
||||
struct ApiDoc;
|
||||
@@ -77,7 +84,6 @@ async fn main() -> Result<(), std::io::Error> {
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to the database
|
||||
let db = DatabaseInfo::Auto.connect().await.unwrap();
|
||||
let ratelimits = ratelimiter::RatelimitStorage::new(ratelimits::AutumnRatelimits);
|
||||
|
||||
@@ -86,7 +92,6 @@ async fn main() -> Result<(), std::io::Error> {
|
||||
ratelimit_storage: ratelimits,
|
||||
};
|
||||
|
||||
// Configure Axum and router
|
||||
let app = Router::new()
|
||||
.merge(Scalar::with_url("/scalar", ApiDoc::openapi()))
|
||||
.nest("/", api::router().await)
|
||||
@@ -95,10 +100,30 @@ async fn main() -> Result<(), std::io::Error> {
|
||||
state.clone(),
|
||||
ratelimiter::ratelimit_middleware,
|
||||
))
|
||||
.layer(CatchPanicLayer::custom(stoat_otel::middleware::axum::panic_response))
|
||||
.layer(from_fn(stoat_otel::middleware::axum::wide_events))
|
||||
.with_state(state);
|
||||
|
||||
// Configure TCP listener and bind
|
||||
let address = SocketAddr::from((Ipv4Addr::UNSPECIFIED, 14704));
|
||||
let listener = TcpListener::bind(&address).await?;
|
||||
axum::serve(listener, app.into_make_service_with_connect_info::<SocketAddr>()).await
|
||||
info!(address = %listener.local_addr()?, "Listening for requests 🍂");
|
||||
|
||||
axum::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(async {
|
||||
let mut terminate =
|
||||
signal(SignalKind::terminate()).expect("sigterm handler");
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
_ = terminate.recv() => {}
|
||||
}
|
||||
|
||||
info!("Service going offline, bye bye");
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::io::Cursor;
|
||||
|
||||
use crate::utils::apply_icc_profile;
|
||||
use image::{GenericImageView, ImageError, ImageReader};
|
||||
use image::{GenericImageView, ImageReader};
|
||||
use revolt_database::Metadata;
|
||||
use revolt_files::{image_size, is_animated, video_size};
|
||||
use tempfile::NamedTempFile;
|
||||
@@ -21,7 +21,6 @@ static SUPPORTED_IMAGE_MIME: [&str; 9] = [
|
||||
"image/webp",
|
||||
];
|
||||
|
||||
/// Generate metadata from file, using mime type as a hint
|
||||
pub fn generate_metadata(f: &NamedTempFile, mime_type: &str) -> Metadata {
|
||||
if SUPPORTED_IMAGE_MIME.contains(&mime_type) {
|
||||
image_size(f)
|
||||
@@ -67,7 +66,6 @@ pub fn generate_metadata(f: &NamedTempFile, mime_type: &str) -> Metadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Subroutine to ensure data isn't corrupted
|
||||
pub fn validate_from_metadata(
|
||||
reader: Cursor<Vec<u8>>,
|
||||
metadata: Metadata,
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Determine the mime type of the given temporary file and filename
|
||||
pub fn determine_mime_type(f: &mut NamedTempFile, buf: &[u8], file_name: &str) -> &'static str {
|
||||
// Force certain extensions into particular mime types
|
||||
if file_name.to_lowercase().ends_with(".apk") {
|
||||
return "application/vnd.android.package-archive";
|
||||
} else if file_name.to_lowercase().ends_with(".exe") {
|
||||
return "application/vnd.microsoft.portable-executable";
|
||||
}
|
||||
|
||||
// Use magic signatures to determine mime type
|
||||
let kind = infer::get_from_path(f.path()).expect("file read successfully");
|
||||
let mime_type = if let Some(kind) = kind {
|
||||
kind.mime_type()
|
||||
@@ -17,7 +14,6 @@ pub fn determine_mime_type(f: &mut NamedTempFile, buf: &[u8], file_name: &str) -
|
||||
"application/octet-stream"
|
||||
};
|
||||
|
||||
// See if the file is actually just plain Unicode/ASCII text
|
||||
if mime_type == "application/octet-stream" && simdutf8::basic::from_utf8(buf).is_ok() {
|
||||
if file_name.to_lowercase().ends_with(".svg") {
|
||||
return "image/svg+xml";
|
||||
|
||||
@@ -36,6 +36,7 @@ deps() {
|
||||
crates/core/result/src \
|
||||
crates/core/coalesced/src \
|
||||
crates/core/ratelimits/src \
|
||||
crates/common/otel/src \
|
||||
crates/services/autumn/src \
|
||||
crates/services/january/src \
|
||||
crates/services/gifbox/src \
|
||||
@@ -61,7 +62,8 @@ deps() {
|
||||
tee crates/core/presence/src/lib.rs |
|
||||
tee crates/core/result/src/lib.rs |
|
||||
tee crates/core/coalesced/src/lib.rs |
|
||||
tee crates/core/ratelimits/src/lib.rs
|
||||
tee crates/core/ratelimits/src/lib.rs |
|
||||
tee crates/common/otel/src/lib.rs
|
||||
|
||||
if [ -z "$TARGETARCH" ]; then
|
||||
cargo build -j "${CARGO_BUILD_JOBS:-10}" --locked --release
|
||||
@@ -85,7 +87,8 @@ apps() {
|
||||
crates/core/presence/src/lib.rs \
|
||||
crates/core/result/src/lib.rs \
|
||||
crates/core/coalesced/src/lib.rs \
|
||||
crates/core/ratelimits/src/lib.rs
|
||||
crates/core/ratelimits/src/lib.rs \
|
||||
crates/common/otel/src/lib.rs
|
||||
|
||||
if [ -z "$TARGETARCH" ]; then
|
||||
cargo build -j "${CARGO_BUILD_JOBS:-10}" --locked --release
|
||||
|
||||
Reference in New Issue
Block a user