From 50b3013f53b54de1975193698799f1a27e3b460f Mon Sep 17 00:00:00 2001 From: Lewis Crichton Date: Thu, 26 Oct 2023 23:00:39 +0100 Subject: [PATCH] feat: anonymous telemetry anonymous in this case meaning we can't trace it back to a particular user THIS IS UNTESTED AND HEAVILY WORK IN PROGRESS. --- .env.example | 2 ++ README.md | 35 +++++++++++++----------- globals/globals.go | 5 ++-- main.go | 66 +++++++++++++++++++++++++++++++++++++++++++-- routes/telemetry.go | 44 ++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 19 deletions(-) create mode 100644 routes/telemetry.go diff --git a/.env.example b/.env.example index 0d25193..04adc88 100644 --- a/.env.example +++ b/.env.example @@ -9,11 +9,13 @@ DISCORD_REDIRECT_URI= PEPPER_SECRETS= PEPPER_SETTINGS= +PEPPER_TELEMETRY= SIZE_LIMIT=33554432 ALLOWED_USERS= PROMETHEUS=false +TELEMETRY=false PROXY_HEADER= diff --git a/README.md b/README.md index bcf7cb5..6c01c51 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Backend + Vencord API ## Hosting + The API has a Docker Compose configuration, so software-wise you shouldn't need much more than just Docker. Docker is the official way of hosting the backend, and other setups (whilst technically supported) will be up to you to manage. @@ -9,26 +11,29 @@ up to you to manage. 1. Clone the repository 2. Copy `.env.example` to `.env` 3. Configure as necessary - - Port and host are irrelevant since it's running in a container, but you can change them if you wish. - - `REDIS_URI` should be changed to `redis:6379`. - - `ROOT_REDIRECT` should be changed to whatever you want the `/` of the API to be set to a different site, + +- Port and host are irrelevant since it's running in a container, but you can change them if you wish. +- `REDIS_URI` should be changed to `redis:6379`. +- `ROOT_REDIRECT` should be changed to whatever you want the `/` of the API to be set to a different site, like your own personal homepage. - - `DISCORD_*` should be configured with your Discord application. The redirect URI is `https:///v1/oauth/callback`. - - `PEPPER_*` should be unique values. These provide extra anonymity and make it more difficult to get user +- `DISCORD_*` should be configured with your Discord application. The redirect URI is `https:///v1/oauth/callback`. +- `PEPPER_*` should be unique values. These provide extra anonymity and make it more difficult to get user info. It [is recommended](https://stackoverflow.com/a/9622855) you use at least 32 bytes of randomness, e.g. through `openssl rand -hex 32`. - - `SIZE_LIMIT` is up to you, but should usually be left as default. This is for the settings sync and how +- `SIZE_LIMIT` is up to you, but should usually be left as default. This is for the settings sync and how much data a user can store. - - `ALLOWED_USERS` restricts what users can use this API instance for operations like settings sync. - - `PROMETHEUS` controls whether or not to expose the `/metrics` endpoint. - - `PROXY_HEADER` should be used if you're running it behind a reverse proxy or another service (i.e., Cloudflare). +- `ALLOWED_USERS` restricts what users can use this API instance for operations like settings sync. +- `PROMETHEUS` controls whether or not to expose the `/metrics` endpoint. +- `TELEMETRY` does not matter on a self-hosted instance, as Vencord will always point to the main instance for it. +- `PROXY_HEADER` should be used if you're running it behind a reverse proxy or another service (i.e., Cloudflare). + 4. Create a `docker-compose.override.yml` that maps your ports, like so: - ```yaml - services: - backend: - ports: - - HOST_PORT:8080 - ``` + ```yaml + services: + backend: + ports: + - HOST_PORT:8080 + ``` 5. `docker compose up -d` Please note that, due to mixed content requirements, you will need HTTPS on your self-hosted instance. diff --git a/globals/globals.go b/globals/globals.go index 3fe9a0c..8fcd15d 100644 --- a/globals/globals.go +++ b/globals/globals.go @@ -19,8 +19,9 @@ var ( DISCORD_CLIENT_SECRET = os.Getenv("DISCORD_CLIENT_SECRET") DISCORD_REDIRECT_URI = os.Getenv("DISCORD_REDIRECT_URI") - PEPPER_SETTINGS = os.Getenv("PEPPER_SETTINGS") - PEPPER_SECRETS = os.Getenv("PEPPER_SECRETS") + PEPPER_SETTINGS = os.Getenv("PEPPER_SETTINGS") + PEPPER_SECRETS = os.Getenv("PEPPER_SECRETS") + PEPPER_TELEMETRY = os.Getenv("PEPPER_TELEMETRY") SIZE_LIMIT int // initialised in main diff --git a/main.go b/main.go index 4010eb9..c727bdd 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ import ( "os" "strconv" "strings" + "time" "github.com/ansrivas/fiberprometheus/v2" "github.com/gofiber/fiber/v2" @@ -109,8 +110,8 @@ func main() { } app := fiber.New(fiber.Config{ - ProxyHeader: os.Getenv("PROXY_HEADER"), - }) + ProxyHeader: os.Getenv("PROXY_HEADER"), + }) g.RDB = redis.NewClient(&redis.Options{ Addr: g.REDIS_URI, @@ -135,6 +136,61 @@ func main() { return float64(count) }) + promUpdate := time.NewTicker(1 * time.Minute) + + activeUsers := promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "vencord_active_users", + Help: "The total number of active Vencord users", + }, []string{"version", "operating_system"}) + + pluginsInUse := promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "vencord_plugins_in_use", + Help: "The total number of current plugins in use by Vencord users", + }, []string{"name"}) + + go func() { + for { + <-promUpdate.C + + iter := g.RDB.Scan(context.Background(), 0, "telemetry:*", 0).Iterator() + + userStats := make(map[string]map[string]int64) + pluginStats := make(map[string]int64) + + for iter.Next(context.Background()) { + telemetryData := g.RDB.HMGet(context.Background(), iter.Val(), "version", "plugins", "operatingSystem").Val() + + version := telemetryData[0].(string) + plugins := telemetryData[1].([]string) + operatingSystem := telemetryData[2].(string) + + if userStats[version] == nil { + userStats[version] = make(map[string]int64) + } + + userStats[version][operatingSystem]++ + + for _, plugin := range plugins { + pluginStats[plugin]++ + } + } + + if err := iter.Err(); err != nil { + panic(err) + } + + for version, operatingSystems := range userStats { + for operatingSystem, count := range operatingSystems { + activeUsers.WithLabelValues(version, operatingSystem).Set(float64(count)) + } + } + + for plugin, count := range pluginStats { + pluginsInUse.WithLabelValues(plugin).Set(float64(count)) + } + } + }() + prometheus := fiberprometheus.New("vencord") prometheus.RegisterAt(app, "/metrics") app.Use(prometheus.Middleware) @@ -164,6 +220,12 @@ func main() { app.Delete("/v1", requireAuth, routes.DELETE) // #endregion + // #region telemetry + if os.Getenv("TELEMETRY") == "true" { + app.Post("/v1/telemetry", routes.POSTTelemetry) + } + // #endregion + app.Get("/v1", routes.GET) app.Get("/", func(c *fiber.Ctx) error { diff --git a/routes/telemetry.go b/routes/telemetry.go new file mode 100644 index 0000000..f284386 --- /dev/null +++ b/routes/telemetry.go @@ -0,0 +1,44 @@ +package routes + +import ( + "time" + + "github.com/gofiber/fiber/v2" + + g "github.com/vencord/backend/globals" + "github.com/vencord/backend/util" +) + +type TelemetryBody struct { + Plugins []string `json:"plugins"` + Version string `json:"version"` + OperatingSystem string `json:"operatingSystem"` +} + +func POSTTelemetry(c *fiber.Ctx) error { + telemetry := new(TelemetryBody) + + if err := c.BodyParser(telemetry); err != nil { + return err + } + + telemetryId := util.Hash(g.PEPPER_TELEMETRY + c.IP()) + + _, err := g.RDB.HSet(c.Context(), "telemetry:"+telemetryId, map[string]interface{}{ + "plugins": telemetry.Plugins, + "version": telemetry.Version, + "operatingSystem": telemetry.OperatingSystem, + }).Result() + + if err != nil { + panic(err) + } + + _, err = g.RDB.Expire(c.Context(), "telemetry:"+telemetryId, 3*24*time.Hour).Result() + + if err != nil { + panic(err) + } + + return c.SendStatus(204) +}