feat: anonymous telemetry

anonymous in this case meaning we can't trace it back to a particular user THIS IS UNTESTED AND HEAVILY WORK IN PROGRESS.
2023-10-26 23:00:39 +01:00
parent 819664f801
commit 50b3013f53
5 changed files with 133 additions and 19 deletions
@@ -9,11 +9,13 @@ DISCORD_REDIRECT_URI=

 PEPPER_SECRETS=
 PEPPER_SETTINGS=
+PEPPER_TELEMETRY=

 SIZE_LIMIT=33554432

 ALLOWED_USERS=

 PROMETHEUS=false
+TELEMETRY=false

 PROXY_HEADER=
@@ -1,7 +1,9 @@
 # Backend
+
 Vencord API

 ## Hosting
+
 The API has a Docker Compose configuration, so software-wise you shouldn't need much more than just Docker.
 Docker is the official way of hosting the backend, and other setups (whilst technically supported) will be
 up to you to manage.
@@ -9,26 +11,29 @@ up to you to manage.
 1. Clone the repository
 2. Copy `.env.example` to `.env`
 3. Configure as necessary
-  - Port and host are irrelevant since it's running in a container, but you can change them if you wish.
-  - `REDIS_URI` should be changed to `redis:6379`.
-  - `ROOT_REDIRECT` should be changed to whatever you want the `/` of the API to be set to a different site,
+
+-   Port and host are irrelevant since it's running in a container, but you can change them if you wish.
+-   `REDIS_URI` should be changed to `redis:6379`.
+-   `ROOT_REDIRECT` should be changed to whatever you want the `/` of the API to be set to a different site,
    like your own personal homepage.
-  - `DISCORD_*` should be configured with your Discord application. The redirect URI is `https://<yourdomain>/v1/oauth/callback`.
-  - `PEPPER_*` should be unique values. These provide extra anonymity and make it more difficult to get user
+-   `DISCORD_*` should be configured with your Discord application. The redirect URI is `https://<yourdomain>/v1/oauth/callback`.
+-   `PEPPER_*` should be unique values. These provide extra anonymity and make it more difficult to get user
    info. It [is recommended](https://stackoverflow.com/a/9622855) you use at least 32 bytes of randomness, e.g. through
    `openssl rand -hex 32`.
-  - `SIZE_LIMIT` is up to you, but should usually be left as default. This is for the settings sync and how
+-   `SIZE_LIMIT` is up to you, but should usually be left as default. This is for the settings sync and how
    much data a user can store.
-  - `ALLOWED_USERS` restricts what users can use this API instance for operations like settings sync.
-  - `PROMETHEUS` controls whether or not to expose the `/metrics` endpoint.
-  - `PROXY_HEADER` should be used if you're running it behind a reverse proxy or another service (i.e., Cloudflare).
+-   `ALLOWED_USERS` restricts what users can use this API instance for operations like settings sync.
+-   `PROMETHEUS` controls whether or not to expose the `/metrics` endpoint.
+-   `TELEMETRY` does not matter on a self-hosted instance, as Vencord will always point to the main instance for it.
+-   `PROXY_HEADER` should be used if you're running it behind a reverse proxy or another service (i.e., Cloudflare).
+
 4. Create a `docker-compose.override.yml` that maps your ports, like so:
-   ```yaml
-   services:
-     backend:
-       ports:
-         - HOST_PORT:8080
-   ```
+    ```yaml
+    services:
+        backend:
+            ports:
+                - HOST_PORT:8080
+    ```
 5. `docker compose up -d`

 Please note that, due to mixed content requirements, you will need HTTPS on your self-hosted instance.
@@ -19,8 +19,9 @@ var (
 	DISCORD_CLIENT_SECRET = os.Getenv("DISCORD_CLIENT_SECRET")
 	DISCORD_REDIRECT_URI  = os.Getenv("DISCORD_REDIRECT_URI")

-	PEPPER_SETTINGS = os.Getenv("PEPPER_SETTINGS")
-	PEPPER_SECRETS  = os.Getenv("PEPPER_SECRETS")
+	PEPPER_SETTINGS  = os.Getenv("PEPPER_SETTINGS")
+	PEPPER_SECRETS   = os.Getenv("PEPPER_SECRETS")
+	PEPPER_TELEMETRY = os.Getenv("PEPPER_TELEMETRY")

 	SIZE_LIMIT int // initialised in main

@@ -22,6 +22,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"

 	"github.com/ansrivas/fiberprometheus/v2"
 	"github.com/gofiber/fiber/v2"
@@ -109,8 +110,8 @@ func main() {
 	}

 	app := fiber.New(fiber.Config{
-        ProxyHeader: os.Getenv("PROXY_HEADER"),
-    })
+		ProxyHeader: os.Getenv("PROXY_HEADER"),
+	})

 	g.RDB = redis.NewClient(&redis.Options{
 		Addr: g.REDIS_URI,
@@ -135,6 +136,61 @@ func main() {
 			return float64(count)
 		})

+        promUpdate := time.NewTicker(1 * time.Minute)
+
+        activeUsers := promauto.NewGaugeVec(prometheus.GaugeOpts{
+            Name: "vencord_active_users",
+            Help: "The total number of active Vencord users",
+        }, []string{"version", "operating_system"})
+
+        pluginsInUse := promauto.NewGaugeVec(prometheus.GaugeOpts{
+            Name: "vencord_plugins_in_use",
+            Help: "The total number of current plugins in use by Vencord users",
+        }, []string{"name"})
+
+        go func() {
+            for {
+                <-promUpdate.C
+
+                iter := g.RDB.Scan(context.Background(), 0, "telemetry:*", 0).Iterator()
+
+                userStats := make(map[string]map[string]int64)
+                pluginStats := make(map[string]int64)
+
+				for iter.Next(context.Background()) {
+					telemetryData := g.RDB.HMGet(context.Background(), iter.Val(), "version", "plugins", "operatingSystem").Val()
+
+                    version := telemetryData[0].(string)
+                    plugins := telemetryData[1].([]string)
+                    operatingSystem := telemetryData[2].(string)
+
+                    if userStats[version] == nil {
+                        userStats[version] = make(map[string]int64)
+                    }
+
+                    userStats[version][operatingSystem]++
+
+                    for _, plugin := range plugins {
+                        pluginStats[plugin]++
+                    }
+				}
+
+				if err := iter.Err(); err != nil {
+					panic(err)
+				}
+
+                for version, operatingSystems := range userStats {
+                    for operatingSystem, count := range operatingSystems {
+                        activeUsers.WithLabelValues(version, operatingSystem).Set(float64(count))
+                    }
+                }
+
+                for plugin, count := range pluginStats {
+                    pluginsInUse.WithLabelValues(plugin).Set(float64(count))
+                }
+            }
+        }()
+
 		prometheus := fiberprometheus.New("vencord")
 		prometheus.RegisterAt(app, "/metrics")
 		app.Use(prometheus.Middleware)
@@ -164,6 +220,12 @@ func main() {
 	app.Delete("/v1", requireAuth, routes.DELETE)
 	// #endregion

+	// #region telemetry
+	if os.Getenv("TELEMETRY") == "true" {
+		app.Post("/v1/telemetry", routes.POSTTelemetry)
+	}
+	// #endregion
+
 	app.Get("/v1", routes.GET)

 	app.Get("/", func(c *fiber.Ctx) error {
@@ -0,0 +1,44 @@
+package routes
+
+import (
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+
+	g "github.com/vencord/backend/globals"
+	"github.com/vencord/backend/util"
+)
+
+type TelemetryBody struct {
+	Plugins         []string `json:"plugins"`
+	Version         string   `json:"version"`
+	OperatingSystem string   `json:"operatingSystem"`
+}
+
+func POSTTelemetry(c *fiber.Ctx) error {
+	telemetry := new(TelemetryBody)
+
+	if err := c.BodyParser(telemetry); err != nil {
+		return err
+	}
+
+	telemetryId := util.Hash(g.PEPPER_TELEMETRY + c.IP())
+
+	_, err := g.RDB.HSet(c.Context(), "telemetry:"+telemetryId, map[string]interface{}{
+		"plugins":         telemetry.Plugins,
+		"version":         telemetry.Version,
+		"operatingSystem": telemetry.OperatingSystem,
+	}).Result()
+
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = g.RDB.Expire(c.Context(), "telemetry:"+telemetryId, 3*24*time.Hour).Result()
+
+	if err != nil {
+		panic(err)
+	}
+
+	return c.SendStatus(204)
+}