fix(app-proxy): ask search engines not to index the web app (#3512)

fluxer_app_proxy/src/routes/mod.rs

@@ -29,6 +29,7 @@ use tower_http::{
 };
 
 const PERMISSIONS_POLICY_VALUE: &str = "accelerometer=(), camera=(self), ch-dpr=(self), ch-save-data=(self), ch-viewport-width=(self), ch-width=(self), geolocation=(), gyroscope=(), magnetometer=(), microphone=(self), payment=(), usb=()";
+const X_ROBOTS_TAG_VALUE: &str = "noindex, nofollow";
 
 pub fn build_router(state: AppState) -> Router {
     Router::new()
@@ -73,6 +74,10 @@ async fn security_headers_middleware(request: Request, next: Next) -> Response {
 
 fn set_security_headers(headers: &mut axum::http::HeaderMap) {
     fluxer_common::server::set_baseline_security_headers(headers, PERMISSIONS_POLICY_VALUE);
+    headers.insert(
+        HeaderName::from_static("x-robots-tag"),
+        HeaderValue::from_static(X_ROBOTS_TAG_VALUE),
+    );
 }
 
 async fn cache_headers_middleware(request: Request, next: Next) -> Response {

fluxer_app_proxy/src/routes/spa_static.rs

@@ -449,6 +449,38 @@ mod tests {
         std::fs::remove_dir_all(root).unwrap();
     }
 
+    #[tokio::test]
+    async fn every_response_asks_search_engines_not_to_index_it() {
+        let root = static_dir_with_generated_files();
+        std::fs::write(root.join("robots.txt"), "User-agent: *\nAllow: /\n").unwrap();
+
+        for path in [
+            "/",
+            "/login",
+            "/channels/@me",
+            "/invite/abc",
+            "/gift/abc",
+            "/robots.txt",
+            "/manifest.json",
+            "/version.json",
+            "/_health",
+            "/web/missing.png",
+        ] {
+            let response = fetch(state_serving(&root, false), path).await;
+            assert_eq!(
+                response
+                    .headers()
+                    .get("x-robots-tag")
+                    .and_then(|value| value.to_str().ok()),
+                Some("noindex, nofollow"),
+                "{path} answered {} without a noindex directive",
+                response.status()
+            );
+        }
+
+        std::fs::remove_dir_all(root).unwrap();
+    }
+
     const GENERATED_BROWSERCONFIG: &str = r#"<?xml version="1.0" encoding="utf-8"?>
 <browserconfig>
   <msapplication>