robots.txt

febs-gateway/src/main/java/cc/mrbird/febs/gateway/controller/RobotsController.java

@@ -1,23 +0,0 @@
-package cc.mrbird.febs.gateway.controller;
-
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RestController;
-import reactor.core.publisher.Mono;
-
-/**
- * @author MrBird
- */
-@RestController
-public class RobotsController {
-
-    @GetMapping("/robots.txt")
-    public Mono<String> robotsTxt() {
-        StringBuilder builder = new StringBuilder();
-        String lineSeparator = System.getProperty("line.separator", "\n");
-        builder.append("User-agent: Baiduspider").append(lineSeparator);
-        builder.append("Disallow: ").append("/").append(lineSeparator);
-        builder.append("User-agent: Googlebot").append(lineSeparator);
-        builder.append("Disallow: ").append("/").append(lineSeparator);
-        return Mono.just(builder.toString());
-    }
-}

febs-gateway/src/main/resources/static/robots.txt

@@ -0,0 +1,8 @@
+# robots.txt
+User-agent: Googlebot
+Disallow: 
+User-agent: baiduspider
+Disallow: 
+User-agent: *
+Disallow: 
+Crawl-delay: 60