allow /find for robots

app/rest/api/migrator.go

@@ -103,7 +103,7 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 }
 
 // GET /export?site=site-id&secret=12345
-// exports all comments for siteID as json stream or gz file
+// exports all comments for siteID as gz file
 func (m *Migrator) exportCtrl(w http.ResponseWriter, r *http.Request) {
 
 	secret := r.URL.Query().Get("secret")

app/rest/api/rest.go

@@ -166,7 +166,7 @@ func (s *Rest) routes() chi.Router {
 
 	router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil))).
 		Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
-			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n")
+			render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\n")
 		})
 
 	// file server for static content from /web