From 81da79374acde777beb5bf59b218cd0cc84a6e07 Mon Sep 17 00:00:00 2001 From: Umputun Date: Tue, 5 Jun 2018 11:12:34 -0500 Subject: [PATCH] extends robots.txt to all public api --- app/rest/api/rest.go | 9 ++++++--- app/rest/api/rest_public_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/app/rest/api/rest.go b/app/rest/api/rest.go index e412aa0f..1da98274 100644 --- a/app/rest/api/rest.go +++ b/app/rest/api/rest.go @@ -164,10 +164,13 @@ func (s *Rest) routes() chi.Router { }) }) - router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(10, nil))). + router.With(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))). Get("/robots.txt", func(w http.ResponseWriter, r *http.Request) { - render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+ - "Allow: /api/v1/find\nAllow: /api/v1/last\n") + allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/img"} + for i := range allowed { + allowed[i] = "Allow: /api/v1" + allowed[i] + } + render.PlainText(w, r, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\n"+strings.Join(allowed, "\n")) }) // file server for static content from /web diff --git a/app/rest/api/rest_public_test.go b/app/rest/api/rest_public_test.go index fffdb54b..77a8b102 100644 --- a/app/rest/api/rest_public_test.go +++ b/app/rest/api/rest_public_test.go @@ -423,3 +423,13 @@ func TestRest_Info(t *testing.T) { _, code = get(t, ts.URL+"/api/v1/info?site=radio-t-no&url=https://radio-t.com/blah-no") assert.Equal(t, 400, code) } + +func TestRest_Robots(t *testing.T) { + srv, ts := prep(t) + assert.NotNil(t, srv) + defer cleanup(ts) + + body, code := get(t, ts.URL+"/robots.txt") + assert.Equal(t, 200, code) + assert.Equal(t, "User-agent: *\nDisallow: /auth/\nDisallow: /api/\nAllow: /api/v1/find\nAllow: /api/v1/last\nAllow: /api/v1/id\nAllow: /api/v1/count\nAllow: /api/v1/counts\nAllow: /api/v1/list\nAllow: /api/v1/config\nAllow: /api/v1/img", string(body)) +}