Commit aca0cff3 silenced the path-traversal, SSRF and XSS taint rules
project-wide as "false positives" while fixing image-proxy SSRF. With
the path-traversal and TitleExtractor SSRF gaps now closed, restore the
rules so future regressions get flagged. The four genuine false positives
that remain (image proxy http.NewRequest, QR png Write, two RSS XML
Writes) get individual //nolint:gosec comments naming the reason.
524 lines
18 KiB
Go
524 lines
18 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha1" // nolint
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
cache "github.com/go-pkgz/lcw/v2"
|
|
log "github.com/go-pkgz/lgr"
|
|
R "github.com/go-pkgz/rest"
|
|
"github.com/google/uuid"
|
|
"github.com/skip2/go-qrcode"
|
|
|
|
"github.com/umputun/remark42/backend/app/rest"
|
|
"github.com/umputun/remark42/backend/app/store"
|
|
"github.com/umputun/remark42/backend/app/store/image"
|
|
"github.com/umputun/remark42/backend/app/store/service"
|
|
)
|
|
|
|
type public struct {
|
|
dataService pubStore
|
|
cache LoadingCache
|
|
readOnlyAge int
|
|
commentFormatter *store.CommentFormatter
|
|
imageService *image.Service
|
|
}
|
|
|
|
type pubStore interface {
|
|
Create(comment store.Comment) (commentID string, err error)
|
|
Get(locator store.Locator, commentID string, user store.User) (store.Comment, error)
|
|
FindSince(locator store.Locator, sort string, user store.User, since time.Time) ([]store.Comment, error)
|
|
Last(siteID string, limit int, since time.Time, user store.User) ([]store.Comment, error)
|
|
User(siteID, userID string, limit, skip int, user store.User) ([]store.Comment, error)
|
|
UserCount(siteID, userID string) (int, error)
|
|
Count(locator store.Locator) (int, error)
|
|
List(siteID string, limit, skip int) ([]store.PostInfo, error)
|
|
Info(locator store.Locator, readonlyAge int) (store.PostInfo, error)
|
|
|
|
ValidateComment(c *store.Comment) error
|
|
IsReadOnly(locator store.Locator) bool
|
|
Counts(siteID string, postIDs []string) ([]store.PostInfo, error)
|
|
}
|
|
|
|
// GET /find?site=siteID&url=post-url&format=[tree|plain]&sort=[+/-time|+/-score|+/-controversy]&view=[user|all]&since=unix_ts_msec&limit=100&offset_id={id}
|
|
// find comments for given post. Returns in tree or plain formats, sorted.
|
|
//
|
|
// When `url` parameter is not set (e.g. request is for site-wide comments), does not return deleted comments.
|
|
//
|
|
// When `limit` is set, first {limit} comments are returned. When `offset_id` is set, comments are returned starting
|
|
// after the comment with the given id.
|
|
// format="tree" limits comments by top-level comments and all their replies,
|
|
// and never returns parent comment with only part of replies.
|
|
//
|
|
// `count` in the response refers to total number of non-deleted comments,
|
|
// `count_left` to amount of comments left to be returned _including deleted_.
|
|
func (s *public) findCommentsCtrl(w http.ResponseWriter, r *http.Request) {
|
|
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
|
|
sort := r.URL.Query().Get("sort")
|
|
if strings.HasPrefix(sort, " ") { // restore + replaced by " "
|
|
sort = "+" + sort[1:]
|
|
}
|
|
|
|
view := r.URL.Query().Get("view")
|
|
since, err := s.parseSince(r)
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't parse since", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
format := r.URL.Query().Get("format")
|
|
if format == "tree" {
|
|
since = time.Time{} // since doesn't make sense for tree
|
|
}
|
|
|
|
limitParam := r.URL.Query().Get("limit")
|
|
var limit int
|
|
if limitParam != "" {
|
|
if limit, err = strconv.Atoi(limitParam); err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "bad limit value", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
}
|
|
|
|
offsetID := r.URL.Query().Get("offset_id")
|
|
if offsetID != "" {
|
|
if _, err = uuid.Parse(offsetID); err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "bad offset_id value", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
}
|
|
|
|
log.Printf("[DEBUG] get comments for %+v, sort %s, format %s, since %v, limit %d, offset %s", locator, sort, format, since, limit, offsetID)
|
|
|
|
key := cache.NewKey(locator.SiteID).ID(URLKeyWithUser(r)).Scopes(locator.SiteID, locator.URL)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
comments, e := s.dataService.FindSince(locator, sort, rest.GetUserOrEmpty(r), since)
|
|
if e != nil {
|
|
comments = []store.Comment{} // error should clear comments and continue for post info
|
|
}
|
|
comments = s.applyView(comments, view)
|
|
|
|
var commentsInfo store.PostInfo
|
|
if info, ee := s.dataService.Info(locator, s.readOnlyAge); ee == nil {
|
|
commentsInfo = info
|
|
}
|
|
|
|
if !since.IsZero() { // if since is set, number of comments can be different from total in the DB
|
|
commentsInfo.Count = 0
|
|
for _, c := range comments {
|
|
if !c.Deleted {
|
|
commentsInfo.Count++
|
|
}
|
|
}
|
|
}
|
|
|
|
// post might be readonly without any comments, Info call will fail then and ReadOnly flag should be checked separately
|
|
if !commentsInfo.ReadOnly && locator.URL != "" && s.dataService.IsReadOnly(locator) {
|
|
commentsInfo.ReadOnly = true
|
|
}
|
|
|
|
var b []byte
|
|
switch format {
|
|
case "tree":
|
|
withInfo := treeWithInfo{Tree: service.MakeTree(comments, sort, limit, offsetID), Info: commentsInfo}
|
|
withInfo.Info.CountLeft = withInfo.CountLeft()
|
|
withInfo.Info.LastComment = withInfo.LastComment()
|
|
if withInfo.Nodes == nil { // eliminate json nil serialization
|
|
withInfo.Nodes = []*service.Node{}
|
|
}
|
|
b, e = encodeJSONWithHTML(withInfo)
|
|
default:
|
|
if limit > 0 || offsetID != "" {
|
|
comments, commentsInfo.CountLeft = limitComments(comments, limit, offsetID)
|
|
}
|
|
if limit > 0 && len(comments) > 0 {
|
|
commentsInfo.LastComment = comments[len(comments)-1].ID
|
|
}
|
|
withInfo := commentsWithInfo{Comments: comments, Info: commentsInfo}
|
|
b, e = encodeJSONWithHTML(withInfo)
|
|
}
|
|
return b, e
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't find comments", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render comments for post %+v", locator)
|
|
}
|
|
}
|
|
|
|
// GET /info?site=siteID&url=post-url - get info about the post
|
|
func (s *public) infoCtrl(w http.ResponseWriter, r *http.Request) {
|
|
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
|
|
|
|
key := cache.NewKey(locator.SiteID).ID(URLKey(r)).Scopes(locator.SiteID, locator.URL)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
info, e := s.dataService.Info(locator, s.readOnlyAge)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
return encodeJSONWithHTML(info)
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get post info", rest.ErrPostNotFound)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render info for post %+v", locator)
|
|
}
|
|
}
|
|
|
|
// GET /last/{limit}?site=siteID&since=unix_ts_msec - last comments for the siteID, across all posts, sorted by time, optionally
|
|
// limited with "since" param
|
|
func (s *public) lastCommentsCtrl(w http.ResponseWriter, r *http.Request) {
|
|
siteID := r.URL.Query().Get("site")
|
|
log.Printf("[DEBUG] get last comments for %s", siteID)
|
|
|
|
limit, err := strconv.Atoi(chi.URLParam(r, "limit"))
|
|
if err != nil {
|
|
limit = 0
|
|
}
|
|
|
|
sinceTime, err := s.parseSince(r)
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't translate since parameter", rest.ErrDecode)
|
|
return
|
|
}
|
|
|
|
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(lastCommentsScope)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
comments, e := s.dataService.Last(siteID, limit, sinceTime, rest.GetUserOrEmpty(r))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
// filter deleted from last comments view. Blocked marked as deleted and will sneak in without
|
|
filterDeleted := filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
|
|
return encodeJSONWithHTML(filterDeleted)
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't get last comments", rest.ErrInternal)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render last comments for site %s", siteID)
|
|
}
|
|
}
|
|
|
|
// GET /id/{id}?site=siteID&url=post-url - gets a comment by id
|
|
func (s *public) commentByIDCtrl(w http.ResponseWriter, r *http.Request) {
|
|
id := chi.URLParam(r, "id")
|
|
siteID := r.URL.Query().Get("site")
|
|
url := r.URL.Query().Get("url")
|
|
|
|
log.Printf("[DEBUG] get comments by id %s, %s %s", id, siteID, url)
|
|
|
|
comment, err := s.dataService.Get(store.Locator{SiteID: siteID, URL: url}, id, rest.GetUserOrEmpty(r))
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by id", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONWithHTML(w, r, comment); err != nil {
|
|
log.Printf("[WARN] can't render last comments for url=%s, id=%s", url, id)
|
|
}
|
|
}
|
|
|
|
// GET /comments?site=siteID&user=id&limit=123&skip=10 - returns comments for given userID
|
|
func (s *public) findUserCommentsCtrl(w http.ResponseWriter, r *http.Request) {
|
|
userID := r.URL.Query().Get("user")
|
|
siteID := r.URL.Query().Get("site")
|
|
|
|
getNumWithDef := func(key string) int {
|
|
res, err := strconv.Atoi(r.URL.Query().Get(key))
|
|
if err != nil {
|
|
res = 0
|
|
}
|
|
return res
|
|
}
|
|
|
|
limit, skip := getNumWithDef("limit"), getNumWithDef("skip")
|
|
|
|
resp := struct {
|
|
Comments []store.Comment `json:"comments"`
|
|
Count int `json:"count"`
|
|
}{}
|
|
|
|
log.Printf("[DEBUG] get comments for userID %s, %s", userID, siteID)
|
|
|
|
key := cache.NewKey(siteID).ID(URLKeyWithUser(r)).Scopes(userID, siteID)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
comments, e := s.dataService.User(siteID, userID, limit, skip, rest.GetUserOrEmpty(r))
|
|
if e != nil {
|
|
if strings.Contains(e.Error(), "no comments for user") { // store returns this error when no comments found
|
|
resp.Comments, resp.Count = []store.Comment{}, 0
|
|
return encodeJSONWithHTML(resp)
|
|
}
|
|
return nil, e
|
|
}
|
|
comments = filterComments(comments, func(c store.Comment) bool { return !c.Deleted })
|
|
count, e := s.dataService.UserCount(siteID, userID)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
resp.Comments, resp.Count = comments, count
|
|
return encodeJSONWithHTML(resp)
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get comment by user id", rest.ErrCommentNotFound)
|
|
return
|
|
}
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render found comments for user %s", userID)
|
|
}
|
|
}
|
|
|
|
// GET /count?site=siteID&url=post-url - get number of comments for given post
|
|
func (s *public) countCtrl(w http.ResponseWriter, r *http.Request) {
|
|
locator := store.Locator{SiteID: r.URL.Query().Get("site"), URL: r.URL.Query().Get("url")}
|
|
count, err := s.dataService.Count(locator)
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get count", rest.ErrPostNotFound)
|
|
return
|
|
}
|
|
R.RenderJSON(w, R.JSON{"count": count, "locator": locator})
|
|
}
|
|
|
|
// POST /counts?site=siteID - get number of comments for posts from post body
|
|
func (s *public) countMultiCtrl(w http.ResponseWriter, r *http.Request) {
|
|
const countBodyLimit int64 = 1024 * 128 // count request can be big for some site because it lists all urls
|
|
siteID := r.URL.Query().Get("site")
|
|
posts := []string{}
|
|
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, countBodyLimit)).Decode(&posts); err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of posts from request", rest.ErrSiteNotFound)
|
|
return
|
|
}
|
|
|
|
// key could be long for multiple posts, make it sha1
|
|
k := URLKey(r) + strings.Join(posts, ",")
|
|
h := sha1.Sum([]byte(k)) // nolint
|
|
sha := base64.URLEncoding.EncodeToString(h[:])
|
|
|
|
key := cache.NewKey(siteID).ID(sha).Scopes(siteID)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
counts, e := s.dataService.Counts(siteID, posts)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
return encodeJSONWithHTML(counts)
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get counts for "+siteID, rest.ErrSiteNotFound)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render comments counters site %s", siteID)
|
|
}
|
|
}
|
|
|
|
// GET /list?site=siteID&limit=50&skip=10 - list posts with comments
|
|
func (s *public) listCtrl(w http.ResponseWriter, r *http.Request) {
|
|
siteID := r.URL.Query().Get("site")
|
|
limit, skip := 0, 0
|
|
|
|
if v, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil {
|
|
limit = v
|
|
}
|
|
if v, err := strconv.Atoi(r.URL.Query().Get("skip")); err == nil {
|
|
skip = v
|
|
}
|
|
|
|
key := cache.NewKey(siteID).ID(URLKey(r)).Scopes(siteID)
|
|
data, err := s.cache.Get(key, func() ([]byte, error) {
|
|
posts, e := s.dataService.List(siteID, limit, skip)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
return encodeJSONWithHTML(posts)
|
|
})
|
|
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't get list of comments for "+siteID, rest.ErrSiteNotFound)
|
|
return
|
|
}
|
|
|
|
if err = R.RenderJSONFromBytes(w, r, data); err != nil {
|
|
log.Printf("[WARN] can't render posts list for site %s", siteID)
|
|
}
|
|
}
|
|
|
|
// safePictureSegment reports whether seg is acceptable as a path segment in
|
|
// the picture URL (no traversal markers, no path separators, no control
|
|
// characters). Picture IDs are server-generated hashes plus a known
|
|
// extension, so any value carrying these characters is hostile and must be
|
|
// rejected before reaching the store. Rejecting controls (CR, LF, TAB, NUL,
|
|
// etc.) also closes a log-injection vector since the rejected segment is
|
|
// echoed into the access log.
|
|
func safePictureSegment(seg string) bool {
|
|
if seg == "" || seg == "." {
|
|
return false
|
|
}
|
|
if strings.ContainsAny(seg, "/\\") {
|
|
return false
|
|
}
|
|
if strings.Contains(seg, "..") { // also covers seg == ".."
|
|
return false
|
|
}
|
|
for _, r := range seg {
|
|
if unicode.IsControl(r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// GET /picture/{user}/{id} - get picture
|
|
func (s *public) loadPictureCtrl(w http.ResponseWriter, r *http.Request) {
|
|
user, imgID := chi.URLParam(r, "user"), chi.URLParam(r, "id")
|
|
if user == "" || imgID == "" || !safePictureSegment(user) || !safePictureSegment(imgID) {
|
|
log.Printf("[WARN] rejected picture request with unsafe id segments user=%q id=%q", user, imgID)
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("invalid picture id"), "invalid picture id", rest.ErrAssetNotFound)
|
|
return
|
|
}
|
|
id := user + "/" + imgID
|
|
img, err := s.imageService.Load(id)
|
|
if err != nil {
|
|
log.Printf("[WARN] can't load image %s: %v", id, err)
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("image not found"), "can't get image", rest.ErrAssetNotFound)
|
|
return
|
|
}
|
|
// enforce client-side caching
|
|
etag := `"` + id + `"`
|
|
w.Header().Set("Etag", etag)
|
|
w.Header().Set("Cache-Control", "max-age=604800") // 7 days
|
|
if match := r.Header.Get("If-None-Match"); match != "" {
|
|
if strings.Contains(match, etag) {
|
|
w.WriteHeader(http.StatusNotModified)
|
|
return
|
|
}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", s.imageService.ImgContentType(img))
|
|
w.Header().Set("Content-Length", strconv.Itoa(len(img)))
|
|
w.WriteHeader(http.StatusOK)
|
|
if _, err = io.Copy(w, bytes.NewReader(img)); err != nil {
|
|
log.Printf("[WARN] can't send response to %s, %s", r.RemoteAddr, err)
|
|
}
|
|
}
|
|
|
|
// GET /robots.txt
|
|
func (s *public) robotsCtrl(w http.ResponseWriter, _ *http.Request) {
|
|
allowed := []string{"/find", "/last", "/id", "/count", "/counts", "/list", "/config", "/user",
|
|
"/img", "/avatar", "/picture"}
|
|
for i := range allowed {
|
|
allowed[i] = "Allow: /api/v1" + allowed[i]
|
|
}
|
|
responseText := fmt.Sprintf("User-agent: *\nDisallow: /auth/\nDisallow: /api/\n%s\n", strings.Join(allowed, "\n"))
|
|
rest.PlainTextResponse(w, http.StatusOK, responseText)
|
|
}
|
|
|
|
// GET /qr/telegram - generates QR for provided URL, used for Telegram auth and notifications subscription. The first
|
|
// step of both is the user opening a link to Telegram and writing bot a message, and the user might try to log in
|
|
// from a computer but have Telegram installed only on the mobile device.
|
|
// Provided text should start with https://t.me/.
|
|
func (s *public) telegramQrCtrl(w http.ResponseWriter, r *http.Request) {
|
|
text := r.URL.Query().Get("url")
|
|
if text == "" {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("missing parameter"), "text parameter is required", rest.ErrInternal)
|
|
return
|
|
}
|
|
|
|
if !strings.HasPrefix(text, "https://t.me/") {
|
|
rest.SendErrorJSON(w, r, http.StatusBadRequest, fmt.Errorf("wrong parameter"), "text parameter should start with https://t.me/", rest.ErrInternal)
|
|
return
|
|
}
|
|
|
|
q, err := qrcode.New(text, qrcode.Medium)
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't generate QR", rest.ErrInternal)
|
|
return
|
|
}
|
|
q.DisableBorder = true
|
|
png, err := q.PNG(256)
|
|
if err != nil {
|
|
rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't generate QR", rest.ErrInternal)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "image/png")
|
|
if _, err = w.Write(png); err != nil { //nolint:gosec // png bytes from go-qrcode, not HTML
|
|
log.Printf("[WARN] can't render qr, %v", err)
|
|
}
|
|
}
|
|
|
|
func (s *public) applyView(comments []store.Comment, view string) []store.Comment {
|
|
if strings.EqualFold(view, "user") {
|
|
projection := make([]store.Comment, 0, len(comments))
|
|
for _, c := range comments {
|
|
if c.Deleted {
|
|
continue
|
|
}
|
|
p := store.Comment{
|
|
ID: c.ID,
|
|
User: c.User,
|
|
}
|
|
projection = append(projection, p)
|
|
}
|
|
return projection
|
|
}
|
|
|
|
return comments
|
|
}
|
|
|
|
func (s *public) parseSince(r *http.Request) (time.Time, error) {
|
|
sinceTS := time.Time{}
|
|
if since := r.URL.Query().Get("since"); since != "" {
|
|
unixTS, e := strconv.ParseInt(since, 10, 64)
|
|
if e != nil {
|
|
return time.Time{}, fmt.Errorf("can't translate since parameter: %w", e)
|
|
}
|
|
sinceTS = time.Unix(unixTS/1000, 1000000*(unixTS%1000)) // since param in msec timestamp
|
|
}
|
|
return sinceTS, nil
|
|
}
|
|
|
|
// limitComments returns limited list of comments and count of comments left after limit.
|
|
// If offsetID is provided, the list will be sliced starting from the comment with this ID.
|
|
// If offsetID is not found, the full list will be returned.
|
|
// It's used for only "
|
|
func limitComments(c []store.Comment, limit int, offsetID string) (comments []store.Comment, countLeft int) {
|
|
if offsetID != "" {
|
|
for i, comment := range c {
|
|
if comment.ID == offsetID {
|
|
c = c[i+1:]
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if limit > 0 && len(c) > limit {
|
|
countLeft = len(c) - limit
|
|
c = c[:limit]
|
|
}
|
|
|
|
return c, countLeft
|
|
}
|