Limit reading bytes instead of ReadAll (#35928)

Author: wxiaoguang

modules/actions/workflows.go

@@ -5,14 +5,14 @@ package actions
 
 import (
 	"bytes"
-	"io"
 	"slices"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/glob"
 	"code.gitea.io/gitea/modules/log"
 	api "code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
 
 	"github.com/nektos/act/pkg/jobparser"
@@ -77,7 +77,7 @@ func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	content, err := io.ReadAll(f)
+	content, err := util.ReadWithLimit(f, 1024*1024)
 	_ = f.Close()
 	if err != nil {
 		return nil, err

modules/issue/template/unmarshal.go

@@ -5,7 +5,6 @@ package template
 
 import (
 	"fmt"
-	"io"
 	"path"
 	"strconv"
 
@@ -76,7 +75,7 @@ func unmarshalFromEntry(entry *git.TreeEntry, filename string) (*api.IssueTempla
 	}
 	defer r.Close()
 
-	content, err := io.ReadAll(r)
+	content, err := util.ReadWithLimit(r, 1024*1024)
 	if err != nil {
 		return nil, fmt.Errorf("read all: %w", err)
 	}

modules/packages/nuget/metadata.go

@@ -216,7 +216,7 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) {
 	if p.Metadata.Readme != "" {
 		f, err := archive.Open(p.Metadata.Readme)
 		if err == nil {
-			buf, _ := io.ReadAll(f)
+			buf, _ := util.ReadWithLimit(f, 1024*1024)
 			m.Readme = string(buf)
 			_ = f.Close()
 		}

modules/packages/pub/metadata.go

@@ -89,7 +89,7 @@ func ParsePackage(r io.Reader) (*Package, error) {
 				return nil, err
 			}
 		} else if strings.EqualFold(hd.Name, "readme.md") {
-			data, err := io.ReadAll(tr)
+			data, err := util.ReadWithLimit(tr, 1024*1024)
 			if err != nil {
 				return nil, err
 			}

modules/util/io.go

@@ -29,7 +29,7 @@ func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
 // ReadWithLimit reads at most "limit" bytes from r into buf.
 // If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
 func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
-	return readWithLimit(r, 1024, n)
+	return readWithLimit(r, 4*1024, n)
 }
 
 func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {

routers/web/repo/wiki.go

@@ -133,7 +133,7 @@ func wikiContentsByEntry(ctx *context.Context, entry *git.TreeEntry) []byte {
 		return nil
 	}
 	defer reader.Close()
-	content, err := io.ReadAll(reader)
+	content, err := util.ReadWithLimit(reader, 5*1024*1024) // 5MB should be enough for a wiki page
 	if err != nil {
 		ctx.ServerError("ReadAll", err)
 		return nil

services/issue/template.go

@@ -5,7 +5,6 @@ package issue
 
 import (
 	"fmt"
-	"io"
 	"net/url"
 	"path"
 	"strings"
@@ -15,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/issue/template"
 	"code.gitea.io/gitea/modules/log"
 	api "code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
 
 	"gopkg.in/yaml.v3"
 )
@@ -65,7 +65,7 @@ func GetTemplateConfig(gitRepo *git.Repository, path string, commit *git.Commit)
 
 	defer reader.Close()
 
-	configContent, err := io.ReadAll(reader)
+	configContent, err := util.ReadWithLimit(reader, 1024*1024)
 	if err != nil {
 		return GetDefaultTemplateConfig(), err
 	}

services/repository/generate.go

@@ -7,7 +7,9 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -138,31 +140,37 @@ func (gt *giteaTemplateFileMatcher) Match(s string) bool {
 	return false
 }
 
-func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) {
-	localPath := filepath.Join(tmpDir, ".gitea", "template")
-	if _, err := os.Stat(localPath); os.IsNotExist(err) {
-		return nil, nil
-	} else if err != nil {
+func readLocalTmpRepoFileContent(localPath string, limit int) ([]byte, error) {
+	ok, err := util.IsRegularFile(localPath)
+	if err != nil {
 		return nil, err
+	} else if !ok {
+		return nil, fs.ErrNotExist
 	}
 
-	content, err := os.ReadFile(localPath)
+	f, err := os.Open(localPath)
 	if err != nil {
 		return nil, err
 	}
+	defer f.Close()
+
+	return util.ReadWithLimit(f, limit)
+}
 
+func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) {
+	localPath := filepath.Join(tmpDir, ".gitea", "template")
+	content, err := readLocalTmpRepoFileContent(localPath, 1024*1024)
+	if err != nil {
+		return nil, err
+	}
 	return newGiteaTemplateFileMatcher(localPath, content), nil
 }
 
 func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, templateRepo, generateRepo *repo_model.Repository) error {
 	tmpFullPath := filepath.Join(tmpDir, tmpDirSubPath)
-	if ok, err := util.IsRegularFile(tmpFullPath); !ok {
-		return err
-	}
-
-	content, err := os.ReadFile(tmpFullPath)
+	content, err := readLocalTmpRepoFileContent(tmpFullPath, 1024*1024)
 	if err != nil {
-		return err
+		return util.Iif(errors.Is(err, fs.ErrNotExist), nil, err)
 	}
 	if err := util.Remove(tmpFullPath); err != nil {
 		return err
@@ -172,7 +180,7 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t
 	substSubPath := filepath.Clean(filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo)))
 	newLocalPath := filepath.Join(tmpDir, substSubPath)
 	regular, err := util.IsRegularFile(newLocalPath)
-	if canWrite := regular || os.IsNotExist(err); !canWrite {
+	if canWrite := regular || errors.Is(err, fs.ErrNotExist); !canWrite {
 		return nil
 	}
 	if err := os.MkdirAll(filepath.Dir(newLocalPath), 0o755); err != nil {
@@ -242,15 +250,15 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
 
 	// Variable expansion
 	fileMatcher, err := readGiteaTemplateFile(tmpDir)
-	if err != nil {
-		return fmt.Errorf("readGiteaTemplateFile: %w", err)
-	}
-
-	if fileMatcher != nil {
+	if err == nil {
 		err = processGiteaTemplateFile(ctx, tmpDir, templateRepo, generateRepo, fileMatcher)
 		if err != nil {
-			return err
+			return fmt.Errorf("processGiteaTemplateFile: %w", err)
 		}
+	} else if errors.Is(err, fs.ErrNotExist) {
+		log.Debug("skip processing repo template files: no available .gitea/template")
+	} else {
+		return fmt.Errorf("readGiteaTemplateFile: %w", err)
 	}
 
 	if err = git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil {

services/repository/generate_test.go

@@ -4,6 +4,7 @@
 package repository
 
 import (
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
@@ -175,6 +176,31 @@ func TestProcessGiteaTemplateFile(t *testing.T) {
 		// subst from a link, skip, and the target is unchanged
 		assertSymLink("subst-${TEMPLATE_NAME}-from-link", tmpDir+"/sub/link-target")
 	}
+
+	{
+		templateFilePath := tmpDir + "/.gitea/template"
+
+		_ = os.Remove(templateFilePath)
+		_, err := os.Lstat(templateFilePath)
+		require.ErrorIs(t, err, fs.ErrNotExist)
+		_, err = readGiteaTemplateFile(tmpDir) // no template file
+		require.ErrorIs(t, err, fs.ErrNotExist)
+
+		_ = os.WriteFile(templateFilePath+".target", []byte("test-data-target"), 0o644)
+		_ = os.Symlink(templateFilePath+".target", templateFilePath)
+		content, _ := os.ReadFile(templateFilePath)
+		require.Equal(t, "test-data-target", string(content))
+		_, err = readGiteaTemplateFile(tmpDir) // symlinked template file
+		require.ErrorIs(t, err, fs.ErrNotExist)
+
+		_ = os.Remove(templateFilePath)
+		_ = os.WriteFile(templateFilePath, []byte("test-data-regular"), 0o644)
+		content, _ = os.ReadFile(templateFilePath)
+		require.Equal(t, "test-data-regular", string(content))
+		fm, err := readGiteaTemplateFile(tmpDir) // regular template file
+		require.NoError(t, err)
+		assert.Len(t, fm.globs, 1)
+	}
 }
 
 func TestTransformers(t *testing.T) {

services/webhook/deliver.go

@@ -30,6 +30,7 @@ import (
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
 )
 
@@ -264,7 +265,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
 		t.ResponseInfo.Headers[k] = strings.Join(vals, ",")
 	}
 
-	p, err := io.ReadAll(resp.Body)
+	p, err := util.ReadWithLimit(resp.Body, 1024*1024)
 	if err != nil {
 		t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)
 		return fmt.Errorf("unable to deliver webhook task[%d] in %s as unable to read response body: %w", t.ID, w.URL, err)