Ignore Sync errors on pipes when doing CheckAttributeReader.CheckPath, fix the hang of git cat-file

modules/git/repo_attribute.go

@@ -12,6 +12,8 @@ import (
 	"os"
 	"strconv"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
 )
 
 // CheckAttributeOpts represents the possible options to CheckAttribute
@@ -112,13 +114,15 @@ func (c *CheckAttributeReader) Init(ctx context.Context) error {
 
 	if len(c.IndexFile) > 0 && CheckGitVersionAtLeast("1.7.8") == nil {
 		cmdArgs = append(cmdArgs, "--cached")
-		c.env = []string{"GIT_INDEX_FILE=" + c.IndexFile}
+		c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile)
 	}
 
 	if len(c.WorkTree) > 0 && CheckGitVersionAtLeast("1.7.8") == nil {
-		c.env = []string{"GIT_WORK_TREE=" + c.WorkTree}
+		c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree)
 	}
 
+	c.env = append(c.env, "GIT_FLUSH=1")
+
 	if len(c.Attributes) > 0 {
 		cmdArgs = append(cmdArgs, c.Attributes...)
 		cmdArgs = append(cmdArgs, "--")
@@ -155,13 +159,14 @@ func (c *CheckAttributeReader) Init(ctx context.Context) error {
 
 // Run run cmd
 func (c *CheckAttributeReader) Run() error {
+	defer func() {
+		_ = c.Close()
+	}()
 	stdErr := new(bytes.Buffer)
 	err := c.cmd.RunInDirTimeoutEnvFullPipelineFunc(c.env, -1, c.Repo.Path, c.stdOut, stdErr, c.stdinReader, func(_ context.Context, _ context.CancelFunc) error {
 		close(c.running)
 		return nil
 	})
-	defer c.cancel()
-	_ = c.stdOut.Close()
 	if err != nil && c.ctx.Err() != nil && err.Error() != "signal: killed" {
 		return fmt.Errorf("failed to run attr-check. Error: %w\nStderr: %s", err, stdErr.String())
 	}
@@ -170,27 +175,31 @@ func (c *CheckAttributeReader) Run() error {
 }
 
 // CheckPath check attr for given path
-func (c *CheckAttributeReader) CheckPath(path string) (map[string]string, error) {
+func (c *CheckAttributeReader) CheckPath(path string) (rs map[string]string, err error) {
+	defer func() {
+		if err != nil {
+			log.Error("CheckPath returns error: %v", err)
+		}
+	}()
+
 	select {
 	case <-c.ctx.Done():
 		return nil, c.ctx.Err()
 	case <-c.running:
 	}
 
-	if _, err := c.stdinWriter.Write([]byte(path + "\x00")); err != nil {
-		defer c.cancel()
+	if _, err = c.stdinWriter.Write([]byte(path + "\x00")); err != nil {
+		defer c.Close()
 		return nil, err
 	}
 
-	if err := c.stdinWriter.Sync(); err != nil {
-		defer c.cancel()
-		return nil, err
-	}
-
-	rs := make(map[string]string)
+	rs = make(map[string]string)
 	for range c.Attributes {
 		select {
-		case attr := <-c.stdOut.ReadAttribute():
+		case attr, ok := <-c.stdOut.ReadAttribute():
+			if !ok {
+				return nil, c.ctx.Err()
+			}
 			rs[attr.Attribute] = attr.Value
 		case <-c.ctx.Done():
 			return nil, c.ctx.Err()
@@ -201,13 +210,16 @@ func (c *CheckAttributeReader) CheckPath(path string) (map[string]string, error)
 
 // Close close pip after use
 func (c *CheckAttributeReader) Close() error {
+	err := c.stdinWriter.Close()
+	_ = c.stdinReader.Close()
+	_ = c.stdOut.Close()
+	c.cancel()
 	select {
 	case <-c.running:
 	default:
 		close(c.running)
 	}
-	defer c.cancel()
-	return c.stdinWriter.Close()
+	return err
 }
 
 type attributeWriter interface {

modules/indexer/stats/db.go

@@ -38,11 +38,11 @@ func (db *DBIndexer) Index(id int64) error {
 	// Get latest commit for default branch
 	commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
 	if err != nil {
-		if git.IsErrBranchNotExist(err) || git.IsErrNotExist((err)) {
-			log.Debug("Unable to get commit ID for defaultbranch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath())
+		if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) {
+			log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.RepoPath())
 			return nil
 		}
-		log.Error("Unable to get commit ID for defaultbranch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err)
+		log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.RepoPath(), err)
 		return err
 	}
 
@@ -54,10 +54,17 @@ func (db *DBIndexer) Index(id int64) error {
 	// Calculate and save language statistics to database
 	stats, err := gitRepo.GetLanguageStats(commitID)
 	if err != nil {
-		log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
+		log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
 		return err
 	}
-	return repo.UpdateLanguageStats(commitID, stats)
+	err = repo.UpdateLanguageStats(commitID, stats)
+	if err != nil {
+		log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
+		return err
+	}
+
+	log.Info("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.RepoPath(), len(stats))
+	return nil
 }
 
 // Close dummy function