Skip to content

Commit 68c1b2d

Browse files
imsodincalmh
imsodin
authored and
calmh
committed
all: Revert simultaneously walk fs and db on scan (fixes syncthing#4756) (syncthing#4757)
This reverts commit 6d3f9d5.
1 parent b57c9b6 commit 68c1b2d

File tree

9 files changed

+258
-677
lines changed

9 files changed

+258
-677
lines changed

‎lib/db/set.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,9 +194,9 @@ func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
194194
s.db.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn))
195195
}
196196

197-
func (s *FileSet) WithPrefixedHave(device protocol.DeviceID, prefix string, fn Iterator) {
198-
l.Debugf("%s WithPrefixedHave(%v)", s.folder, device)
199-
s.db.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), false, nativeFileIterator(fn))
197+
func (s *FileSet) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
198+
l.Debugf("%s WithPrefixedHaveTruncated(%v)", s.folder, device)
199+
s.db.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn))
200200
}
201201
func (s *FileSet) WithGlobal(fn Iterator) {
202202
l.Debugf("%s WithGlobal()", s.folder)

‎lib/fs/walkfs.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,7 @@
1010

1111
package fs
1212

13-
import (
14-
"path/filepath"
15-
"sort"
16-
)
13+
import "path/filepath"
1714

1815
// WalkFunc is the type of the function called for each file or directory
1916
// visited by Walk. The path argument contains the argument to Walk as a
@@ -57,7 +54,6 @@ func (f *walkFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error
5754
if err != nil {
5855
return walkFn(path, info, err)
5956
}
60-
sort.Strings(names)
6157

6258
for _, name := range names {
6359
filename := filepath.Join(path, name)

‎lib/model/model.go

Lines changed: 91 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1395,21 +1395,14 @@ func (m *Model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
13951395
return fs.GetGlobal(file)
13961396
}
13971397

1398-
type haveWalker struct {
1399-
fset *db.FileSet
1398+
type cFiler struct {
1399+
m *Model
1400+
r string
14001401
}
14011402

1402-
func (h haveWalker) Walk(prefix string, ctx context.Context, out chan<- protocol.FileInfo) {
1403-
ctxChan := ctx.Done()
1404-
h.fset.WithPrefixedHave(protocol.LocalDeviceID, prefix, func(fi db.FileIntf) bool {
1405-
f := fi.(protocol.FileInfo)
1406-
select {
1407-
case out <- f:
1408-
case <-ctxChan:
1409-
return false
1410-
}
1411-
return true
1412-
})
1403+
// Implements scanner.CurrentFiler
1404+
func (cf cFiler) CurrentFile(file string) (protocol.FileInfo, bool) {
1405+
return cf.m.CurrentFolderFile(cf.r, file)
14131406
}
14141407

14151408
// Connection returns the current connection for device, and a boolean wether a connection was found.
@@ -1962,14 +1955,13 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
19621955

19631956
runner.setState(FolderScanning)
19641957

1965-
haveWalker := haveWalker{fset}
1966-
rchan := scanner.Walk(ctx, scanner.Config{
1958+
fchan := scanner.Walk(ctx, scanner.Config{
19671959
Folder: folderCfg.ID,
19681960
Subs: subDirs,
19691961
Matcher: ignores,
19701962
BlockSize: protocol.BlockSize,
19711963
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
1972-
Have: haveWalker,
1964+
CurrentFiler: cFiler{m, folder},
19731965
Filesystem: mtimefs,
19741966
IgnorePerms: folderCfg.IgnorePerms,
19751967
AutoNormalize: folderCfg.AutoNormalize,
@@ -1986,17 +1978,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
19861978
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
19871979
batchSizeBytes := 0
19881980
changes := 0
1989-
checkBatch := func() error {
1990-
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
1991-
if err := runner.CheckHealth(); err != nil {
1992-
return err
1993-
}
1994-
m.updateLocalsFromScanning(folder, batch)
1995-
batch = batch[:0]
1996-
batchSizeBytes = 0
1997-
}
1998-
return nil
1999-
}
20001981

20011982
// Schedule a pull after scanning, but only if we actually detected any
20021983
// changes.
@@ -2006,49 +1987,98 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
20061987
}
20071988
}()
20081989

2009-
var delDirStack []protocol.FileInfo
2010-
for r := range rchan {
2011-
if err := checkBatch(); err != nil {
2012-
l.Debugln("Stopping scan of folder %s due to: %s", folderCfg.Description(), err)
2013-
return err
2014-
}
2015-
2016-
// Append deleted dirs from stack if the current file isn't a child,
2017-
// which means all children were already processed.
2018-
for len(delDirStack) != 0 && !strings.HasPrefix(r.New.Name, delDirStack[len(delDirStack)-1].Name+string(fs.PathSeparator)) {
2019-
lastDelDir := delDirStack[len(delDirStack)-1]
2020-
batch = append(batch, lastDelDir)
2021-
batchSizeBytes += lastDelDir.ProtoSize()
2022-
changes++
2023-
if err := checkBatch(); err != nil {
1990+
for f := range fchan {
1991+
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
1992+
if err := runner.CheckHealth(); err != nil {
20241993
l.Debugln("Stopping scan of folder %s due to: %s", folderCfg.Description(), err)
20251994
return err
20261995
}
2027-
delDirStack = delDirStack[:len(delDirStack)-1]
2028-
}
2029-
2030-
// Delay appending deleted dirs until all its children are processed
2031-
if r.Old.IsDirectory() && (r.New.Deleted || !r.New.IsDirectory()) {
2032-
delDirStack = append(delDirStack, r.New)
2033-
continue
1996+
m.updateLocalsFromScanning(folder, batch)
1997+
batch = batch[:0]
1998+
batchSizeBytes = 0
20341999
}
20352000

2036-
l.Debugln("Appending", r)
2037-
batch = append(batch, r.New)
2038-
batchSizeBytes += r.New.ProtoSize()
2001+
batch = append(batch, f)
2002+
batchSizeBytes += f.ProtoSize()
20392003
changes++
20402004
}
20412005

2042-
// Append remaining deleted dirs.
2043-
for i := len(delDirStack) - 1; i >= 0; i-- {
2044-
if err := checkBatch(); err != nil {
2045-
l.Debugln("Stopping scan of folder %s due to: %s", folderCfg.Description(), err)
2046-
return err
2047-
}
2006+
if err := runner.CheckHealth(); err != nil {
2007+
l.Debugln("Stopping scan of folder %s due to: %s", folderCfg.Description(), err)
2008+
return err
2009+
} else if len(batch) > 0 {
2010+
m.updateLocalsFromScanning(folder, batch)
2011+
}
20482012

2049-
batch = append(batch, delDirStack[i])
2050-
batchSizeBytes += delDirStack[i].ProtoSize()
2051-
changes++
2013+
if len(subDirs) == 0 {
2014+
// If we have no specific subdirectories to traverse, set it to one
2015+
// empty prefix so we traverse the entire folder contents once.
2016+
subDirs = []string{""}
2017+
}
2018+
2019+
// Do a scan of the database for each prefix, to check for deleted and
2020+
// ignored files.
2021+
batch = batch[:0]
2022+
batchSizeBytes = 0
2023+
for _, sub := range subDirs {
2024+
var iterError error
2025+
2026+
fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
2027+
f := fi.(db.FileInfoTruncated)
2028+
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
2029+
if err := runner.CheckHealth(); err != nil {
2030+
iterError = err
2031+
return false
2032+
}
2033+
m.updateLocalsFromScanning(folder, batch)
2034+
batch = batch[:0]
2035+
batchSizeBytes = 0
2036+
}
2037+
2038+
switch {
2039+
case !f.IsInvalid() && ignores.Match(f.Name).IsIgnored():
2040+
// File was valid at last pass but has been ignored. Set invalid bit.
2041+
l.Debugln("setting invalid bit on ignored", f)
2042+
nf := f.ConvertToInvalidFileInfo(m.id.Short())
2043+
batch = append(batch, nf)
2044+
batchSizeBytes += nf.ProtoSize()
2045+
changes++
2046+
2047+
case !f.IsInvalid() && !f.IsDeleted():
2048+
// The file is valid and not deleted. Lets check if it's
2049+
// still here.
2050+
2051+
if _, err := mtimefs.Lstat(f.Name); err != nil {
2052+
// We don't specifically verify that the error is
2053+
// fs.IsNotExist because there is a corner case when a
2054+
// directory is suddenly transformed into a file. When that
2055+
// happens, files that were in the directory (that is now a
2056+
// file) are deleted but will return a confusing error ("not a
2057+
// directory") when we try to Lstat() them.
2058+
2059+
nf := protocol.FileInfo{
2060+
Name: f.Name,
2061+
Type: f.Type,
2062+
Size: 0,
2063+
ModifiedS: f.ModifiedS,
2064+
ModifiedNs: f.ModifiedNs,
2065+
ModifiedBy: m.id.Short(),
2066+
Deleted: true,
2067+
Version: f.Version.Update(m.shortID),
2068+
}
2069+
2070+
batch = append(batch, nf)
2071+
batchSizeBytes += nf.ProtoSize()
2072+
changes++
2073+
}
2074+
}
2075+
return true
2076+
})
2077+
2078+
if iterError != nil {
2079+
l.Debugln("Stopping scan of folder %s due to: %s", folderCfg.Description(), iterError)
2080+
return iterError
2081+
}
20522082
}
20532083

20542084
if err := runner.CheckHealth(); err != nil {

0 commit comments

Comments
 (0)