Skip to content

Commit 158859a

Browse files
imsodincalmh
imsodin
authored and
calmh
committed
lib: Handle metadata changes for send-only folders (fixes syncthing#4616, fixes syncthing#4627) (syncthing#4750)
Unignored files are marked as conflicting while scanning, which is then resolved in the subsequent pull. Automatically reconciles needed items on send-only folders, if they do not actually differ except for internal metadata.
1 parent 5822222 commit 158859a

File tree

10 files changed

+424
-240
lines changed

10 files changed

+424
-240
lines changed

‎lib/db/leveldb_dbinstance.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
324324
return devices
325325
}
326326

327-
func (db *Instance) withNeed(folder, device []byte, truncate bool, needAllInvalid bool, fn Iterator) {
327+
func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
328328
t := db.newReadOnlyTransaction()
329329
defer t.close()
330330

@@ -351,12 +351,6 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, needAllInvali
351351
if bytes.Equal(v.Device, device) {
352352
have = true
353353
haveFileVersion = v
354-
// We need invalid files regardless of version when
355-
// ignore patterns changed
356-
if v.Invalid && needAllInvalid {
357-
need = true
358-
break
359-
}
360354
// XXX: This marks Concurrent (i.e. conflicting) changes as
361355
// needs. Maybe we should do that, but it needs special
362356
// handling in the puller.

‎lib/db/set.go

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -164,24 +164,12 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
164164

165165
func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
166166
l.Debugf("%s WithNeed(%v)", s.folder, device)
167-
s.db.withNeed([]byte(s.folder), device[:], false, false, nativeFileIterator(fn))
167+
s.db.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn))
168168
}
169169

170170
func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
171171
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
172-
s.db.withNeed([]byte(s.folder), device[:], true, false, nativeFileIterator(fn))
173-
}
174-
175-
// WithNeedOrInvalid considers all invalid files as needed, regardless of their version
176-
// (e.g. for pulling when ignore patterns changed)
177-
func (s *FileSet) WithNeedOrInvalid(device protocol.DeviceID, fn Iterator) {
178-
l.Debugf("%s WithNeedExcludingInvalid(%v)", s.folder, device)
179-
s.db.withNeed([]byte(s.folder), device[:], false, true, nativeFileIterator(fn))
180-
}
181-
182-
func (s *FileSet) WithNeedOrInvalidTruncated(device protocol.DeviceID, fn Iterator) {
183-
l.Debugf("%s WithNeedExcludingInvalidTruncated(%v)", s.folder, device)
184-
s.db.withNeed([]byte(s.folder), device[:], true, true, nativeFileIterator(fn))
172+
s.db.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn))
185173
}
186174

187175
func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) {

‎lib/model/folder.go

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313

1414
"github.com/syncthing/syncthing/lib/config"
1515
"github.com/syncthing/syncthing/lib/ignore"
16+
"github.com/syncthing/syncthing/lib/protocol"
1617
"github.com/syncthing/syncthing/lib/sync"
1718
"github.com/syncthing/syncthing/lib/watchaggregator"
1819
)
@@ -23,16 +24,21 @@ type folder struct {
2324
stateTracker
2425
config.FolderConfiguration
2526

27+
model *Model
28+
shortID protocol.ShortID
29+
ctx context.Context
30+
cancel context.CancelFunc
31+
2632
scan folderScanner
27-
model *Model
28-
ctx context.Context
29-
cancel context.CancelFunc
3033
initialScanFinished chan struct{}
31-
watchCancel context.CancelFunc
32-
watchChan chan []string
33-
restartWatchChan chan struct{}
34-
watchErr error
35-
watchErrMut sync.Mutex
34+
35+
pullScheduled chan struct{}
36+
37+
watchCancel context.CancelFunc
38+
watchChan chan []string
39+
restartWatchChan chan struct{}
40+
watchErr error
41+
watchErrMut sync.Mutex
3642
}
3743

3844
func newFolder(model *Model, cfg config.FolderConfiguration) folder {
@@ -42,14 +48,19 @@ func newFolder(model *Model, cfg config.FolderConfiguration) folder {
4248
stateTracker: newStateTracker(cfg.ID),
4349
FolderConfiguration: cfg,
4450

51+
model: model,
52+
shortID: model.shortID,
53+
ctx: ctx,
54+
cancel: cancel,
55+
4556
scan: newFolderScanner(cfg),
46-
ctx: ctx,
47-
cancel: cancel,
48-
model: model,
4957
initialScanFinished: make(chan struct{}),
50-
watchCancel: func() {},
51-
watchErr: errWatchNotStarted,
52-
watchErrMut: sync.NewMutex(),
58+
59+
pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.
60+
61+
watchCancel: func() {},
62+
watchErr: errWatchNotStarted,
63+
watchErrMut: sync.NewMutex(),
5364
}
5465
}
5566

@@ -65,7 +76,16 @@ func (f *folder) IgnoresUpdated() {
6576
}
6677
}
6778

68-
func (f *folder) SchedulePull() {}
79+
func (f *folder) SchedulePull() {
80+
select {
81+
case f.pullScheduled <- struct{}{}:
82+
default:
83+
// We might be busy doing a pull and thus not reading from this
84+
// channel. The channel is 1-buffered, so one notification will be
85+
// queued to ensure we recheck after the pull, but beyond that we must
86+
// make sure to not block index receiving.
87+
}
88+
}
6989

7090
func (f *folder) Jobs() ([]string, []string) {
7191
return nil, nil

‎lib/model/model_test.go

Lines changed: 1 addition & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/d4l3k/messagediff"
2727
"github.com/syncthing/syncthing/lib/config"
2828
"github.com/syncthing/syncthing/lib/db"
29-
"github.com/syncthing/syncthing/lib/events"
3029
"github.com/syncthing/syncthing/lib/fs"
3130
"github.com/syncthing/syncthing/lib/ignore"
3231
"github.com/syncthing/syncthing/lib/osutil"
@@ -50,6 +49,7 @@ func init() {
5049
defaultFolderConfig = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, "testdata")
5150
defaultFolderConfig.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
5251
_defaultConfig := config.Configuration{
52+
Version: config.CurrentVersion,
5353
Folders: []config.FolderConfiguration{defaultFolderConfig},
5454
Devices: []config.DeviceConfiguration{config.NewDeviceConfiguration(device1, "device1")},
5555
Options: config.OptionsConfiguration{
@@ -3442,86 +3442,6 @@ func TestPausedFolders(t *testing.T) {
34423442
}
34433443
}
34443444

3445-
func TestPullInvalid(t *testing.T) {
3446-
if runtime.GOOS != "windows" {
3447-
t.Skip("Windows only")
3448-
}
3449-
3450-
tmpDir, err := ioutil.TempDir(".", "_model-")
3451-
if err != nil {
3452-
panic("Failed to create temporary testing dir")
3453-
}
3454-
defer os.RemoveAll(tmpDir)
3455-
3456-
cfg := defaultConfig.RawCopy()
3457-
cfg.Folders[0] = config.NewFolderConfiguration(protocol.LocalDeviceID, "default", "default", fs.FilesystemTypeBasic, tmpDir)
3458-
cfg.Folders[0].Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
3459-
w := config.Wrap("/tmp/cfg", cfg)
3460-
3461-
db := db.OpenMemory()
3462-
m := NewModel(w, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
3463-
m.AddFolder(cfg.Folders[0])
3464-
m.StartFolder("default")
3465-
m.ServeBackground()
3466-
defer m.Stop()
3467-
m.ScanFolder("default")
3468-
3469-
if err := m.SetIgnores("default", []string{"*:ignored"}); err != nil {
3470-
panic(err)
3471-
}
3472-
3473-
ign := "invalid:ignored"
3474-
del := "invalid:deleted"
3475-
var version protocol.Vector
3476-
version = version.Update(device1.Short())
3477-
3478-
m.IndexUpdate(device1, "default", []protocol.FileInfo{
3479-
{
3480-
Name: ign,
3481-
Size: 1234,
3482-
Type: protocol.FileInfoTypeFile,
3483-
Version: version,
3484-
},
3485-
{
3486-
Name: del,
3487-
Size: 1234,
3488-
Type: protocol.FileInfoTypeFile,
3489-
Version: version,
3490-
Deleted: true,
3491-
},
3492-
})
3493-
3494-
sub := events.Default.Subscribe(events.FolderErrors)
3495-
defer events.Default.Unsubscribe(sub)
3496-
3497-
timeout := time.NewTimer(5 * time.Second)
3498-
for {
3499-
select {
3500-
case ev := <-sub.C():
3501-
t.Fatalf("Errors while pulling: %v", ev)
3502-
case <-timeout.C:
3503-
t.Fatalf("File wasn't added to index until timeout")
3504-
default:
3505-
}
3506-
3507-
file, ok := m.CurrentFolderFile("default", ign)
3508-
if !ok {
3509-
time.Sleep(100 * time.Millisecond)
3510-
continue
3511-
}
3512-
3513-
if !file.Invalid {
3514-
t.Error("Ignored file isn't marked as invalid")
3515-
}
3516-
3517-
if file, ok = m.CurrentFolderFile("default", del); ok {
3518-
t.Error("Deleted invalid file was added to index")
3519-
}
3520-
3521-
return
3522-
}
3523-
}
3524-
35253445
func addFakeConn(m *Model, dev protocol.DeviceID) *fakeConnection {
35263446
fc := &fakeConnection{id: dev, model: m}
35273447
m.AddConnection(fc, protocol.HelloResult{})

0 commit comments

Comments
 (0)