Skip to content

Commit 74ebf67

Browse files
ShazwazzaShazwazza
committed
updates the local storage sync, simplifies options and does more error checking in case sync cannot occur - which will default to normal storage.
1 parent 33036f8 commit 74ebf67

File tree

5 files changed

+131
-118
lines changed

5 files changed

+131
-118
lines changed

‎src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,21 @@ namespace UmbracoExamine.LocalStorage
99
public class LocalTempStorageDirectory : SimpleFSDirectory
1010
{
1111
private readonly Lucene.Net.Store.Directory _realDirectory;
12-
12+
1313
public LocalTempStorageDirectory(
1414
DirectoryInfo tempStorageDir,
1515
Lucene.Net.Store.Directory realDirectory)
1616
: base(tempStorageDir)
1717
{
1818
_realDirectory = realDirectory;
19+
Enabled = true;
1920
}
2021

22+
/// <summary>
23+
/// If initialization fails, it will be disabled and then this will just wrap the 'real directory'
24+
/// </summary>
25+
internal bool Enabled { get; set; }
26+
2127
public override string[] ListAll()
2228
{
2329
//always from the real dir
@@ -49,7 +55,11 @@ public override void TouchFile(string name)
4955
public override void DeleteFile(string name)
5056
{
5157
//perform on both dirs
52-
base.DeleteFile(name);
58+
if (Enabled)
59+
{
60+
base.DeleteFile(name);
61+
}
62+
5363
_realDirectory.DeleteFile(name);
5464
}
5565

@@ -67,24 +77,36 @@ public override long FileLength(string name)
6777
public override IndexOutput CreateOutput(string name)
6878
{
6979
//write to both indexes
80+
if (Enabled)
81+
{
82+
return new MultiIndexOutput(
83+
base.CreateOutput(name),
84+
_realDirectory.CreateOutput(name));
85+
}
7086

71-
return new MultiIndexOutput(
72-
base.CreateOutput(name),
73-
_realDirectory.CreateOutput(name));
87+
return _realDirectory.CreateOutput(name);
7488
}
7589

7690
/// <summary>
7791
/// Returns a stream reading an existing file.
7892
/// </summary>
7993
public override IndexInput OpenInput(string name)
8094
{
81-
//return the reader from the cache, not the real dir
82-
return base.OpenInput(name);
95+
if (Enabled)
96+
{
97+
//return the reader from the cache, not the real dir
98+
return base.OpenInput(name);
99+
}
100+
101+
return _realDirectory.OpenInput(name);
83102
}
84103

85104
public override void Dispose()
86105
{
87-
base.Dispose();
106+
if (Enabled)
107+
{
108+
base.Dispose();
109+
}
88110
_realDirectory.Dispose();
89111
}
90112

‎src/UmbracoExamine/LocalStorage/LocalTempStorageDirectoryTracker.cs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System.Collections.Concurrent;
2+
using System.IO;
3+
using Directory = Lucene.Net.Store.Directory;
4+
5+
namespace UmbracoExamine.LocalStorage
6+
{
7+
internal class LocalTempStorageDirectoryTracker
8+
{
9+
private readonly static LocalTempStorageDirectoryTracker Instance = new LocalTempStorageDirectoryTracker();
10+
private readonly ConcurrentDictionary<string, LocalTempStorageDirectory> _directories = new ConcurrentDictionary<string, LocalTempStorageDirectory>();
11+
12+
public static LocalTempStorageDirectoryTracker Current
13+
{
14+
get { return Instance; }
15+
}
16+
17+
public LocalTempStorageDirectory GetDirectory(DirectoryInfo dir, Directory realDir, bool disable = false)
18+
{
19+
var resolved = _directories.GetOrAdd(dir.FullName, s => new LocalTempStorageDirectory(dir, realDir));
20+
21+
if (disable)
22+
{
23+
resolved.Enabled = false;
24+
}
25+
26+
return resolved;
27+
}
28+
}
29+
}

‎src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs

Lines changed: 68 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,9 @@ internal class LocalTempStorageIndexer
1616
{
1717
private string _tempPath;
1818
public Lucene.Net.Store.Directory LuceneDirectory { get; private set; }
19-
private static readonly object Locker = new object();
19+
private readonly object _locker = new object();
2020
public SnapshotDeletionPolicy Snapshotter { get; private set; }
21-
private bool _syncStorage = false;
22-
21+
2322
public LocalTempStorageIndexer()
2423
{
2524
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
@@ -32,121 +31,101 @@ public void Initialize(NameValueCollection config, string configuredPath, Lucene
3231

3332
_tempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\"));
3433

35-
if (config != null)
36-
{
37-
if (config["syncTempStorage"] != null)
38-
{
39-
var attempt = config["syncTempStorage"].TryConvertTo<bool>();
40-
if (attempt)
41-
{
42-
_syncStorage = attempt.Result;
43-
}
44-
}
45-
}
34+
var success = InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
4635

47-
InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
36+
//create the custom lucene directory which will keep the main and temp FS's in sync
37+
LuceneDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
38+
new DirectoryInfo(_tempPath),
39+
baseLuceneDirectory,
40+
//flag to disable the mirrored folder if not successful
41+
success == false);
4842
}
4943

50-
private void InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
44+
private bool InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
5145
{
52-
lock (Locker)
46+
lock (_locker)
5347
{
5448
if (Directory.Exists(_tempPath) == false)
5549
{
5650
Directory.CreateDirectory(_tempPath);
5751
}
5852

59-
//if we are syncing storage to the main file system to temp files, then sync from the main FS to our temp FS
60-
if (_syncStorage)
53+
//copy index
54+
55+
using (new IndexWriter(
56+
//read from the underlying/default directory, not the temp codegen dir
57+
baseLuceneDirectory,
58+
analyzer,
59+
Snapshotter,
60+
IndexWriter.MaxFieldLength.UNLIMITED))
6161
{
62-
//copy index
63-
64-
using (new IndexWriter(
65-
//read from the underlying/default directory, not the temp codegen dir
66-
baseLuceneDirectory,
67-
analyzer,
68-
Snapshotter,
69-
IndexWriter.MaxFieldLength.UNLIMITED))
62+
try
7063
{
71-
try
72-
{
73-
var basePath = IOHelper.MapPath(configuredPath);
64+
var basePath = IOHelper.MapPath(configuredPath);
7465

75-
var commit = Snapshotter.Snapshot();
76-
var allSnapshotFiles = commit.GetFileNames().Concat(new[] {commit.GetSegmentsFileName()}).ToArray();
66+
var commit = Snapshotter.Snapshot();
67+
var allSnapshotFiles = commit.GetFileNames().Concat(new[] { commit.GetSegmentsFileName() })
68+
.Distinct()
69+
.ToArray();
7770

78-
var tempDir = new DirectoryInfo(_tempPath);
71+
var tempDir = new DirectoryInfo(_tempPath);
7972

80-
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
81-
var toRemove = tempDir.GetFiles()
82-
.Select(x => x.Name)
83-
.Except(allSnapshotFiles);
84-
85-
using (var tempDirectory = new SimpleFSDirectory(tempDir))
73+
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
74+
var toRemove = tempDir.GetFiles()
75+
.Select(x => x.Name)
76+
.Except(allSnapshotFiles);
77+
78+
using (var tempDirectory = new SimpleFSDirectory(tempDir))
79+
{
80+
if (IndexWriter.IsLocked(tempDirectory) == false)
8681
{
87-
if (IndexWriter.IsLocked(tempDirectory) == false)
82+
foreach (var file in toRemove)
8883
{
89-
foreach (var file in toRemove)
84+
try
9085
{
91-
try
92-
{
93-
File.Delete(Path.Combine(_tempPath, file));
94-
}
95-
catch (IOException ex)
96-
{
97-
LogHelper.Error<LocalTempStorageIndexer>("Could not delete index file, could not sync from main storage", ex);
98-
99-
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
100-
return;
101-
}
86+
File.Delete(Path.Combine(_tempPath, file));
87+
}
88+
catch (IOException ex)
89+
{
90+
LogHelper.Error<LocalTempStorageIndexer>("Could not delete index file, could not sync from main storage", ex);
91+
//quit here
92+
return false;
10293
}
103-
}
104-
else
105-
{
106-
LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
107-
108-
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
109-
return;
11094
}
11195
}
112-
113-
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
96+
else
11497
{
115-
try
116-
{
117-
File.Copy(
118-
Path.Combine(basePath, "Index", fileName),
119-
Path.Combine(_tempPath, Path.GetFileName(fileName)), true);
120-
}
121-
catch (IOException ex)
122-
{
123-
LogHelper.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
124-
125-
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
126-
return;
127-
}
98+
LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
99+
//quit here
100+
return false;
128101
}
129-
130102
}
131-
finally
132-
{
133-
Snapshotter.Release();
134-
}
135-
}
136103

137-
//create the custom lucene directory which will keep the main and temp FS's in sync
104+
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
105+
{
106+
try
107+
{
108+
File.Copy(
109+
Path.Combine(basePath, "Index", fileName),
110+
Path.Combine(_tempPath, Path.GetFileName(fileName)), true);
111+
}
112+
catch (IOException ex)
113+
{
114+
LogHelper.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
138115

139-
LuceneDirectory = new LocalTempStorageDirectory(
140-
new DirectoryInfo(_tempPath),
141-
baseLuceneDirectory);
142-
}
143-
else
144-
{
145-
//just return a normal lucene directory that uses the codegen folder
116+
//quit here
117+
return false;
118+
}
119+
}
146120

147-
LuceneDirectory = FSDirectory.Open(new DirectoryInfo(_tempPath));
121+
}
122+
finally
123+
{
124+
Snapshotter.Release();
125+
}
148126
}
149127

128+
return true;
150129
}
151130
}
152131
}

‎src/UmbracoExamine/UmbracoExamine.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@
122122
<Compile Include="IndexTypes.cs" />
123123
<Compile Include="LegacyLibrary.cs" />
124124
<Compile Include="LocalStorage\LocalTempStorageDirectory.cs" />
125+
<Compile Include="LocalStorage\LocalTempStorageDirectoryTracker.cs" />
125126
<Compile Include="LocalStorage\LocalTempStorageIndexer.cs" />
126127
<Compile Include="LocalStorage\MultiIndexOutput.cs" />
127128
<Compile Include="LoggingLevel.cs" />

‎src/UmbracoExamine/UmbracoExamineSearcher.cs

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ public class UmbracoExamineSearcher : LuceneSearcher
2828
private volatile Lucene.Net.Store.Directory _localTempDirectory;
2929
private static readonly object Locker = new object();
3030
private string _localTempPath = null;
31-
private bool _syncTempStorage = false;
3231

3332
#region Constructors
3433

@@ -82,15 +81,6 @@ public override void Initialize(string name, System.Collections.Specialized.Name
8281
var codegenPath = HttpRuntime.CodegenDir;
8382
_localTempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\"));
8483
}
85-
86-
if (config["syncTempStorage"] != null)
87-
{
88-
var attemptSync = config["syncTempStorage"].TryConvertTo<bool>();
89-
if (attemptSync)
90-
{
91-
_syncTempStorage = attemptSync.Result;
92-
}
93-
}
9484
}
9585
}
9686

@@ -182,17 +172,9 @@ protected override Lucene.Net.Store.Directory GetLuceneDirectory()
182172
{
183173
if (_localTempDirectory == null)
184174
{
185-
if (_syncTempStorage)
186-
{
187-
_localTempDirectory = new LocalTempStorageDirectory(
188-
new DirectoryInfo(_localTempPath),
189-
base.GetLuceneDirectory());
190-
}
191-
else
192-
{
193-
//not syncing just use a normal lucene directory
194-
_localTempDirectory = FSDirectory.Open(new DirectoryInfo(_localTempPath));
195-
}
175+
_localTempDirectory = LocalTempStorageDirectoryTracker.Current.GetDirectory(
176+
new DirectoryInfo(_localTempPath),
177+
base.GetLuceneDirectory());
196178
}
197179
}
198180
}

0 commit comments

Comments
 (0)