Skip to content

Commit 0688550

Browse files
ShazwazzaShazwazza
committed
start integrating the local templ storage option for examine
1 parent d43b673 commit 0688550

File tree

7 files changed

+502
-3
lines changed

7 files changed

+502
-3
lines changed

‎src/UmbracoExamine/LocalStorage/LocalTempStorageDirectory.cs

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
using System.Collections.Generic;
2+
using System.IO;
3+
using System.Text;
4+
using System.Threading.Tasks;
5+
using Lucene.Net.Store;
6+
7+
namespace UmbracoExamine.LocalStorage
8+
{
9+
public class LocalTempStorageDirectory : SimpleFSDirectory
10+
{
11+
private readonly Lucene.Net.Store.Directory _realDirectory;
12+
13+
public LocalTempStorageDirectory(
14+
DirectoryInfo tempStorageDir,
15+
Lucene.Net.Store.Directory realDirectory)
16+
: base(tempStorageDir)
17+
{
18+
_realDirectory = realDirectory;
19+
}
20+
21+
public override string[] ListAll()
22+
{
23+
//always from the real dir
24+
return _realDirectory.ListAll();
25+
}
26+
27+
/// <summary>Returns true if a file with the given name exists. </summary>
28+
public override bool FileExists(string name)
29+
{
30+
//always from the real dir
31+
return _realDirectory.FileExists(name);
32+
}
33+
34+
/// <summary>Returns the time the named file was last modified. </summary>
35+
public override long FileModified(string name)
36+
{
37+
//always from the real dir
38+
return _realDirectory.FileModified(name);
39+
}
40+
41+
/// <summary>Set the modified time of an existing file to now. </summary>
42+
public override void TouchFile(string name)
43+
{
44+
//always from the real dir
45+
_realDirectory.TouchFile(name);
46+
}
47+
48+
/// <summary>Removes an existing file in the directory. </summary>
49+
public override void DeleteFile(string name)
50+
{
51+
//perform on both dirs
52+
_realDirectory.DeleteFile(name);
53+
base.DeleteFile(name);
54+
}
55+
56+
/// <summary>Returns the length of a file in the directory. </summary>
57+
public override long FileLength(string name)
58+
{
59+
//always from the real dir
60+
return _realDirectory.FileLength(name);
61+
}
62+
63+
/// <summary>
64+
/// Creates a new, empty file in the directory with the given name.
65+
/// Returns a stream writing this file.
66+
/// </summary>
67+
public override IndexOutput CreateOutput(string name)
68+
{
69+
//write to both indexes
70+
71+
return new MultiIndexOutput(
72+
base.CreateOutput(name),
73+
_realDirectory.CreateOutput(name));
74+
}
75+
76+
/// <summary>
77+
/// Returns a stream reading an existing file.
78+
/// </summary>
79+
public override IndexInput OpenInput(string name)
80+
{
81+
//return the reader from the cache, not the real dir
82+
return base.OpenInput(name);
83+
}
84+
85+
public override void Dispose()
86+
{
87+
base.Dispose();
88+
_realDirectory.Dispose();
89+
}
90+
91+
92+
}
93+
}

‎src/UmbracoExamine/LocalStorage/LocalTempStorageIndexer.cs

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
using System.Collections.Specialized;
2+
using System.IO;
3+
using System.Linq;
4+
using System.Web;
5+
using Lucene.Net.Analysis;
6+
using Lucene.Net.Index;
7+
using Lucene.Net.Store;
8+
using Umbraco.Core;
9+
using Umbraco.Core.IO;
10+
using Umbraco.Core.Logging;
11+
using Directory = System.IO.Directory;
12+
13+
namespace UmbracoExamine.LocalStorage
14+
{
15+
internal class LocalTempStorageIndexer
16+
{
17+
private string _tempPath;
18+
public Lucene.Net.Store.Directory LuceneDirectory { get; private set; }
19+
private static readonly object Locker = new object();
20+
public SnapshotDeletionPolicy Snapshotter { get; private set; }
21+
private bool _syncStorage = false;
22+
23+
public LocalTempStorageIndexer()
24+
{
25+
IndexDeletionPolicy policy = new KeepOnlyLastCommitDeletionPolicy();
26+
Snapshotter = new SnapshotDeletionPolicy(policy);
27+
}
28+
29+
public void Initialize(NameValueCollection config, string configuredPath, Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer)
30+
{
31+
var codegenPath = HttpRuntime.CodegenDir;
32+
33+
_tempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\"));
34+
35+
if (config != null)
36+
{
37+
if (config["syncTempStorage"] != null)
38+
{
39+
var attempt = config["syncTempStorage"].TryConvertTo<bool>();
40+
if (attempt)
41+
{
42+
_syncStorage = attempt.Result;
43+
}
44+
}
45+
}
46+
47+
InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath);
48+
}
49+
50+
private void InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
51+
{
52+
lock (Locker)
53+
{
54+
if (Directory.Exists(_tempPath) == false)
55+
{
56+
Directory.CreateDirectory(_tempPath);
57+
}
58+
59+
//if we are syncing storage to the main file system to temp files, then sync from the main FS to our temp FS
60+
if (_syncStorage)
61+
{
62+
//copy index
63+
64+
using (new IndexWriter(
65+
//read from the underlying/default directory, not the temp codegen dir
66+
baseLuceneDirectory,
67+
analyzer,
68+
Snapshotter,
69+
IndexWriter.MaxFieldLength.UNLIMITED))
70+
{
71+
try
72+
{
73+
var basePath = IOHelper.MapPath(configuredPath);
74+
75+
var commit = Snapshotter.Snapshot();
76+
var allSnapshotFiles = commit.GetFileNames().Concat(new[] {commit.GetSegmentsFileName()}).ToArray();
77+
78+
var tempDir = new DirectoryInfo(_tempPath);
79+
80+
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
81+
var toRemove = tempDir.GetFiles()
82+
.Select(x => x.Name)
83+
.Except(allSnapshotFiles);
84+
85+
using (var tempDirectory = new SimpleFSDirectory(tempDir))
86+
{
87+
if (IndexWriter.IsLocked(tempDirectory) == false)
88+
{
89+
foreach (var file in toRemove)
90+
{
91+
try
92+
{
93+
File.Delete(Path.Combine(_tempPath, file));
94+
}
95+
catch (IOException ex)
96+
{
97+
LogHelper.Error<LocalTempStorageIndexer>("Could not delete index file, could not sync from main storage", ex);
98+
99+
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
100+
return;
101+
}
102+
}
103+
}
104+
else
105+
{
106+
LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
107+
108+
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
109+
return;
110+
}
111+
}
112+
113+
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
114+
{
115+
try
116+
{
117+
File.Copy(
118+
Path.Combine(basePath, "Index", fileName),
119+
Path.Combine(_tempPath, Path.GetFileName(fileName)), true);
120+
}
121+
catch (IOException ex)
122+
{
123+
LogHelper.Error<LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
124+
125+
//quit here and do not assign the lucene directory, this means that the app will now just be working from normal storage
126+
return;
127+
}
128+
}
129+
130+
}
131+
finally
132+
{
133+
Snapshotter.Release();
134+
}
135+
}
136+
137+
//create the custom lucene directory which will keep the main and temp FS's in sync
138+
139+
LuceneDirectory = new LocalTempStorageDirectory(
140+
new DirectoryInfo(_tempPath),
141+
baseLuceneDirectory);
142+
}
143+
else
144+
{
145+
//just return a normal lucene directory that uses the codegen folder
146+
147+
LuceneDirectory = FSDirectory.Open(new DirectoryInfo(_tempPath));
148+
}
149+
150+
}
151+
}
152+
}
153+
}

‎src/UmbracoExamine/LocalStorage/MultiIndexOutput.cs

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
using System;
2+
using System.Linq;
3+
using Lucene.Net.Store;
4+
5+
namespace UmbracoExamine.LocalStorage
6+
{
7+
public class MultiIndexOutput : IndexOutput
8+
{
9+
private readonly IndexOutput[] _outputs;
10+
11+
public MultiIndexOutput(params IndexOutput[] outputs)
12+
{
13+
if (outputs.Length < 1)
14+
{
15+
throw new InvalidOperationException("There must be at least one output specified");
16+
}
17+
_outputs = outputs;
18+
}
19+
20+
public override void WriteByte(byte b)
21+
{
22+
foreach (var output in _outputs)
23+
{
24+
output.WriteByte(b);
25+
}
26+
}
27+
28+
public override void WriteBytes(byte[] b, int offset, int length)
29+
{
30+
foreach (var output in _outputs)
31+
{
32+
output.WriteBytes(b, offset, length);
33+
}
34+
}
35+
36+
public override void Flush()
37+
{
38+
foreach (var output in _outputs)
39+
{
40+
output.Flush();
41+
}
42+
}
43+
44+
public override void Close()
45+
{
46+
foreach (var output in _outputs)
47+
{
48+
output.Close();
49+
}
50+
}
51+
52+
public override long GetFilePointer()
53+
{
54+
//return the first
55+
return _outputs.First().GetFilePointer();
56+
}
57+
58+
public override void Seek(long pos)
59+
{
60+
foreach (var output in _outputs)
61+
{
62+
output.Seek(pos);
63+
}
64+
}
65+
66+
public override long Length()
67+
{
68+
//return the first
69+
return _outputs.First().GetFilePointer();
70+
}
71+
}
72+
}

0 commit comments

Comments
 (0)