/* * Copyright 2012 Splunk, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"): you may * not use this file except in compliance with the License. You may obtain * a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ namespace Splunk { using System; using System.Diagnostics; using System.IO; using System.Net.Sockets; using System.Threading; /// /// The Index class represents the Splunk DB/Index object. /// public class Index : Entity { /// /// Initializes a new instance of the class. /// /// The connected service /// The path public Index(Service service, string path) : base(service, path) { } /// /// Gets or sets a value indicating whether the data retrieved from this /// index has been UTF8-encoded. Indexing performance degrades when this /// parameter is set to true. /// public bool AssureUTF8 { get { return this.GetBoolean("assureUTF8"); } set { this.SetCacheValue("assureUTF8", value); } } /// /// Gets the block signature database for this index. /// public string BlockSignatureDatabase { get { return this.GetString("blockSignatureDatabase"); } } /// /// Gets or sets the block sign size for this index. This value defines /// the number of events that make up a block for block signatures. A /// value of 0 means block signing is disabled. /// public int BlockSignSize { get { return this.GetInteger("blockSignSize"); } set { this.SetCacheValue("blockSignSize", value); } } /// /// Gets the total size of all bloom filter files. /// public int BloomfilterTotalSizeKB { get { return this.GetInteger("bloomfilterTotalSizeKB", 0); } } /// /// Gets or sets the suggestion Splunk bucket rebuild process for the /// size of the time-series (tsidx) file to make. /// Caution: This is an advanced parameter. Inappropriate use of this /// parameter causes splunkd to not start if rebuild is required. Do not /// set this parameter unless instructed by Splunk Support. /// This is introduced in Splunk 5.0. The default is /// "auto". /// public string BucketRebuildMemoryHint { get { return this.GetString("bucketRebuildMemoryHint", null); } set { this.SetCacheValue("bucketRebuildMemoryHint", value); } } /// /// Gets the absolute file path to the cold database for this index. /// This value may contain shell expansion terms. /// public string ColdPath { get { return this.GetString("coldPath", null); } } /// /// Gets the expanded absolute file path to the cold database for this /// index. /// public string ColdPathExpanded { get { return this.GetString("coldPath_expanded", null); } } /// /// Gets or sets the frozen archive destination path for this index. /// public string ColdToFrozenDir { get { return this.GetString("coldToFrozenDir", null); } set { this.SetCacheValue("coldToFrozenDir", value); } } /// /// Gets or sets the path to the archiving script. /// public string ColdToFrozenScript { get { return this.GetString("coldToFrozenScript", null); } set { this.SetCacheValue("coldToFrozenScript", value); } } /// /// Gets a value indicating whether the raw data is compressed. /// public bool CompressRawdata { get { return this.GetBoolean("compressRawdata"); } } /// /// Gets the current size of this index. /// public int CurrentDBSizeMB { get { return this.GetInteger("currentDBSizeMB"); } } /// /// Gets the default index name of the Splunk instance. /// public string DefaultDatabase { get { return this.GetString("defaultDatabase"); } } /// /// Gets or sets a value indicating whether asnychronous "online fsck" /// bucket repair is enabled. When this feature is enabled, you don't /// have to wait for buckets to be repaired before starting Splunk, but /// one might notice a slight degradation in performance as a result. /// public bool EnableOnlineBucketRepair { get { return this.GetBoolean("enableOnlineBucketRepair"); } set { this.SetCacheValue("enableOnlineBucketRepair", value); } } /// /// Gets a value indicating whether real-time search is enabled for this /// index. /// public bool EnableRealtimeSearch { get { return this.GetBoolean("enableRealtimeSearch"); } } /// /// Gets or sets the maximum age for a bucket, after which the data in /// this index rolls to frozen. If archiving is necessary for frozen /// data, see the coldToFrozen attributes. /// public int FrozenTimePeriodInSecs { get { return this.GetInteger("frozenTimePeriodInSecs"); } set { this.SetCacheValue("frozenTimePeriodInSecs", value); } } /// /// Gets the absolute path to both hot and warm buckets for this index. /// This value may contain shell expansion terms. /// public string HomePath { get { return this.GetString("homePath", null); } } /// /// Gets the expanded absolute path to both hot and warm buckets for /// this index. /// public string HomePathExpanded { get { return this.GetString("homePath_expanded", null); } } /// /// Gets the number of threads for this index. Note can be "auto" /// instead of a number. /// public string IndexThreads { get { return this.GetString("indexThreads"); } } /// /// Gets the last initialization time for this index. /// public DateTime LastInitTime { get { return this.GetDate("lastInitTime", DateTime.MaxValue); } } /// /// Gets or sets if a warm or cold bucket is older than the specified /// age, do not create or rebuild its bloomfilter. Specify 0 to never /// rebuild bloomfilters. /// public string MaxBloomBackfillBucketAge { get { return this.GetString("maxBloomBackfillBucketAge"); } set { this.SetCacheValue("maxBloomBackfillBucketAge", value); } } /// /// Gets or sets the maximum number of concurrent optimize processes /// that can run against a hot bucket for this index. /// public int MaxConcurrentOptimizes { get { return this.GetInteger("maxConcurrentOptimizes"); } set { this.SetCacheValue("maxConcurrentOptimizes", value); } } /// /// Gets or sets the maximum data size before triggering a roll from /// hot to warm buckets for this index. Note that the maximum data size, /// in MB, or "auto" (which means 750MB), or "auto_high_volume" (which /// means 10GB on a 64-bit system, or 1GB on a 32-bit system). /// public string MaxDataSize { get { return this.GetString("maxDataSize"); } set { this.SetCacheValue("maxDataSize", value); } } /// /// Gets or sets the maximum number of hot buckets that can exist for /// this index. /// public int MaxHotBuckets { get { return this.GetInteger("maxHotBuckets"); } set { this.SetCacheValue("maxHotBuckets", value); } } /// /// Gets or sets the maximum lifetime of a hot bucket for this index. /// If a hot bucket exceeds this value, Splunk rolls it to warm. /// A value of 0 means an infinite lifetime. /// public int MaxHotIdleSecs { get { return this.GetInteger("maxHotIdleSecs"); } set { this.SetCacheValue("maxHotIdleSecs", value); } } /// /// Gets or sets the upper bound of the target maximum timespan of /// hot and warm buckets for this index. /// public int MaxHotSpanSecs { get { return this.GetInteger("maxHotSpanSecs"); } set { this.SetCacheValue("maxHotSpanSecs", value); } } /// /// Gets or sets the amount of memory to allocate for buffering /// a single .tsidx file into memory before flushing to disk. /// public int MaxMemMB { get { return this.GetInteger("maxMemMB"); } set { this.SetCacheValue("maxMemMB", value); } } /// /// Gets or sets the maximum number of unique lines that are allowed /// in a bucket's .data files for this index. A value of 0 means /// infinite lines. /// public int MaxMetaEntries { get { return this.GetInteger("maxMetaEntries"); } set { this.SetCacheValue("maxMetaEntries", value); } } /// /// Gets the maximum number of concurrent helper processes for this /// index. /// public int MaxRunningProcessGroups { get { return this.GetInteger("maxRunningProcessGroups", 0); } } /// /// Gets timestamp of the newest event time in the index. If not /// available, the Systems concept of minimum time is returned. /// public DateTime MaxTime { get { return this.GetDate("maxTime", DateTime.MinValue); } } /// /// Gets or sets the Upper limit, in seconds, on how long an event can /// sit in raw slice. Applies only if replication is enabled for this /// index, otherwise is ignored. If there are any acknowledged events /// sharing this raw slice, this paramater does not apply. In this case, /// maxTimeUnreplicatedWithAcks applies. Highest legal value is /// 2147483647. To disable this parameter, set to 0. /// Note: this is an advanced parameter. Understand the consequences /// before changing. This is introduced in Splunk 5.0. /// public int MaxTimeUnreplicatedNoAcks { get { return this.GetInteger("maxTimeUnreplicatedNoAcks", -1); } set { this.SetCacheValue("maxTimeUnreplicatedNoAcks", value); } } /// /// Gets or sets the maximum size for this index. If an index /// grows larger than this value, the oldest data is frozen. /// public int MaxTotalDataSizeMB { get { return this.GetInteger("maxTotalDataSizeMB"); } set { this.SetCacheValue("maxTotalDataSizeMB", value); } } /// /// Gets or sets the maximum number of warm buckets for this index. If /// this value is exceeded, the warm buckets with the lowest value for /// their latest times are moved to cold. /// public int MaxWarmDBCount { get { return this.GetInteger("maxWarmDBCount"); } set { this.SetCacheValue("maxWarmDBCount", value); } } /// /// Gets the memory pool size for this index. Note that the value can /// be a number or "auto". /// public string MemPoolMB { get { return this.GetString("memPoolMB"); } } /// /// Gets or sets the frequency at which Splunkd forces a filesystem sync /// while compressing journal slices for this index. A value of /// "disable" disables this feature completely, while a value of 0 /// forces a file-system sync after completing compression of every /// journal slice. /// public string MinRawFileSyncSecs { get { return this.GetString("minRawFileSyncSecs"); } set { this.SetCacheValue("minRawFileSyncSecs", value); } } /// /// Gets the timestamp of the oldest event time in the index. If not /// available, the Systems concept of maximum time is returned. /// public DateTime MinTime { get { return this.GetDate("minTime", DateTime.MaxValue); } } /// /// Gets the number of hot buckets that were created for this index. /// public int NumHotBuckets { get { return this.GetInteger("numHotBuckets", 0); } } /// /// Gets the umber of warm buckets created for this index. /// public int NumWarmBuckets { get { return this.GetInteger("numWarmBuckets", 0); } } /// /// Gets the number of bloom filters created for this index. /// public int NumBloomfilters { get { return this.GetInteger("numBloomfilters", 0); } } /// /// Gets or sets the frequency, in seconds, at which metadata is for /// partially synced (synced in-place) for this index. A value of 0 /// disables partial syncing, so metadata is only synced on the /// ServiceMetaPeriod interval. /// public int PartialServiceMetaPeriod { get { return this.GetInteger("partialServiceMetaPeriod"); } set { this.SetCacheValue("partialServiceMetaPeriod", value); } } /// /// Gets or sets the future event-time quarantine for this index. Events /// that are newer than now plus this value are quarantined. /// public int QuarantineFutureSecs { get { return this.GetInteger("quarantineFutureSecs"); } set { this.SetCacheValue("quarantineFutureSecs", value); } } /// /// Gets or sets the past event-time quarantine for this index. Events /// that are older than now minus this value are quarantined. /// public int QuarantinePastSecs { get { return this.GetInteger("quarantinePastSecs"); } set { this.SetCacheValue("quarantinePastSecs", value); } } /// /// Gets or sets the target uncompressed size of individual raw slices /// in the raw data journal for this index. WARNING:This is an advanced /// parameter. Only change it if you are instructed to do so by Splunk /// Support. /// public int RawChunkSizeBytes { get { return this.GetInteger("rawChunkSizeBytes"); } set { this.SetCacheValue("rawChunkSizeBytes", value); } } /// /// Gets or sets the replication factor. Valid Values are either a /// non-negative number or "auto." This parameter only applies to Splunk /// clustering slaves. "auto": Use the value as configured with the /// master. "0": Specify zero to turn off replication for this index. /// This is introduced in Splunk 5.0. /// public string ReplicationFactor { get { return this.GetString("repFactor", null); } set { this.SetCacheValue("repFactor", value); } } /// /// Gets or sets the frequency to check for the need to create a new hot /// bucket and the need to roll or freeze any warm or cold buckets for /// this index. /// public int RotatePeriodInSecs { get { return this.GetInteger("rotatePeriodInSecs"); } set { this.SetCacheValue("rotatePeriodInSecs", value); } } /// /// Gets or sets the frequency at which metadata is synced to disk for /// this index. /// public int ServiceMetaPeriod { get { return this.GetInteger("serviceMetaPeriod"); } set { this.SetCacheValue("serviceMetaPeriod", value); } } /// /// Gets a comma separated list of indexes that suppress "index missing" /// messages. /// public string SuppressBannerList { get { return this.GetString("suppressBannerList", null); } } /// /// Gets a value that specifies the number of events that trigger the /// indexer to sync events. /// public int Sync { get { return this.GetInteger("sync"); } } /// /// Gets or sets a value indicating whether the sync operation is /// invoked before the file descriptor is closed on metadata updates. /// WARNING:This is an advanced parameter. Only change it if you are /// instructed to do so by Splunk Support. /// public bool SyncMeta { get { return this.GetBoolean("syncMeta"); } set { this.SetCacheValue("syncMeta", value); } } /// /// Gets the absolute path to the thawed index for this index. This /// value may contain shell expansion terms. /// public string ThawedPath { get { return this.GetString("thawedPath", null); } } /// /// Gets the expanded absolute path to the thawed index for this index. /// public string ThawedPathExpanded { get { return this.GetString("thawedPath_expanded", null); } } /// /// Gets or sets the frequency at which Splunk checks for an index /// throttling condition. /// public int ThrottleCheckPeriod { get { return this.GetInteger("throttleCheckPeriod"); } set { this.SetCacheValue("throttleCheckPeriod", value); } } /// /// Gets the total event count for this index. /// public int TotalEventCount { get { return this.GetInteger("totalEventCount"); } } /// /// Gets a value indicating whether this index is an internal index. /// public bool IsInternal { get { return this.GetBoolean("isInternal"); } } /// /// Creates a writable socket to this index. /// /// The network stream public Stream Attach() { Receiver receiver = this.Service.GetReceiver(); return receiver.Attach(this.Name); } /// /// Creates a writable socket to this index, adding optional arguments. /// /// The arguments /// The network stream public Stream Attach(Args args) { Receiver receiver = this.Service.GetReceiver(); return receiver.Attach(this.Name, args); } /// /// Cleans this index removing all events, with specific timeout value. /// -1 Waits forever. /// /// The maximum number of seconds to wait /// before returning /// The index public Index Clean(int maxSeconds) { Args saved = new Args(); saved.Add("maxTotalDataSizeMB", this.MaxTotalDataSizeMB); saved.Add("frozenTimePeriodInSecs", this.FrozenTimePeriodInSecs); Args reset = new Args(); reset.Add("maxTotalDataSizeMB", "1"); reset.Add("frozenTimePeriodInSecs", "1"); this.Update(reset); this.RollHotBuckets(); while (maxSeconds != 0) { Thread.Sleep(1000); // 1000ms (1 second sleep) maxSeconds = maxSeconds - 1; if (this.TotalEventCount == 0) { this.Update(saved); return this; } this.Refresh(); } throw new SplunkException( SplunkException.TIMEOUT, "Index cleaning timed out"); } /// /// Performs rolling hot buckets for this index. /// public void RollHotBuckets() { ResponseMessage response = this.Service.Post(this.Path + "/roll-hot-buckets"); Debug.Assert(response.Status == 200, "Status not 200!"); } /// /// Submits an event to this index through an HTTP POST request. /// /// The event data to submit public void Submit(string data) { Receiver receiver = this.Service.GetReceiver(); receiver.Submit(this.Name, data); } /// /// Submits an event to this index through an HTTP POST request. /// /// The optional arguments /// The event data to submit public void Submit(Args args, string data) { Receiver receiver = this.Service.GetReceiver(); receiver.Submit(this.Name, args, data); } /// /// Uploads a file to this index as an event stream. Note: This file /// must be directly accessible by the Splunk server, by the file-path /// supplied. /// /// The file public void Upload(string filepath) { EntityCollection uploads = this.Service.GetUploads(); Args args = new Args("index", this.Name); uploads.Create(filepath, args); } } }