add deduplicate processor ref doc (DataDog#25144)

maycmlee · web-flow · commit fcee55be70e7 · 2024-09-09T13:46:08.000-07:00
diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml
@@ -4667,51 +4667,56 @@ menu:
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_add_hostname
       weight: 601
+    - name: Deduplicate
+      url: observability_pipelines/processors/dedupe
+      parent: observability_pipelines_processors
+      identifier: observability_pipelines_processors_dedupe
+      weight: 602
     - name: Edit fields
       url: observability_pipelines/processors/edit_fields
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_edit_fields
-      weight: 602
+      weight: 604
     - name: Enrichment Table
       url: observability_pipelines/processors/enrichment_table
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_enrichment_table
-      weight: 603
+      weight: 605
     - name: Filter
       url: observability_pipelines/processors/filter
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_filter
-      weight: 604
+      weight: 606
     - name: Grok Parser
       url: observability_pipelines/processors/grok_parser
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_grok_parser
-      weight: 605
+      weight: 607
     - name: Parse JSON
       url: observability_pipelines/processors/parse_json
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_parse_json
-      weight: 606
+      weight: 608
     - name: Quota
       url: observability_pipelines/processors/quota
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_quota
-      weight: 607
+      weight: 609
     - name: Reduce
       url: observability_pipelines/processors/reduce
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_reduce
-      weight: 608
+      weight: 610
     - name: Sample
       url: observability_pipelines/processors/sample
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_sample
-      weight: 609
+      weight: 611
     - name: Sensitive Data Scanner
       url: observability_pipelines/processors/sensitive_data_scanner
       parent: observability_pipelines_processors
       identifier: observability_pipelines_processors_sensitive_data_scanner
-      weight: 610
+      weight: 612
     - name: Destinations
       url: observability_pipelines/destinations/
       parent: observability_pipelines
diff --git a/content/en/observability_pipelines/processors/_index.md b/content/en/observability_pipelines/processors/_index.md
@@ -27,6 +27,7 @@ Use Observability Pipelines' processors to parse, structure, and enrich your log
 
 {{< whatsnext desc=" " >}}
     {{< nextlink href="observability_pipelines/processors/add_hostname" >}}Add hostname{{< /nextlink >}}
+    {{< nextlink href="observability_pipelines/processors/dedupe" >}}Deduplicate{{< /nextlink >}}
     {{< nextlink href="observability_pipelines/processors/edit_fields" >}}Edit fields{{< /nextlink >}}
     {{< nextlink href="observability_pipelines/processors/enrichment_table" >}}Enrichment table{{< /nextlink >}}
     {{< nextlink href="observability_pipelines/processors/filter" >}}Filter{{< /nextlink >}}
diff --git a/content/en/observability_pipelines/processors/dedupe.md b/content/en/observability_pipelines/processors/dedupe.md
@@ -0,0 +1,8 @@
+---
+title: Deduplicate Processor
+disable_toc: false
+---
+
+{{% observability_pipelines/processors/dedupe %}}
+
+{{% observability_pipelines/processors/filter_syntax %}}
diff --git a/layouts/shortcodes/observability_pipelines/processors/dedupe.md b/layouts/shortcodes/observability_pipelines/processors/dedupe.md
@@ -1,4 +1,4 @@
-The dedupe processor removes copies of data to reduce volume and noise. It caches 5000 messages at a time and compares your incoming logs traffic against the cached messages. For example, this processor can be used to keep only unique warning logs in the case where multiple identical warning logs are sent in succession.
+The deduplicate processor removes copies of data to reduce volume and noise. It caches 5,000 messages at a time and compares your incoming logs traffic against the cached messages. For example, this processor can be used to keep only unique warning logs in the case where multiple identical warning logs are sent in succession.
 
 To set up the deduplicate processor:
 1. Define a **filter query**. Only logs that match the specified [filter query](#filter-query-syntax) are processed. Deduped logs and logs that do not match the filter query are sent to the next step in the pipeline.

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-The dedupe processor removes copies of data to reduce volume and noise. It caches 5000 messages at a time and compares your incoming logs traffic against the cached messages. For example, this processor can be used to keep only unique warning logs in the case where multiple identical warning logs are sent in succession.`
	`1`	`+The deduplicate processor removes copies of data to reduce volume and noise. It caches 5,000 messages at a time and compares your incoming logs traffic against the cached messages. For example, this processor can be used to keep only unique warning logs in the case where multiple identical warning logs are sent in succession.`
`2`	`2`
`3`	`3`	`To set up the deduplicate processor:`
`4`	`4`	`1. Define a filter query. Only logs that match the specified [filter query](#filter-query-syntax) are processed. Deduped logs and logs that do not match the filter query are sent to the next step in the pipeline.`