Log to Metric

Example Configuration

Counter

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "counter"
6  field = "status"
7  name = "response_total"
8  namespace = "service"
9
10    [transforms.my_transform_id.metrics.tags]
11    status = "{{status}}"
12    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "Sent 200 in 54.2ms",
5    "status": 200
6  }
7}

1[
2  {
3    "metric": {
4      "kind": "incremental",
5      "name": "response_total",
6      "namespace": "service",
7      "tags": {
8        "status": "200",
9        "host": "10.22.11.222"
10      },
11      "counter": {
12        "value": 1
13      }
14    }
15  }
16]

Sum

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "counter"
6  field = "total"
7  name = "order_total"
8  increment_by_value = true
9
10    [transforms.my_transform_id.metrics.tags]
11    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "Order placed for $122.20",
5    "total": 122.2
6  }
7}

1[
2  {
3    "metric": {
4      "kind": "incremental",
5      "name": "order_total",
6      "tags": {
7        "host": "10.22.11.222"
8      },
9      "counter": {
10        "value": 122.2
11      }
12    }
13  }
14]

Gauges

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "gauge"
6  field = "1m_load_avg"
7
8    [transforms.my_transform_id.metrics.tags]
9    host = "{{host}}"
10
11  [[transforms.my_transform_id.metrics]]
12  type = "gauge"
13  field = "5m_load_avg"
14
15    [transforms.my_transform_id.metrics.tags]
16    host = "{{host}}"
17
18  [[transforms.my_transform_id.metrics]]
19  type = "gauge"
20  field = "15m_load_avg"
21
22    [transforms.my_transform_id.metrics.tags]
23    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "CPU activity sample",
5    "1m_load_avg": 78.2,
6    "5m_load_avg": 56.2,
7    "15m_load_avg": 48.7
8  }
9}

1[
2  {
3    "metric": {
4      "kind": "absolute",
5      "name": "1m_load_avg",
6      "tags": {
7        "host": "10.22.11.222"
8      },
9      "gauge": {
10        "value": 78.2
11      }
12    }
13  },
14  {
15    "metric": {
16      "kind": "absolute",
17      "name": "5m_load_avg",
18      "tags": {
19        "host": "10.22.11.222"
20      },
21      "gauge": {
22        "value": 56.2
23      }
24    }
25  },
26  {
27    "metric": {
28      "kind": "absolute",
29      "name": "15m_load_avg",
30      "tags": {
31        "host": "10.22.11.222"
32      },
33      "gauge": {
34        "value": 48.7
35      }
36    }
37  }
38]

Histogram distribution

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "histogram"
6  field = "time"
7  name = "time_ms"
8
9    [transforms.my_transform_id.metrics.tags]
10    status = "{{status}}"
11    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "Sent 200 in 54.2ms",
5    "status": 200,
6    "time": 54.2
7  }
8}

1[
2  {
3    "metric": {
4      "kind": "incremental",
5      "name": "time_ms",
6      "tags": {
7        "status": "200",
8        "host": "10.22.11.222"
9      },
10      "distribution": {
11        "samples": [
12          {
13            "value": 54.2,
14            "rate": 1
15          }
16        ],
17        "statistic": "histogram"
18      }
19    }
20  }
21]

Summary distribution

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "summary"
6  field = "time"
7  name = "time_ms"
8
9    [transforms.my_transform_id.metrics.tags]
10    status = "{{status}}"
11    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "Sent 200 in 54.2ms",
5    "status": 200,
6    "time": 54.2
7  }
8}

1[
2  {
3    "metric": {
4      "kind": "incremental",
5      "name": "time_ms",
6      "tags": {
7        "status": "200",
8        "host": "10.22.11.222"
9      },
10      "distribution": {
11        "samples": [
12          {
13            "value": 54.2,
14            "rate": 1
15          }
16        ],
17        "statistic": "summary"
18      }
19    }
20  }
21]

Set

Config

Input

Output

1[transforms.my_transform_id]
2type = "log_to_metric"
3
4  [[transforms.my_transform_id.metrics]]
5  type = "set"
6  field = "remote_addr"
7  namespace = "{{branch}}"
8
9    [transforms.my_transform_id.metrics.tags]
10    host = "{{host}}"

1{
2  "log": {
3    "host": "10.22.11.222",
4    "message": "Sent 200 in 54.2ms",
5    "remote_addr": "233.221.232.22",
6    "branch": "dev"
7  }
8}

1[
2  {
3    "metric": {
4      "kind": "incremental",
5      "name": "remote_addr",
6      "namespace": "dev",
7      "tags": {
8        "host": "10.22.11.222"
9      },
10      "set": {
11        "values": [
12          "233.221.232.22"
13        ]
14      }
15    }
16  }
17]

Configuration Options

Required Options

metrics(required)

A table of key/value pairs representing the keys to be added to the event.

Type	Syntax	Default	Example
array	literal		[]

inputs(required)

A list of upstream source or transform IDs. Wildcards (*) are supported.

See configuration for more info.

Type	Syntax	Default	Example
array	literal		["my-source-or-transform-id","prefix-*"]

type(required)

The component type. This is a required field for all components and tells Vector which component to use.

Type	Syntax	Default	Example
string	literal		["log_to_metric"]

Advanced Options

How it Works

Multiple Metrics

For clarification, when you convert a single log event into multiple metric events, the metric events are not emitted as a single array. They are emitted individually, and the downstream components treat them as individual events. Downstream components are not aware they were derived from a single log event.

Reducing

It's important to understand that this transform does not reduce multiple logs to a single metric. Instead, this transform converts logs into granular individual metrics that can then be reduced at the edge. Where the reduction happens depends on your metrics storage. For example, the prometheus_exporter sink will reduce logs in the sink itself for the next scrape, while other metrics sinks will proceed to forward the individual metrics for reduction in the metrics storage itself.

State

This component is stateless, meaning its behavior is consistent across each input.

Null Fields

If the target log field contains a null value it will ignored, and a metric will not be emitted.