Estimate job model memory usage

POST /_ml/anomaly_detectors/_estimate_model_memory

Auth

Make an estimation of the memory usage for an anomaly detection job model. The estimate is based on analysis configuration details for the job and cardinality estimates for the fields it references.

Required privileges

Cluster privileges manage_ml

Request body

analysis_config object: For a list of the properties that you can specify in the analysis_config component of the body of this API.
max_bucket_cardinality map string to number: Estimates of the highest cardinality in a single bucket that is observed for influencer fields over the time period that the job analyzes data. To produce a good answer, values must be provided for all influencer fields. Providing values for fields that are not listed as influencers has no effect on the estimation.
overall_cardinality map string to number: Estimates of the cardinality that is observed for fields over the whole time period that the job analyzes data. To produce a good answer, values must be provided for fields referenced in the by_field_name, over_field_name and partition_field_name of any detectors. Providing values for other fields has no effect on the estimation. It can be omitted from the request if no detectors have a by_field_name, over_field_name or partition_field_name.

Response

application/json

model_memory_estimate string required

Examples

Request Console 1 / 8 languages

Run POST _ml/anomaly_detectors/_estimate_model_memory to estimate the model memory limit based on the analysis configuration details provided in the request body.

Console Kibana Dev Tools

POST _ml/anomaly_detectors/_estimate_model_memory
{
  "analysis_config": {
    "bucket_span": "5m",
    "detectors": [
      {
        "function": "sum",
        "field_name": "bytes",
        "by_field_name": "status",
        "partition_field_name": "app"
      }
    ],
    "influencers": [
      "source_ip",
      "dest_ip"
    ]
  },
  "overall_cardinality": {
    "status": 10,
    "app": 50
  },
  "max_bucket_cardinality": {
    "source_ip": 300,
    "dest_ip": 30
  }
}

JavaScript Elasticsearch JavaScript Client

const response = await client.ml.estimateModelMemory({
  analysis_config: {
    bucket_span: "5m",
    detectors: [
      {
        function: "sum",
        field_name: "bytes",
        by_field_name: "status",
        partition_field_name: "app",
      },
    ],
    influencers: ["source_ip", "dest_ip"],
  },
  overall_cardinality: {
    status: 10,
    app: 50,
  },
  max_bucket_cardinality: {
    source_ip: 300,
    dest_ip: 30,
  },
});

Python Elasticsearch Python Client

resp = client.ml.estimate_model_memory(
    analysis_config={
        "bucket_span": "5m",
        "detectors": [
            {
                "function": "sum",
                "field_name": "bytes",
                "by_field_name": "status",
                "partition_field_name": "app"
            }
        ],
        "influencers": [
            "source_ip",
            "dest_ip"
        ]
    },
    overall_cardinality={
        "status": 10,
        "app": 50
    },
    max_bucket_cardinality={
        "source_ip": 300,
        "dest_ip": 30
    },
)

curl Shell

curl -X POST "$ELASTICSEARCH_URL/_ml/anomaly_detectors/_estimate_model_memory" \
  -H "Authorization: ApiKey $ELASTIC_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "analysis_config": {
      "bucket_span": "5m",
      "detectors": [
        {
          "function": "sum",
          "field_name": "bytes",
          "by_field_name": "status",
          "partition_field_name": "app"
        }
      ],
      "influencers": [
        "source_ip",
        "dest_ip"
      ]
    },
    "overall_cardinality": {
      "status": 10,
      "app": 50
    },
    "max_bucket_cardinality": {
      "source_ip": 300,
      "dest_ip": 30
    }
  }'

Java Elasticsearch Java Client

client.ml().estimateModelMemory(e -> e
    .analysisConfig(a -> a
        .bucketSpan(b -> b
            .time("5m")
        )
        .detectors(d -> d
            .byFieldName("status")
            .fieldName("bytes")
            .function("sum")
            .partitionFieldName("app")
        )
        .influencers(List.of("source_ip","dest_ip"))
    )
    .maxBucketCardinality(Map.of("dest_ip", 30L,"source_ip", 300L))
    .overallCardinality(Map.of("app", 50L,"status", 10L))
);

C# Elasticsearch .NET Client

var response = await client.MachineLearning
    .EstimateModelMemoryAsync(d1 => d1
        .AnalysisConfig(d2 => d2
            .BucketSpan("5m")
            .Detectors(d3 => d3
                .ByFieldName(x => x.Status)
                .FieldName(x => x.Bytes)
                .Function("sum")
                .PartitionFieldName(x => x.App)
            )
            .Influencers(x => x.SourceIp, x => x.DestIp)
        )
        .MaxBucketCardinality(new Dictionary<Field, long>()
        {
            { global::Elastic.Clients.Elasticsearch.Infer.Field<MyDocument>(x => x.SourceIp), 300L },
            { global::Elastic.Clients.Elasticsearch.Infer.Field<MyDocument>(x => x.DestIp), 30L }
        })
        .OverallCardinality(new Dictionary<Field, long>()
        {
            { global::Elastic.Clients.Elasticsearch.Infer.Field<MyDocument>(x => x.Status), 10L },
            { global::Elastic.Clients.Elasticsearch.Infer.Field<MyDocument>(x => x.App), 50L }
        })
    );

PHP Elasticsearch PHP Client

$resp = $client->ml()->estimateModelMemory([
    "body" => [
        "analysis_config" => [
            "bucket_span" => "5m",
            "detectors" => array(
                [
                    "function" => "sum",
                    "field_name" => "bytes",
                    "by_field_name" => "status",
                    "partition_field_name" => "app",
                ],
            ),
            "influencers" => array(
                "source_ip",
                "dest_ip",
            ),
        ],
        "overall_cardinality" => [
            "status" => 10,
            "app" => 50,
        ],
        "max_bucket_cardinality" => [
            "source_ip" => 300,
            "dest_ip" => 30,
        ],
    ],
]);

Ruby Elasticsearch Ruby Client

response = client.ml.estimate_model_memory(
  body: {
    "analysis_config": {
      "bucket_span": "5m",
      "detectors": [
        {
          "function": "sum",
          "field_name": "bytes",
          "by_field_name": "status",
          "partition_field_name": "app"
        }
      ],
      "influencers": [
        "source_ip",
        "dest_ip"
      ]
    },
    "overall_cardinality": {
      "status": 10,
      "app": 50
    },
    "max_bucket_cardinality": {
      "source_ip": 300,
      "dest_ip": 30
    }
  }
)

Response

A successful response from POST _ml/anomaly_detectors/_estimate_model_memory.

{
  "model_memory_estimate": "21mb"
}