druid.io kafka-Index-service.json任务

最新推荐文章于 2025-06-03 09:03:13 发布

原创最新推荐文章于 2025-06-03 09:03:13 发布 · 1.2k 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#druid.io

Druid.io 同时被 2 个专栏收录

19 篇文章

订阅专栏

druid.io

16 篇文章

订阅专栏

supervisor-spec.json如下:

{
  "type": "kafka",
  "dataSchema": {
    "dataSource": "metrics-kafka",
    "parser": {
      "type": "string",
      "parseSpec": {
        "format": "json",
        "timestampSpec": {
          "column": "timestamp",
          "format": "auto"
        },
        "dimensionsSpec": {
          "dimensions": [],
          "dimensionExclusions": [
            "timestamp",
            "value"
          ]
        }
      }
    },
    "metricsSpec": [
      {
        "name": "count",
        "type": "count"
      },
      {
        "name": "value_sum",
        "fieldName": "value",
        "type": "doubleSum"
      },
      {
        "name": "value_min",
        "fieldName": "value",
        "type": "doubleMin"
      },
      {
        "name": "value_max",
        "fieldName": "value",
        "type": "doubleMax"
      }
    ],
    "granularitySpec": {
      "type": "uniform",
      "segmentGranularity": "HOUR",
      "queryGranularity": "NONE"
    }
  },
  "tuningConfig": {
    "type": "kafka",
    "maxRowsPerSegment": 5000000
  },
  "ioConfig": {
    "topic": "metrics",
    "consumerProperties": {
      "bootstrap.servers": "localhost:9092"
    },
    "taskCount": 1,
    "replicas": 1,
    "taskDuration": "PT1H"
  }
}

创建Kafka-index-service任务

curl -X POST -H 'Content-Type: application/json' -d @supervisor-spec.json http://192.168.165.130:8090/druid/indexer/v1/supervisor

返回结果:

{"id":"metrics-kafka"}

获取supervisor的描述文件:

curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka

返回结果集:

{"id":"metrics-kafka"}[root@emr-header-1 kafka-index-service]# curl -X GET -H 'Content-Type: application/json' -d @supervisor-spec1.json http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka
{"type":"kafka","dataSchema":{"dataSource":"metrics-kafka","parser":{"type":"string","parseSpec":{"format":"json","timestampSpec":{"column":"timestamp","format":"auto"},"dimensionsSpec":{"dimensions":[],"dimensionExclusions":["timestamp","value"]}}},"metricsSpec":[{"type":"count","name":"count"},{"type":"doubleSum","name":"value_sum","fieldName":"value","expression":null},{"type":"doubleMin","name":"value_min","fieldName":"value","expression":null},{"type":"doubleMax","name":"value_max","fieldName":"value","expression":null}],"granularitySpec":{"type":"uniform","segmentGranularity":"HOUR","queryGranularity":{"type":"none"},"rollup":true,"intervals":null}},"tuningConfig":{"type":"kafka","maxRowsInMemory":75000,"maxRowsPerSegment":5000000,"intermediatePersistPeriod":"PT10M","basePersistDirectory":"/opt/apps/ecm/service/druid/0.11.0-1.2.0/package/druid-0.11.0-1.2.0/var/tmp/1534915930015-0","maxPendingPersists":0,"indexSpec":{"bitmap":{"type":"concise"},"dimensionCompression":"lz4","metricCompression":"lz4","longEncoding":"longs"},"buildV9Directly":true,"reportParseExceptions":false,"handoffConditionTimeout":0,"resetOffsetAutomatically":false,"workerThreads":null,"chatThreads":null,"chatRetries":8,"httpTimeout":"PT10S","shutdownTimeout":"PT80S","offsetFetchPeriod":"PT30S"},"ioConfig":{"topic":"metrics","replicas":1,"taskCount":1,"taskDuration":"PT3600S","consumerProperties":{"bootstrap.servers":"localhost:9092"},"startDelay":"PT5S","period":"PT30S","useEarliestOffset":false,"completionTimeout":"PT1800S","lateMessageRejectionPeriod":null,"earlyMessageRejectionPeriod":null,"skipOffsetGaps":false},"context":null}

获取kafka-index-service任务状态:

curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka/status

返回结果:

{"id":"metrics-kafka","generationTime":"2018-08-22T05:33:59.253Z","payload":{"dataSource":"metrics-kafka","topic":"metrics","partitions":0,"replicas":1,"durationSeconds":3600,"activeTasks":[],"publishingTasks":[],"minimumLag":{},"aggregateLag":0,"offsetsLastUpdated":null}}

查询kafaka-index-service任务的历史记录:

curl -X GET http://192.168.165.130:18090/druid/indexer/v1/supervisor/metrics-kafka/history

返回结果:

[{"spec":{"type":"kafka","dataSchema":{"dataSource":"metrics-kafka","parser":{"type":"string","parseSpec":{"format":"json","timestampSpec":{"column":"timestamp","format":"auto"},"dimensionsSpec":{"dimensions":[],"dimensionExclusions":["timestamp","value"]}}},"metricsSpec":[{"type":"count","name":"count"},{"type":"doubleSum","name":"value_sum","fieldName":"value","expression":null},{"type":"doubleMin","name":"value_min","fieldName":"value","expression":null},{"type":"doubleMax","name":"value_max","fieldName":"value","expression":null}],"granularitySpec":{"type":"uniform","segmentGranularity":"HOUR","queryGranularity":{"type":"none"},"rollup":true,"intervals":null}},"tuningConfig":{"type":"kafka","maxRowsInMemory":75000,"maxRowsPerSegment":5000000,"intermediatePersistPeriod":"PT10M","basePersistDirectory":"/opt/apps/ecm/service/druid/0.11.0-1.2.0/package/druid-0.11.0-1.2.0/var/tmp/1534915930015-0","maxPendingPersists":0,"indexSpec":{"bitmap":{"type":"concise"},"dimensionCompression":"lz4","metricCompression":"lz4","longEncoding":"longs"},"buildV9Directly":true,"reportParseExceptions":false,"handoffConditionTimeout":0,"resetOffsetAutomatically":false,"workerThreads":null,"chatThreads":null,"chatRetries":8,"httpTimeout":"PT10S","shutdownTimeout":"PT80S","offsetFetchPeriod":"PT30S"},"ioConfig":{"topic":"metrics","replicas":1,"taskCount":1,"taskDuration":"PT3600S","consumerProperties":{"bootstrap.servers":"localhost:9092"},"startDelay":"PT5S","period":"PT30S","useEarliestOffset":false,"completionTimeout":"PT1800S","lateMessageRejectionPeriod":null,"earlyMessageRejectionPeriod":null,"skipOffsetGaps":false},"context":null},"version":"2018-08-22T05:32:10.018Z"}]