logstash多个mysql的input导出到多个output

最新推荐文章于 2022-10-28 16:02:26 发布

小龙在山东

最新推荐文章于 2022-10-28 16:02:26 发布

阅读量6.5k

点赞数

CC 4.0 BY-SA版权

分类专栏： elasticsearch

本文链接：https://blog.youkuaiyun.com/lilongsy/article/details/78283122

elasticsearch 专栏收录该内容

18 篇文章

订阅专栏

本文介绍了一种使用Logstash从MySQL数据库中提取数据并同步到Elasticsearch的方法。通过两个具体的例子展示了如何配置Logstash输入插件来实现增量数据抓取，并根据不同类型的数据设置不同的输出格式。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

input {
    jdbc {
      jdbc_driver_library => "mysql-connector-java-5.1.33-bin.jar"
      jdbc_driver_class => "com.mysql.jdbc.Driver"
      jdbc_user => "usr"
      jdbc_password=> "pwd"
      jdbc_connection_string => "jdbc:mysql://localhost:3306/db"
      jdbc_validate_connection => true
      schedule => "* * * * *"
      use_column_value => true
      tracking_column => "id"
      last_run_metadata_path => "/home/www/.logstash_headlines_last_run"
      statement => "SELECT * FROM tb WHERE id > :sql_last_value AND content <> '' ORDER BY id ASC"
      jdbc_paging_enabled => true
      jdbc_page_size => 10000
      type => "headlines"
    }

    jdbc {
      jdbc_driver_library => "mysql-connector-java-5.1.33-bin.jar"
      jdbc_driver_class => "com.mysql.jdbc.Driver"
      jdbc_user => "usr"
      jdbc_password=> "pwd"
      jdbc_connection_string => "jdbc:mysql://localhost:3306/db"
      jdbc_validate_connection => true
      schedule => "* * * * *"
      use_column_value => true
      tracking_column => "id"
      last_run_metadata_path => "/home/www/.logstash_vms_last_run"
      statement => "SELECT a.id, a.title, d.content FROM vms_article a JOIN vms_article_data d ON a.id=d.id WHERE a.id > :sql_last_value AND a.status=100 ORDER BY id ASC"
      jdbc_paging_enabled => true
      jdbc_page_size => 10000
      type => "article"
    }
}
output {
    if [type] == "headlines" {
        elasticsearch {
          hosts => "localhost"
          index => "collection"
          document_id => "%{id}"
        }   
        stdout {
            codec => line {
                    format => "Crawl: %{id} %{title}"
            }   
        }
    } else {
        elasticsearch {
          hosts => "localhost"
          index => "vms"
          document_id => "%{id}"
        }    
        stdout {
            codec => line {
                format => "VMS: %{id} %{title}"
            }   
        }
    }
}