小文件治理之hadoop集群监控：文件目录个数 获取的三种方法

前言

总结一下获取文件目录数的方法

---

一、50070页面

二、jmx获取

jmx的服务端是servlet，通过httpClient调用就行

[root@hadoop101 software]# curl hadoop101:50070/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo
{
  "beans" : [ {
    "name" : "Hadoop:service=NameNode,name=NameNodeInfo",
    "modelerType" : "org.apache.hadoop.hdfs.server.namenode.FSNamesystem",
    "Total" : 3170276474880,
    "UpgradeFinalized" : true,
    "ClusterId" : "CID-f451766f-cd7d-4ade-994c-cc5359e5a60e",
    "Version" : "2.7.2, rb165c4fe8a74265c792ce23f546c64604acf0e41",
    "Used" : 91505946150,
    "Free" : 2914452942848,
    "Safemode" : "",
    "NonDfsUsedSpace" : 164317585882,
    "PercentUsed" : 2.8863711,
    "BlockPoolUsedSpace" : 91505946150,
    "PercentBlockPoolUsed" : 2.8863711,
    "PercentRemaining" : 91.930565,
    "CacheCapacity" : 0,
    "CacheUsed" : 0,
    "TotalBlocks" : 65507,
    "TotalFiles" : 66218,
    "NumberOfMissingBlocks" : 0,
    "NumberOfMissingBlocksWithReplicationFactorOne" : 0,
    "LiveNodes" : "{\"hadoop102:50010\":{\"infoAddr\":\"10.233.65.29:50075\",\"infoSecureAddr\":\"10.233.65.29:0\",\"xferaddr\":\"10.233.65.29:50010\",\"lastContact\":1,\"usedSpace\":26866655231,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54853758977,\"capacity\":1056758824960,\"numBlocks\":40849,\"version\":\"2.7.2\",\"used\":26866655231,\"remaining\":975038410752,\"blockScheduled\":0,\"blockPoolUsed\":26866655231,\"blockPoolUsedPercent\":2.5423641,\"volfails\":0},\"hadoop101:50010\":{\"infoAddr\":\"10.233.65.15:50075\",\"infoSecureAddr\":\"10.233.65.15:0\",\"xferaddr\":\"10.233.65.15:50010\",\"lastContact\":2,\"usedSpace\":23621656822,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54771056394,\"capacity\":1056758824960,\"numBlocks\":53006,\"version\":\"2.7.2\",\"used\":23621656822,\"remaining\":978366111744,\"blockScheduled\":0,\"blockPoolUsed\":23621656822,\"blockPoolUsedPercent\":2.2352931,\"volfails\":0},\"hadoop103:50010\":{\"infoAddr\":\"10.233.65.40:50075\",\"infoSecureAddr\":\"10.233.65.40:0\",\"xferaddr\":\"10.233.65.40:50010\",\"lastContact\":2,\"usedSpace\":41017634097,\"adminState\":\"In Service\",\"nonDfsUsedSpace\":54692770511,\"capacity\":1056758824960,\"numBlocks\":37161,\"version\":\"2.7.2\",\"used\":41017634097,\"remaining\":961048420352,\"blockScheduled\":0,\"blockPoolUsed\":41017634097,\"blockPoolUsedPercent\":3.8814566,\"volfails\":0}}",
    "DeadNodes" : "{}",
    "DecomNodes" : "{}",
    "BlockPoolId" : "BP-1805414249-10.233.65.15-1641883882370",
    "NameDirStatuses" : "{\"active\":{\"/data/hadoop/hdfs/name\":\"IMAGE_AND_EDITS\"},\"failed\":{}}",
    "NodeUsage" : "{\"nodeUsage\":{\"min\":\"2.24%\",\"median\":\"2.54%\",\"max\":\"3.88%\",\"stdDev\":\"0.71%\"}}",
    "NameJournalStatus" : "[{\"manager\":\"QJM to [10.233.65.15:8485, 10.233.65.29:8485, 10.233.65.40:8485]\",\"stream\":\"open for read\",\"disabled\":\"false\",\"required\":\"true\"}]",
    "JournalTransactionInfo" : "{\"MostRecentCheckpointTxId\":\"4931140\",\"LastAppliedOrWrittenTxId\":\"4936276\"}",
    "NNStarted" : "Wed Jan 26 16:54:26 CST 2022",
    "CompileInfo" : "2016-01-26T00:08Z by jenkins from (detached from b165c4f)",
    "CorruptFiles" : "[]",
    "DistinctVersionCount" : 1,
    "DistinctVersions" : [ {
      "key" : "2.7.2",
      "value" : 3
    } ],
    "SoftwareVersion" : "2.7.2",
    "RollingUpgradeStatus" : null,
    "Threads" : 37
  } ]
}[

三、使用hadoop fs -count 获取

[root@hadoop101 software]#  hadoop fs -count   /
         714        65506        44795164671 /

---

总结

介绍了文件目录树可以通过web,jmx,hadoop fs -count获取