文本查重工具

最新推荐文章于 2025-05-22 10:25:24 发布
铁头大蚂蚁
最新推荐文章于 2025-05-22 10:25:24 发布
阅读量227
点赞数 3
文章标签： css css3 前端
本文链接：https://blog.youkuaiyun.com/u011739075/article/details/146158977
版权
其实excel可以实现这个功能，但是对于excel小白来说处理起来还是有点难度的
隔壁小姐姐请求我帮忙做了这个工具，贡献给大家吧
左侧输入待排查数据，一行一个，右侧可以自动计算出哪些重复重复几次
贴上代码
<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>文本查重工具</title>
    <script src="xlsx.full.min.js"></script>
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
            font-family: 'Microsoft YaHei', Arial, sans-serif;
        }
        
        body {
            background-color: #f5f7fa;
            color: #333;
            padding: 20px;
        }
        
        .container {
            display: flex;
            max-width: 1200px;
            margin: 0 auto;
            gap: 20px;
            height: calc(100vh - 320px);
        }
        
        .header {
            text-align: center;
            margin-bottom: 30px;
        }
        
        .header h1 {
            font-size: 28px;
            color: #2c3e50;
            margin-bottom: 10px;
        }
        
        .header p {
            color: #7f8c8d;
            font-size: 16px;
        }
        
        .panel {
            flex: 1;
            background: #fff;
            border: 1px solid #e0e0e0;
            box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05);
            display: flex;
            flex-direction: column;
        }
        
        .panel-header {
            padding: 15px;
            background: #2c3e50;
            color: white;
            font-weight: bold;
            display: flex;
            justify-content: space-between;
            align-items: center;
        }
        
        .panel-content {
            flex: 1;
            padding: 15px;
            overflow: hidden; /* 修改这里，从auto改为hidden */
        }
        
        textarea {
            width: 100%;
            height: 100%;
            border: none;
            resize: none;
            outline: none;
            font-size: 14px;
            line-height: 1.6;
            overflow: auto; /* 确保textarea保留滚动条 */
        }
        
        .result-list {
            list-style: none;
        }
        
        .result-list li {
            padding: 8px 10px;
            border-bottom: 1px solid #eee;
        }
        
        .result-list li:nth-child(odd) {
            background-color: #f9f9f9;
        }
        
        .controls {
            display: flex;
            justify-content: center;
            gap: 15px;
            margin-top: 20px;
        }
        
        button {
            padding: 10px 20px;
            background: #3498db;
            color: white;
            border: none;
            cursor: pointer;
            font-size: 14px;
            min-width: 120px;
            transition: background 0.3s;
        }
        
        button:hover {
            background: #2980b9;
        }
        
        .stats {
            margin-top: 15px;
            font-weight: bold;
            color: #e74c3c;
        }
        
        .export-btn {
            background: #27ae60;
        }
        
        .export-btn:hover {
            background: #219653;
        }
        
        .clear-btn {
            background: #e74c3c;
        }
        
        .clear-btn:hover {
            background: #c0392b;
        }
    </style>
</head>
<body>
    <div class="header">
        <h1>文本查重工具</h1>
        <p>在左侧输入框中一行一个输入数据，点击查重按钮分析重复内容</p>
    </div>
    
    <div class="container">
        <div class="panel">
            <div class="panel-header">
                <span>输入数据（一行一个）</span>
            </div>
            <div class="panel-content">
                <textarea id="inputText" placeholder="请在此处输入数据，每行一个..."></textarea>
            </div>
        </div>
        
        <div class="panel">
            <div class="panel-header">
                <span>重复数据</span>
                <div class="stats" id="duplicateStats"></div>
            </div>
            <div class="panel-content">
                <ul class="result-list" id="duplicateList"></ul>
            </div>
        </div>
    </div>
    
    <div class="controls">
        <button id="checkBtn">查重分析</button>
        <button id="exportUniqueBtn" class="export-btn">导出唯一数据</button>
        <button id="exportDuplicateBtn" class="export-btn">导出重复数据</button>
        <button id="clearBtn" class="clear-btn">清空数据</button>
    </div>

    <script>
        document.addEventListener('DOMContentLoaded', function() {
            const inputText = document.getElementById('inputText');
            const duplicateList = document.getElementById('duplicateList');
            const duplicateStats = document.getElementById('duplicateStats');
            const checkBtn = document.getElementById('checkBtn');
            const exportUniqueBtn = document.getElementById('exportUniqueBtn');
            const exportDuplicateBtn = document.getElementById('exportDuplicateBtn');
            const clearBtn = document.getElementById('clearBtn');
            
            let uniqueItems = [];
            let duplicateItems = [];
            
            // 查重分析
            checkBtn.addEventListener('click', function() {
                const lines = inputText.value.split('\n').map(line => line.trim()).filter(line => line !== '');
                
                if (lines.length === 0) {
                    alert('请输入数据');
                    return;
                }
                
                const itemCount = {};
                uniqueItems = [];
                duplicateItems = [];
                
                // 统计每行出现的次数
                lines.forEach(line => {
                    itemCount[line] = (itemCount[line] || 0) + 1;
                });
                
                // 分离唯一项和重复项
                for (const item in itemCount) {
                    if (itemCount[item] === 1) {
                        uniqueItems.push(item);
                    } else {
                        duplicateItems.push({
                            text: item,
                            count: itemCount[item]
                        });
                    }
                }
                
                // 更新输入框，只保留唯一项
                inputText.value = uniqueItems.join('\n');
                
                // 更新重复项列表
                duplicateList.innerHTML = '';
                duplicateItems.forEach(item => {
                    const li = document.createElement('li');
                    li.textContent = `${item.text} (重复 ${item.count} 次)`;
                    duplicateList.appendChild(li);
                });
                
                // 更新统计信息
                duplicateStats.textContent = `发现 ${duplicateItems.length} 个重复项`;
            });
            
            // 导出唯一数据
            exportUniqueBtn.addEventListener('click', function() {
                if (uniqueItems.length === 0) {
                    alert('没有唯一数据可导出');
                    return;
                }
                
                exportToExcel(uniqueItems.map(item => [item]), '唯一数据');
            });
            
            // 导出重复数据
            exportDuplicateBtn.addEventListener('click', function() {
                if (duplicateItems.length === 0) {
                    alert('没有重复数据可导出');
                    return;
                }
                
                const data = duplicateItems.map(item => [item.text, item.count]);
                exportToExcel(data, '重复数据', ['内容', '重复次数']);
            });
            
            // 清空数据
            clearBtn.addEventListener('click', function() {
                inputText.value = '';
                duplicateList.innerHTML = '';
                duplicateStats.textContent = '';
                uniqueItems = [];
                duplicateItems = [];
            });
            
            // 导出Excel函数
            function exportToExcel(data, sheetName, headers) {
                const ws = XLSX.utils.aoa_to_sheet(headers ? [headers, ...data] : data);
                const wb = XLSX.utils.book_new();
                XLSX.utils.book_append_sheet(wb, ws, sheetName);
                XLSX.writeFile(wb, `${sheetName}_${formatDate()}.xlsx`);
            }
            
            // 格式化日期为文件名
            function formatDate() {
                const now = new Date();
                return `${now.getFullYear()}${padZero(now.getMonth() + 1)}${padZero(now.getDate())}_${padZero(now.getHours())}${padZero(now.getMinutes())}`;
            }
            
            function padZero(num) {
                return num.toString().padStart(2, '0');
            }
        });
    </script>
</body>
</html>
还需要引入一个第三方插件xlsx.full.min.js或者直接下载解压就可以使用了