之前爬虫写的逻辑有点错误,导致重复爬取了某些内容,全部删掉再爬也麻烦,所以写了个sql语句直接清掉重复项。
delete FROM tablename WHERE 去重字段 in
(
SELECT 去重字段
FROM tablename
GROUP BY 去重字段
HAVING COUNT(*) > 1
)
and id not in (
select min(id)
from tablename
group by 去重字段
having count(*)>1
);