本文目录导读：
我来帮你写一个图片去重脚本，这里提供几种不同的方案,从简单到复杂：
基于文件哈希的简单去重（最快）
import os
import hashlib
from collections import defaultdict
def get_file_hash(filepath, chunk_size=8192):
"""计算文件哈希值"""
hash_md5 = hashlib.md5()
with open(filepath, 'rb') as f:
for chunk in iter(lambda: f.read(chunk_size), b''):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def find_duplicate_files(folder_path):
"""查找重复文件"""
hash_dict = defaultdict(list)
# 遍历文件夹
for root, dirs, files in os.walk(folder_path):
for filename in files:
filepath = os.path.join(root, filename)
if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
try:
file_hash = get_file_hash(filepath)
hash_dict[file_hash].append(filepath)
except Exception as e:
print(f"Error processing {filepath}: {e}")
# 找出重复文件
duplicates = {hash_val: paths for hash_val, paths in hash_dict.items() if len(paths) > 1}
return duplicates
def delete_duplicates(duplicates, keep_first=True):
"""删除重复文件"""
deleted_count = 0
for hash_val, paths in duplicates.items():
if keep_first:
# 保留第一个，删除其余
for filepath in paths[1:]:
try:
os.remove(filepath)
print(f"Deleted: {filepath}")
deleted_count += 1
except Exception as e:
print(f"Error deleting {filepath}: {e}")
else:
# 全部删除
for filepath in paths:
try:
os.remove(filepath)
print(f"Deleted: {filepath}")
deleted_count += 1
except Exception as e:
print(f"Error deleting {filepath}: {e}")
print(f"Total deleted: {deleted_count}")
# 使用示例
if __name__ == "__main__":
target_folder = "path/to/your/images"
duplicates = find_duplicate_files(target_folder)
if duplicates:
print("Found duplicates:")
for hash_val, paths in duplicates.items():
print(f"Hash: {hash_val}")
for path in paths:
print(f" - {path}")
# 询问是否删除
choice = input("Delete duplicates? (y/n): ").lower()
if choice == 'y':
delete_duplicates(duplicates)
else:
print("No duplicates found!")
基于图像特征的感知哈希（能检测相似图片）
import os
import hashlib
from PIL import Image
import numpy as np
from collections import defaultdict
def average_hash(image_path, hash_size=16):
"""计算图片的平均哈希值"""
try:
img = Image.open(image_path)
img = img.resize((hash_size, hash_size), Image.LANCZOS)
img = img.convert('L') # 转为灰度
pixels = np.array(img)
avg = pixels.mean()
# 生成哈希
diff = pixels > avg
hash_str = ''.join(str(int(x)) for x in diff.flatten())
return hash_str
except Exception as e:
print(f"Error processing {image_path}: {e}")
return None
def find_similar_images(folder_path, threshold=20):
"""查找相似图片"""
image_hashes = []
for root, dirs, files in os.walk(folder_path):
for filename in files:
if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
filepath = os.path.join(root, filename)
hash_str = average_hash(filepath)
if hash_str:
image_hashes.append((filepath, hash_str))
# 找出相似的图片
duplicates = []
processed = set()
for i, (filepath1, hash1) in enumerate(image_hashes):
if i in processed:
continue
similar_group = [filepath1]
for j, (filepath2, hash2) in enumerate(image_hashes):
if i != j and j not in processed:
# 计算汉明距离
hamming_dist = sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
if hamming_dist < threshold:
similar_group.append(filepath2)
processed.add(j)
if len(similar_group) > 1:
duplicates.append(similar_group)
return duplicates
# 使用示例
if __name__ == "__main__":
target_folder = "path/to/your/images"
print("Finding similar images...")
similar_groups = find_similar_images(target_folder)
if similar_groups:
print("Found similar image groups:")
for group in similar_groups:
print("Group:")
for path in group:
print(f" - {path}")
choice = input("Remove similar images (keep first of each group)? (y/n): ").lower()
if choice == 'y':
for group in similar_groups:
for path in group[1:]:
try:
os.remove(path)
print(f"Deleted: {path}")
except Exception as e:
print(f"Error deleting {path}: {e}")
else:
print("No similar images found!")
完整的去重工具（带GUI界面）
import os
import hashlib
import tkinter as tk
from tkinter import filedialog, messagebox, ttk
from PIL import Image, ImageTk
import threading
class ImageDedupTool:
def __init__(self, root):
self.root = root
self.root.title("图片去重工具")
self.root.geometry("800x600")
self.folder_path = None
self.duplicates = {}
# 创建界面元素
self.setup_ui()
def setup_ui(self):
# 顶部按钮框架
top_frame = tk.Frame(self.root)
top_frame.pack(pady=10)
tk.Button(top_frame, text="选择文件夹", command=self.select_folder).pack(side=tk.LEFT, padx=5)
tk.Button(top_frame, text="查找重复", command=self.find_duplicates).pack(side=tk.LEFT, padx=5)
tk.Button(top_frame, text="删除重复", command=self.delete_duplicates).pack(side=tk.LEFT, padx=5)
# 进度条
self.progress = ttk.Progressbar(self.root, length=500, mode='indeterminate')
self.progress.pack(pady=10)
# 结果列表
self.tree = ttk.Treeview(self.root, columns=('type', 'path'), show="tree")
self.tree.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
# 状态栏
self.status_label = tk.Label(self.root, text="Ready")
self.status_label.pack(side=tk.BOTTOM, pady=5)
def select_folder(self):
self.folder_path = filedialog.askdirectory()
if self.folder_path:
self.status_label.config(text=f"Selected folder: {self.folder_path}")
def find_duplicates(self):
if not self.folder_path:
messagebox.showerror("Error", "请先选择文件夹！")
return
self.progress.start()
threading.Thread(target=self.find_duplicates_worker).start()
def find_duplicates_worker(self):
hash_dict = {}
for root, dirs, files in os.walk(self.folder_path):
for filename in files:
if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
filepath = os.path.join(root, filename)
try:
file_hash = self.calculate_md5(filepath)
if file_hash in hash_dict:
hash_dict[file_hash].append(filepath)
else:
hash_dict[file_hash] = [filepath]
except Exception as e:
self.root.after(0, self.update_status, f"Error: {filepath}")
self.duplicates = {h: paths for h, paths in hash_dict.items() if len(paths) > 1}
self.root.after(0, self.display_results)
self.root.after(0, self.progress.stop)
def calculate_md5(self, filepath):
hash_md5 = hashlib.md5()
with open(filepath, 'rb') as f:
for chunk in iter(lambda: f.read(8192), b''):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def display_results(self):
# 清空列表
for item in self.tree.get_children():
self.tree.delete(item)
if not self.duplicates:
messagebox.showinfo("Info", "没有找到重复文件！")
return
for hash_val, paths in self.duplicates.items():
parent = self.tree.insert('', 'end', text=f"重复组 (Hash: {hash_val[:8]}...)", open=True)
for path in paths:
self.tree.insert(parent, 'end', text=path)
self.update_status(f"找到 {len(self.duplicates)} 组重复文件")
def delete_duplicates(self):
if not self.duplicates:
messagebox.showinfo("Info", "没有找到重复文件！")
return
deleted_count = 0
for hash_val, paths in self.duplicates.items():
# 保留每组中的第一个，删除其余
for path in paths[1:]:
try:
os.remove(path)
deleted_count += 1
except Exception as e:
self.update_status(f"删除失败: {path} - {e}")
self.update_status(f"Successfully deleted {deleted_count} files")
messagebox.showinfo("Info", f"成功删除 {deleted_count} 个重复文件")
self.find_duplicates() # 刷新显示
def update_status(self, msg):
self.status_label.config(text=msg)
# 运行应用
if __name__ == "__main__":
root = tk.Tk()
app = ImageDedupTool(root)
root.mainloop()
使用说明
- 方案一是最简单的基于MD5的完全重复检测
- 方案二使用感知哈希，可以检测经过轻微修改（如旋转、缩放）的相似图片
- 方案三提供GUI界面，更加友好
安装依赖
# 基础版本 pip install os-sys # 如果需要感知哈希或GUI功能 pip install pillow numpy tkinter
注意事项
- 使用前建议先备份重要图片
- 删除操作不可逆，请谨慎操作
- 可以根据需要调整哈希阈值和文件类型
选择适合你需求的方案,如果有特殊要求可以进一步调整代码！
