As noted by others, there might be issues here with shell scripting depending on how the duplicates were made. You could theoretically avoid those by copying the folder and trying it first, but the only way to totally verify is if you went through them by hand which somewhat defeats the purpose.
This is a Python file I use for similar deduplication that I might have. It scans the folder for files by using their sha256, retains the highest file name alphabetically (which should generally keep the cleaner file names) and deletes the others. You can change the dry_run variable on the 3rd to last line to true like this:
if __name__ == '__main__':
d = Deduplicator(path, dry_run=True)
d.deduplicate()
To verify that the files you want to delete are actually what are going to be deleted. And, of course, change line 4 from /path/to/your/files to the actual directory where the files are.
To run, on a mac you should simply be able to run:
python /path/to/your/deduplicate.py
Where /path/to/your/script is wherever you save this .py file. Basically, put the following in a text file and name it deduplicate.py:
import os
import hashlib
path = '/path/to/your/files'
class Deduplicator:
def __init__(self, path, dry_run=True):
self.path = path
self.file_dict = dict()
self.dry_run = dry_run
def deduplicate(self):
self.get_files()
self.clean_files()
def get_file_hash(self, file_path):
with open(file_path, 'rb') as f:
file = f.read()
hash = hashlib.sha256(file).hexdigest()
return hash
def get_files(self):
# Loop through the directory
for file in os.listdir(path):
# Get the file hash
hash = self.get_file_hash(os.path.join(self.path, file))
# If we haven't seen the hash yet, go ahead and initiate the list
if not self.file_dict.get(hash):
self.file_dict[hash] = list()
# Then add this filename to that hashed value
self.file_dict[hash].append(file)
def clean_files(self):
for hash, file_names in self.file_dict.items():
file_names.sort(reverse=True)
files_to_delete = file_names[:-1]
print(f"File to keep: {file_names[0]}")
print(f'Files to delete: {files_to_delete}')
print('-'*50)
if not self.dry_run:
for file in files_to_delete:
full_path = os.path.join(path,file)
print(f"...deleting: {full_path}")
os.remove(full_path)
if __name__ == '__main__':
d = Deduplicator(path, dry_run=False)
d.deduplicate()
