利用Python轻松实现找出同步日志中的重复数据

你知道我知道 發表於 2026-1-7 14:53:02

利用Python轻松实现找出同步日志中的重复数据

在做接口对接时，对方提交过来的数据存在重复数据，这时候可以通过 Python 轻松提取出来
syncDevice_2026-01-07.log
<blockquote>2026-01-07 11:41:33 | [{"deviceMac":"ED:0C:51:C2:B2:EA","deviceSn":"240103 P50162"},{"deviceMac":"C0:7A:A1:6C:67:AA","deviceSn":"221130P50012"},{"deviceMac":"D0:D0:02:39:83:D4","deviceSn":"221130P50012"}] 2026-01-07 11:41:33 | [{"deviceMac":"D0:D0:02:39:83:D4","deviceSn":"221130P50012"},{"deviceMac":"DC:8E:33:BA:3D:6D","deviceSn":"221130P50013"}]</blockquote>
Python 代码如下：
<div class="jb51code"><pre class="brush:py;">import json
import re
from collections import defaultdict

def find_duplicate_devices_unique_mac(log_file_path):
# 按 deviceSn 分组，每组内用集合去重 deviceMac
devices_by_sn = defaultdict(list)
mac_seen_by_sn = defaultdict(set)# 用于跟踪每个 SN 下已见过的 deviceMac

# 读取日志文件
with open(log_file_path, 'r', encoding='utf-8') as file:
 for line_num, line in enumerate(file, 1):
 # 使用正则表达式提取 JSON 部分
 match = re.search(r'\[.*\]', line)
 if match:
 try:
 # 解析 JSON 数组
 devices = json.loads(match.group())

 # 将每个设备添加到对应 deviceSn 的分组中，并去重 deviceMac
 for device in devices:
 device_sn = device.get('deviceSn')
 device_mac = device.get('deviceMac')

 if device_sn and device_mac:
 # 如果这个 SN 下还没见过这个 MAC，则添加
 if device_mac not in mac_seen_by_sn:
 devices_by_sn.append(device)
 mac_seen_by_sn.add(device_mac)
 except json.JSONDecodeError as e:
 print(f"第 {line_num} 行解析 JSON 时出错: {e}")
 continue

# 找出重复的 deviceSn（去重 MAC 后仍然有多个记录的）
duplicate_devices = {}
for device_sn, devices in devices_by_sn.items():
 if len(devices) > 1:
 duplicate_devices = devices

return duplicate_devices

def print_duplicate_devices(duplicate_devices):
if not duplicate_devices:
 print("没有找到重复的 deviceSn")
 return

print("找到以下重复的 deviceSn (已对 deviceMac 去重):\n")
for device_sn, devices in duplicate_devices.items():
 print(f"deviceSn: {device_sn} (去重后出现 {len(devices)} 次)")
 print("-" * 50)

 for i, device in enumerate(devices, 1):
 print(f"第 {i} 条记录:")
 # 美化输出 JSON
 print(json.dumps(device, indent=2, ensure_ascii=False))
 print()

 print("=" * 80)

# 版本2：更简洁的实现，直接输出去重结果
def find_and_print_duplicates_unique(log_file):
# 存储去重后的设备
unique_devices_by_sn = defaultdict(list)
seen_mac_by_sn = defaultdict(set)

with open(log_file, 'r') as f:
 for line in f:
 # 提取 JSON 数组部分
 json_match = re.search(r'\[.*\]', line)
 if json_match:
 try:
 devices = json.loads(json_match.group())
 for device in devices:
 sn = device.get('deviceSn')
 mac = device.get('deviceMac')

 if sn and mac:
 # 如果这个 MAC 还没在这个 SN 组中出现过
 if mac not in seen_mac_by_sn:
 unique_devices_by_sn.append(device)
 seen_mac_by_sn.add(mac)
 except:
 continue

# 找出并打印重复项
print("重复的设备SN及其数据 (已对deviceMac去重):")
print("=" * 80)

found_duplicates = False
for sn, devices in unique_devices_by_sn.items():
 if len(devices) > 1:
 found_duplicates = True
 print(f"\n设备SN: {sn} (去重后出现 {len(devices)} 次)")
 print("-" * 50)

 for i, device in enumerate(devices, 1):
 print(f"记录 {i}:")
 # 格式化时间戳
 if 'productionDate' in device:
 import datetime
 timestamp = device['productionDate'] / 1000
 dt = datetime.datetime.fromtimestamp(timestamp)
 device['productionDate_formatted'] = dt.strftime('%Y-%m-%d %H:%M:%S')

 print(json.dumps(device, indent=2, ensure_ascii=False))
 print()

if not found_duplicates:
 print("没有找到重复的 deviceSn (或所有重复都是相同的 deviceMac)")

# 主程序
if __name__ == "__main__":
log_file_path = "syncDevice_2026-01-07.log"

try:
 print("=" * 80)
 print("方法1：详细版")
 print("=" * 80)
 # 查找重复设备（去重 MAC）
 duplicate_devices = find_duplicate_devices_unique_mac(log_file_path)

 # 打印结果
 print_duplicate_devices(duplicate_devices)

 # 统计信息
 print("\n统计信息:")
 print(f"总共有 {len(duplicate_devices)} 个重复的 deviceSn")
 for device_sn, devices in duplicate_devices.items():
 print(f"- {device_sn}: {len(devices)} 条不重复的记录")

 print("\n" + "=" * 80)
 print("方法2：简洁版")
 print("=" * 80)
 find_and_print_duplicates_unique(log_file_path)

except FileNotFoundError:
 print(f"错误: 找不到文件 {log_file_path}")
except Exception as e:
 print(f"处理文件时出错: {e}")
</pre></div>
输入结果：
<blockquote>================================================================================ 方法1：详细版 ================================================================================ 找到以下重复的 deviceSn (已对 deviceMac 去重): deviceSn: 221130P50012 (去重后出现 2 次) -------------------------------------------------- 第 1 条记录: {   "deviceMac": "C0:7A:A1:6C:67:AA",   "deviceSn": "221130P50012" } 第 2 条记录: {   "deviceMac": "D0:D0:02:39:83:D4",   "deviceSn": "221130P50012" } ================================================================================ 统计信息: 总共有 1 个重复的 deviceSn   - 221130P50012: 2 条不重复的记录 ================================================================================ 方法2：简洁版 ================================================================================ 重复的设备SN及其数据 (已对deviceMac去重): ================================================================================ 设备SN: 221130P50012 (去重后出现 2 次) -------------------------------------------------- 记录 1: {   "deviceMac": "C0:7A:A1:6C:67:AA",   "deviceSn": "221130P50012" } 记录 2: {   "deviceMac": "D0:D0:02:39:83:D4",   "deviceSn": "221130P50012" } Process finished with exit code 0</blockquote>

頁: [1]

琼殿技术论坛's Archiver

利用Python轻松实现找出同步日志中的重复数据