87 lines
3.2 KiB
Python
87 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
安全检查音素数据集文件
|
||
处理大文件和可能的损坏问题
|
||
"""
|
||
|
||
import pickle
|
||
import os
|
||
|
||
def safe_inspect_phoneme_dataset(file_path):
|
||
"""安全地检查音素数据集文件"""
|
||
|
||
print(f"检查文件: {file_path}")
|
||
|
||
if not os.path.exists(file_path):
|
||
print("文件不存在!")
|
||
return
|
||
|
||
# 文件基本信息
|
||
file_size = os.path.getsize(file_path)
|
||
print(f"文件大小: {file_size / (1024*1024*1024):.2f} GB")
|
||
|
||
try:
|
||
# 尝试逐步读取
|
||
with open(file_path, 'rb') as f:
|
||
print("开始读取pickle文件...")
|
||
|
||
# 尝试读取文件头部
|
||
try:
|
||
data = pickle.load(f)
|
||
print("文件读取成功!")
|
||
|
||
print(f"数据类型: {type(data)}")
|
||
|
||
if isinstance(data, dict):
|
||
print(f"\n=== 数据集统计 ===")
|
||
print(f"音素类型数量: {len(data)}")
|
||
|
||
# 统计总片段数（不加载所有数据到内存）
|
||
total_segments = 0
|
||
phoneme_counts = {}
|
||
|
||
for phoneme, segments in data.items():
|
||
segment_count = len(segments) if segments else 0
|
||
phoneme_counts[phoneme] = segment_count
|
||
total_segments += segment_count
|
||
|
||
# 只显示前20个音素，避免输出过长
|
||
if len(phoneme_counts) <= 20:
|
||
print(f" {phoneme}: {segment_count} 个片段")
|
||
|
||
if len(phoneme_counts) > 20:
|
||
print(f" ... 还有 {len(phoneme_counts) - 20} 个其他音素")
|
||
|
||
print(f"\n总片段数: {total_segments}")
|
||
|
||
# 查看第一个音素的第一个片段示例
|
||
if data:
|
||
first_phoneme = list(data.keys())[0]
|
||
if data[first_phoneme]:
|
||
first_segment = data[first_phoneme][0]
|
||
print(f"\n=== 数据片段示例 (音素: {first_phoneme}) ===")
|
||
for key, value in first_segment.items():
|
||
if hasattr(value, 'shape'):
|
||
print(f" {key}: shape {value.shape}")
|
||
elif hasattr(value, '__len__'):
|
||
print(f" {key}: length {len(value)}")
|
||
else:
|
||
print(f" {key}: {value}")
|
||
|
||
else:
|
||
print(f"数据不是字典格式: {type(data)}")
|
||
if hasattr(data, '__len__'):
|
||
print(f"数据长度: {len(data)}")
|
||
|
||
except EOFError as e:
|
||
print(f"文件可能损坏或不完整: {e}")
|
||
except pickle.UnpicklingError as e:
|
||
print(f"Pickle解析错误: {e}")
|
||
|
||
except Exception as e:
|
||
print(f"读取文件时发生错误: {e}")
|
||
print(f"错误类型: {type(e)}")
|
||
|
||
if __name__ == "__main__":
|
||
file_path = "../phoneme_segmented_data/phoneme_dataset_20251007_194413.pkl"
|
||
safe_inspect_phoneme_dataset(file_path) |