feat: 实现去重检测

- 基于字段精确匹配去重：批内去重 + 与种子数据去重
- SynthesisResult 新增 dedup_count 统计
- CLI 输出显示去重数量
- validate=False 时跳过去重
- 新增 3 个测试 (71 tests)

Generated with [Claude Code](https://claude.ai/code)
via [Happy](https://happy.engineering)

Author: liuxiaotong

src/datasynth/cli.py

@@ -114,6 +114,8 @@ def on_progress(current, total):
         click.echo(f"✓ 生成成功: {result.output_path}")
         click.echo(f"  生成数量: {result.generated_count}")
         click.echo(f"  失败数量: {result.failed_count}")
+        if result.dedup_count:
+            click.echo(f"  去重数量: {result.dedup_count}")
         click.echo(f"  Token 用量: {result.total_tokens:,}")
         click.echo(f"  预计成本: ${result.estimated_cost:.4f}")
         click.echo(f"  耗时: {result.duration_seconds:.1f}s")

src/datasynth/synthesizer.py

@@ -24,6 +24,7 @@ class SynthesisResult:
     output_path: str = ""
     generated_count: int = 0
     failed_count: int = 0
+    dedup_count: int = 0
     total_tokens: int = 0
     estimated_cost: float = 0.0
     duration_seconds: float = 0.0
@@ -44,6 +45,11 @@ def __init__(self, config: Optional[SynthesisConfig] = None):
         self._client = None
         self._provider = None
 
+    @staticmethod
+    def _fingerprint(sample: Dict[str, Any]) -> str:
+        """Create a dedup key from a sample by sorting and serializing."""
+        return json.dumps(sample, sort_keys=True, ensure_ascii=False)
+
     def _init_client(self):
         """Initialize LLM client based on provider."""
         if self._client is not None:
@@ -107,9 +113,16 @@ def synthesize(
             # Determine target count
             count = target_count or self.config.target_count
 
+            # Build dedup index from seed samples
+            seen: set[str] = set()
+            if self.config.validate:
+                for seed in seed_samples:
+                    seen.add(self._fingerprint(seed))
+
             # Generate in batches
             all_samples = []
             failed = 0
+            deduped = 0
             total_tokens = 0
             batches = (count + self.config.batch_size - 1) // self.config.batch_size
 
@@ -143,6 +156,19 @@ def synthesize(
 
                         # Parse response
                         samples = parse_generated_samples(response_text, data_schema)
+
+                        # Dedup
+                        if self.config.validate:
+                            unique = []
+                            for s in samples:
+                                fp = self._fingerprint(s)
+                                if fp not in seen:
+                                    seen.add(fp)
+                                    unique.append(s)
+                                else:
+                                    deduped += 1
+                            samples = unique
+
                         all_samples.extend(samples)
                         batch_success = True
                         break
@@ -167,6 +193,7 @@ def synthesize(
 
             result.generated_count = len(all_samples)
             result.failed_count = failed
+            result.dedup_count = deduped
             result.total_tokens = total_tokens
             result.estimated_cost = self._estimate_cost(total_tokens)
 

tests/test_synthesizer.py

@@ -187,6 +187,83 @@ def test_synthesize_retry_succeeds(self, tmp_path):
         assert result.failed_count == 0
         assert s._call_llm.call_count == 2
 
+    def test_dedup_within_batch(self, tmp_path):
+        """Test that duplicate samples within a batch are removed."""
+        duped_response = json.dumps(
+            [
+                {"instruction": "Q1", "response": "A1"},
+                {"instruction": "Q1", "response": "A1"},  # duplicate
+                {"instruction": "Q2", "response": "A2"},
+            ],
+            ensure_ascii=False,
+        )
+        cfg = SynthesisConfig(target_count=3, batch_size=3)
+        s = DataSynthesizer(cfg)
+        s._client = MagicMock()
+        s._provider = "anthropic"
+        s._call_llm = MagicMock(return_value=(duped_response, 300))
+
+        result = s.synthesize(
+            schema=SAMPLE_SCHEMA,
+            seed_samples=SAMPLE_SEEDS,
+            output_path=str(tmp_path / "out.json"),
+            target_count=3,
+        )
+
+        assert result.generated_count == 2
+        assert result.dedup_count == 1
+
+    def test_dedup_against_seeds(self, tmp_path):
+        """Test that samples matching seed data are removed."""
+        # Return a sample identical to a seed
+        seed_copy_response = json.dumps(
+            [
+                {"instruction": "什么是 AI？", "response": "AI 是人工智能的缩写..."},  # same as seed
+                {"instruction": "新问题", "response": "新回答"},
+            ],
+            ensure_ascii=False,
+        )
+        cfg = SynthesisConfig(target_count=2, batch_size=2)
+        s = DataSynthesizer(cfg)
+        s._client = MagicMock()
+        s._provider = "anthropic"
+        s._call_llm = MagicMock(return_value=(seed_copy_response, 200))
+
+        result = s.synthesize(
+            schema=SAMPLE_SCHEMA,
+            seed_samples=SAMPLE_SEEDS,
+            output_path=str(tmp_path / "out.json"),
+            target_count=2,
+        )
+
+        assert result.generated_count == 1
+        assert result.dedup_count == 1
+
+    def test_dedup_disabled(self, tmp_path):
+        """Test that dedup is skipped when validate=False."""
+        duped_response = json.dumps(
+            [
+                {"instruction": "Q1", "response": "A1"},
+                {"instruction": "Q1", "response": "A1"},  # duplicate
+            ],
+            ensure_ascii=False,
+        )
+        cfg = SynthesisConfig(target_count=2, batch_size=2, validate=False)
+        s = DataSynthesizer(cfg)
+        s._client = MagicMock()
+        s._provider = "anthropic"
+        s._call_llm = MagicMock(return_value=(duped_response, 200))
+
+        result = s.synthesize(
+            schema=SAMPLE_SCHEMA,
+            seed_samples=SAMPLE_SEEDS,
+            output_path=str(tmp_path / "out.json"),
+            target_count=2,
+        )
+
+        assert result.generated_count == 2  # duplicates kept
+        assert result.dedup_count == 0
+
     def test_synthesize_from_datarecipe(self, tmp_path):
         """Test synthesize_from_datarecipe reads correct files."""
         # Set up directory structure