diff --git a/build.gradle b/build.gradle index 934b78b..a2880cb 100644 --- a/build.gradle +++ b/build.gradle @@ -53,7 +53,7 @@ dependencies { tasks.named('test') { useJUnitPlatform { - excludeTags 'benchmark', 'minio-integration', 'claim-concurrency', 'local-e2e', 'vector-search-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load' + excludeTags 'benchmark', 'minio-integration', 'claim-concurrency', 'local-e2e', 'vector-search-performance', 'worker-indexing-throughput', 'worker-horizontal-scaling', 'worker-queue-backpressure', 'document-indexing-e2e-load', 'chunk-quality-performance' } } @@ -131,6 +131,28 @@ tasks.register('bgeBatchPerformanceTest', Exec) { outputs.upToDateWhen { false } } +tasks.register('chunkQualityPerformanceTest', Test) { + group = 'verification' + description = '실제 BGE-M3에서 Chunk Size·Overlap별 검색 품질과 임베딩 비용을 비교합니다.' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + useJUnitPlatform { + includeTags 'chunk-quality-performance' + } + maxParallelForks = 1 + systemProperties System.properties.findAll { key, value -> + key.toString().startsWith('chunk.quality.performance.') + } + if (System.getProperty('chunk.quality.performance.output') == null) { + systemProperty( + 'chunk.quality.performance.output', + layout.buildDirectory.file('reports/chunk-quality/chunk-quality-latest.json').get().asFile.absolutePath + ) + } + // 실제 외부 모델을 반복 호출하는 장시간 Benchmark이므로 일반 Test와 Build Cache에서 분리한다. + outputs.upToDateWhen { false } +} + tasks.register('vectorSearchPerformanceTest', Test) { group = 'verification' description = 'PostgreSQL pgvector의 Exact Seq Scan과 HNSW를 데이터 규모별로 비교합니다.' diff --git a/docs/design/gimin-#147-chunk-size-overlap-quality-benchmark.md b/docs/design/gimin-#147-chunk-size-overlap-quality-benchmark.md new file mode 100644 index 0000000..7a3bdf6 --- /dev/null +++ b/docs/design/gimin-#147-chunk-size-overlap-quality-benchmark.md @@ -0,0 +1,176 @@ +# Chunk Size·Overlap 검색 품질 및 비용 비교 Benchmark 설계 + +- 관련 이슈: [#147](https://github.com/DocGrid/backend/issues/147) +- 작성일: 2026-08-11 +- 상태: 구현 및 실측 완료 + +## 1. 배경 + +문서 인덱싱 파이프라인은 Unicode Code Point 기준 `FixedSizeChunker`를 사용하고 기본값으로 +`chunkSize=1000`, `overlap=200`을 적용한다. 이 값은 기능적으로 검증됐지만 Chunk 경계에 걸친 +근거가 검색에서 얼마나 보존되는지, Overlap 증가가 임베딩·저장량을 얼마나 늘리는지 같은 조건에서 +비교한 실측 근거는 없다. + +Chunk 수만 비교하면 검색 품질 손실을 발견할 수 없고, 품질만 비교하면 중복 비용을 판단할 수 없다. +이번 작업은 실제 Chunker와 BAAI/bge-m3를 사용해 품질과 비용을 함께 측정한다. + +## 2. 목표와 성공 기준 + +- 8개 Chunk Size·Overlap Profile을 같은 Corpus에서 비교한다. +- Chunk 경계 주변에 정답 근거를 배치해 Overlap의 Answer Coverage 효과를 검증한다. +- 실제 BAAI/bge-m3 Vector로 Exact Cosine 검색 품질을 측정한다. +- Hit@1·Hit@3·MRR@10과 Chunk·중복량·임베딩 시간을 함께 기록한다. +- Model명, 응답 수, 1024차원과 유한값 불변식을 매 호출마다 검증한다. +- 일반 테스트와 실제 모델 Benchmark를 분리하고 한 명령으로 JSON을 재현한다. +- 결과만으로 운영 기본값을 자동 변경하지 않고 후속 의사결정 근거로 남긴다. + +## 3. 비교 Profile + +| Profile | Chunk Size | Overlap | Overlap 비율 | 비고 | +|---|---:|---:|---:|---| +| `c400-o0` | 400 | 0 | 0% | 작은 Chunk 기준선 | +| `c400-o80` | 400 | 80 | 20% | 작은 Chunk 중첩 | +| `c800-o0` | 800 | 0 | 0% | 중간 Chunk 기준선 | +| `c800-o160` | 800 | 160 | 20% | 중간 Chunk 중첩 | +| `c1000-o0` | 1000 | 0 | 0% | 현재 크기의 중첩 없는 기준선 | +| `c1000-o200` | 1000 | 200 | 20% | 현재 운영 기본 Profile | +| `c1600-o0` | 1600 | 0 | 0% | 큰 Chunk 기준선 | +| `c1600-o320` | 1600 | 320 | 20% | 큰 Chunk 중첩 | + +## 4. 결정적 경계 Corpus + +### 4.1 문서와 Query + +- 400·800·1000·1600 Code Point 경계마다 3개 Case를 만들어 총 12개 문서·질의를 사용한다. +- 각 문서는 약 2,200 Code Point의 중립적인 채움 Text와 한 개의 고유한 근거 문장으로 구성한다. +- 근거 시작점을 해당 경계 직전으로 고정해 Overlap이 없을 때 문장이 두 Chunk로 분리되게 한다. +- Query는 근거 문장만으로 답할 수 있는 고유한 한국어 질문이다. +- Seed와 문자열 Template을 고정해 같은 Commit에서 Corpus가 항상 같게 생성한다. + +### 4.2 Ground Truth + +Chunk가 다음 조건을 모두 만족할 때만 Relevant로 판정한다. + +1. Query의 정답 문서에서 생성됐다. +2. `chunk.charStart <= evidenceStart`다. +3. `chunk.charEnd >= evidenceEnd`다. + +일부 근거 조각만 포함한 Chunk는 Relevant로 취급하지 않는다. 따라서 Answer Coverage Ratio는 +Chunking 단계에서 검색 가능한 완전한 근거가 보존됐는지를 나타낸다. + +## 5. 측정 경계 + +### 5.1 실제 구성요소 + +- 제품 코드의 `FixedSizeChunker` +- 실제 Embedding Server의 `POST /embed/batch` +- 실제 `BAAI/bge-m3` 1024차원 Dense Vector +- Query와 Chunk Vector의 메모리 내 Exact Cosine Similarity + +### 5.2 의도적인 제외 + +PostgreSQL·pgvector·HNSW를 사용하지 않는다. 이 작업의 독립 변수는 Chunk Size와 Overlap이며, +ANN 근사 오차와 DB 실행 계획을 포함하면 Chunking 품질과 Vector 검색 성능을 분리할 수 없다. +Exact·HNSW 성능과 Recall은 별도 Vector Benchmark에서 검증한다. + +PDF·DOCX Parser, OCR, 권한 Filter와 RAG 답변 생성도 이번 측정에서 제외한다. Parser가 생성한 +Page·Section 경계 보존 동작은 제품 Chunker 테스트 범위이며 이번 Corpus는 파라미터 효과만 격리한다. + +## 6. 지표 계약 + +### 6.1 품질 + +| 지표 | 계산 | +|---|---| +| Answer Coverage Ratio | 완전한 근거 Chunk가 존재하는 Query 수 / 전체 Query 수 | +| Answer Hit@1 | 첫 번째 결과가 Relevant인 Query 비율 | +| Answer Hit@3 | 상위 3개 안에 Relevant가 있는 Query 비율 | +| MRR@10 | 상위 10개에서 첫 Relevant 순위 역수의 평균 | + +동점은 Cosine Similarity 내림차순, 문서 ID 오름차순, Chunk Index 오름차순으로 고정한다. + +### 6.2 비용 + +| 지표 | 계산 | +|---|---| +| Chunk Count | Profile이 생성한 전체 Chunk 수 | +| Chunk Code Points | 모든 Chunk Text의 Code Point 수 합계 | +| Duplicate Code Points | `Chunk Code Points - 원문 Code Points` | +| Duplicate Ratio | `Duplicate Code Points / 원문 Code Points` | +| Embedding Median·P95 | Profile별 Chunk Vector 생성 시간의 Round 통계 | +| Search Median·P95 | 전체 Query Exact Ranking 시간의 Round 통계 | + +## 7. 실행 공정성 + +1. Health Check 뒤 Query와 대표 Chunk를 임베딩해 Model을 Warm-up한다. +2. Query Vector는 한 번만 생성해 Profile별 Chunk 임베딩 비용과 분리한다. +3. 각 본 측정 Round에서 Profile 시작 순서를 한 칸씩 회전한다. +4. 한 HTTP 요청의 Text 수는 64개 이하로 나누고 모델 내부 Batch Size 기본값은 32로 둔다. +5. Chunk 생성과 임베딩, Exact 검색 시간을 구분해 기록한다. +6. 같은 Profile의 품질 지표는 Round마다 같아야 하며 다르면 측정을 실패시킨다. + +기본값은 Warm-up 1회와 본 측정 2회다. 안정적인 통계가 필요할 때 System Property로 Round와 +Batch Size를 변경할 수 있다. + +## 8. Vector 불변식 + +모든 Batch 응답에서 다음을 검증한다. + +- Model명이 `BAAI/bge-m3`다. +- 응답 Item 수와 요청 Text 수가 같다. +- Item Index가 요청 순서와 일치한다. +- 모든 Vector가 정확히 1024차원이다. +- 모든 원소가 NaN·Infinity가 아닌 유한값이다. +- Cosine 계산의 Vector Norm이 0보다 크다. + +하나라도 어기면 부분 결과를 정상 수치로 기록하지 않고 Benchmark 전체를 실패시킨다. + +## 9. 실행과 결과 + +일반 테스트는 실제 BGE-M3를 요구하지 않는다. 전용 Task만 외부 모델을 사용한다. + +```bash +docker compose up -d embedding-server +./gradlew chunkQualityPerformanceTest +``` + +기본 결과는 다음 경로에 생성한다. + +```text +build/reports/chunk-quality/chunk-quality-latest.json +``` + +확장 실행 예시는 다음과 같다. + +```bash +./gradlew chunkQualityPerformanceTest \ + -Dchunk.quality.performance.rounds=3 \ + -Dchunk.quality.performance.batch-size=32 \ + -Dchunk.quality.performance.output=build/reports/chunk-quality/chunk-quality.json +``` + +## 10. 결과 해석 + +1. JSON이 생성되면 모든 HTTP·응답 계약·Vector 불변식 검증을 통과한 것으로 본다. +2. Answer Coverage와 Hit@3가 가장 높은 Profile 집합을 확인한다. +3. 같은 품질이면 Duplicate Ratio와 Embedding P95가 낮은 Profile을 선호한다. +4. 한 Profile이 다른 Profile보다 품질은 낮지 않고 비용은 높지 않으면서 한 지표 이상 우수하면 + Pareto 후보로 표시한다. +5. 로컬 CPU 결과는 상대 비교 기준선이며 운영 SLO로 해석하지 않는다. +6. 운영 기본값 변경은 실제 사용자 Corpus 검증을 포함한 별도 의사결정으로 남긴다. + +## 11. 커밋 분할 + +1. `docs: #147 Chunk Size·Overlap 품질 비교 설계` +2. `test: #147 Chunk 품질 지표와 Corpus 계약 추가` +3. `perf: #147 실제 BGE-M3 Chunk 품질 Benchmark 추가` +4. `perf: #147 Chunk Size·Overlap 실측 결과 기록` + +## 12. 완료 조건 + +- 일반 `./gradlew test`가 실제 모델 없이 성공한다. +- 전용 Task가 8개 Profile과 12개 Query를 실제 BGE-M3에서 측정한다. +- 품질 4종, 비용 7종과 실행 환경·설정이 JSON에 기록된다. +- Vector 불변식과 품질 계산 계약이 자동 테스트로 보호된다. +- 실측 비교표, 결론, 한계와 재현 명령이 `docs/test-results/`에 기록된다. +- 제품 Chunking 기본 설정과 Pipeline 동작은 변경하지 않는다. diff --git a/docs/test-results/assets/chunk-quality/chunk-quality-tradeoff.svg b/docs/test-results/assets/chunk-quality/chunk-quality-tradeoff.svg new file mode 100644 index 0000000..96e1c65 --- /dev/null +++ b/docs/test-results/assets/chunk-quality/chunk-quality-tradeoff.svg @@ -0,0 +1,73 @@ + + Chunk Size와 Overlap별 중복 비율 대비 MRR@10 + 실제 BAAI bge-m3로 12개 경계 질의를 측정했다. 가로축은 중복 코드 포인트 비율, 세로축은 MRR@10이다. 800에 160 Overlap 조합이 MRR 0.660으로 가장 높고, 현재 기본값 1000에 200 Overlap은 MRR 0.302다. + + + Chunk Size·Overlap 검색 품질–중복 비용 Trade-off + BAAI/bge-m3 · 12개 경계 Query · Exact Cosine · 2회 측정 + + + + + + + + + + + + + + + + 0.0 + 0.2 + 0.4 + 0.6 + 0.7 + 0% + 4% + 8% + 12% + 16% + 20% + 중복 Code Point 비율 → + MRR@10 → + + + 400/0 · 0.176 + + 800/0 · 0.315 + + 1000/0 · 0.239 + + 1600/0 · 0.093 + + + 400/80 · 0.600 + + 800/160 · 0.660 + + 1000/200 · 0.302 + + 1600/320 · 0.530 + + + Overlap 0 + + Overlap 20% + + 현재 기본값 + 높을수록 품질↑ · 왼쪽일수록 중복 비용↓ + diff --git a/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark-data.json b/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark-data.json new file mode 100644 index 0000000..bee36b4 --- /dev/null +++ b/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark-data.json @@ -0,0 +1,970 @@ +{ + "generatedAt" : "2026-08-11T07:04:50.868837Z", + "model" : "BAAI/bge-m3", + "embeddingEndpoint" : "http://localhost:8000", + "osName" : "Mac OS X", + "osArchitecture" : "aarch64", + "javaVersion" : "17.0.18", + "warmUpRuns" : 1, + "measuredRounds" : 2, + "modelBatchSize" : 32, + "maxTextsPerRequest" : 64, + "documentCount" : 12, + "queryCount" : 12, + "originalCodePoints" : 26400, + "queryEmbeddingMillis" : 572.515916, + "queryEmbeddingRequestCount" : 1, + "profiles" : [ { + "profileId" : "c400-o0", + "chunkSize" : 400, + "overlap" : 0, + "chunkCount" : 72, + "originalCodePoints" : 26400, + "chunkCodePoints" : 26400, + "duplicateCodePoints" : 0, + "duplicateRatio" : 0.0, + "quality" : { + "answerCoverageRatio" : 0.25, + "hitAt1" : 0.16666666666666666, + "hitAt3" : 0.16666666666666666, + "mrrAt10" : 0.17592592592592593, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc10:4", + "topSimilarity" : 0.6735931793412909 + }, { + "queryId" : "q2", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc12:4", + "topSimilarity" : 0.6816941247399084 + }, { + "queryId" : "q3", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:2", + "topSimilarity" : 0.6262402306129986 + }, { + "queryId" : "q4", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:2", + "topSimilarity" : 0.6430223021020572 + }, { + "queryId" : "q5", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc10:4", + "topSimilarity" : 0.642979111245326 + }, { + "queryId" : "q6", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc6:2", + "topSimilarity" : 0.6651210403091457 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc7:2", + "topSimilarity" : 0.6970139101412048 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 9, + "topCandidateId" : "doc10:4", + "topSimilarity" : 0.7126785983266984 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc9:2", + "topSimilarity" : 0.7234677980890809 + }, { + "queryId" : "q10", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc4:2", + "topSimilarity" : 0.6519163693633525 + }, { + "queryId" : "q11", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:2", + "topSimilarity" : 0.6701510837170316 + }, { + "queryId" : "q12", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:2", + "topSimilarity" : 0.6452717413230644 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.026042, + "p95Millis" : 6.467208, + "maxMillis" : 6.467208 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 26146.832042, + "p95Millis" : 26850.768375, + "maxMillis" : 26850.768375 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 3.132959, + "p95Millis" : 8.279875, + "maxMillis" : 8.279875 + }, + "embeddingRequestCount" : 4, + "paretoCandidate" : false, + "rounds" : [ { + "chunkingMillis" : 6.467208, + "embeddingMillis" : 26146.832042, + "searchMillis" : 8.279875, + "embeddingRequestCount" : 2 + }, { + "chunkingMillis" : 1.026042, + "embeddingMillis" : 26850.768375, + "searchMillis" : 3.132959, + "embeddingRequestCount" : 2 + } ] + }, { + "profileId" : "c400-o80", + "chunkSize" : 400, + "overlap" : 80, + "chunkCount" : 84, + "originalCodePoints" : 26400, + "chunkCodePoints" : 32160, + "duplicateCodePoints" : 5760, + "duplicateRatio" : 0.21818181818181817, + "quality" : { + "answerCoverageRatio" : 1.0, + "hitAt1" : 0.5, + "hitAt3" : 0.6666666666666666, + "mrrAt10" : 0.6, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc1:1", + "topSimilarity" : 0.7540549747448035 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc2:1", + "topSimilarity" : 0.749721970477369 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:1", + "topSimilarity" : 0.7462990360204615 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc4:2", + "topSimilarity" : 0.7319008689211068 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc5:2", + "topSimilarity" : 0.7320189139351577 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc6:2", + "topSimilarity" : 0.7345189733256429 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 11, + "topCandidateId" : "doc7:3", + "topSimilarity" : 0.7781056075487018 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 13, + "topCandidateId" : "doc8:3", + "topSimilarity" : 0.7498322369779779 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 5, + "topCandidateId" : "doc9:3", + "topSimilarity" : 0.7924142924877039 + }, { + "queryId" : "q10", + "answerCovered" : true, + "firstRelevantRank" : 6, + "topCandidateId" : "doc9:3", + "topSimilarity" : 0.6870622860146378 + }, { + "queryId" : "q11", + "answerCovered" : true, + "firstRelevantRank" : 3, + "topCandidateId" : "doc9:3", + "topSimilarity" : 0.6900303440416493 + }, { + "queryId" : "q12", + "answerCovered" : true, + "firstRelevantRank" : 2, + "topCandidateId" : "doc9:3", + "topSimilarity" : 0.676789400542001 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 2.682625, + "p95Millis" : 12.084958, + "maxMillis" : 12.084958 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 33289.058042, + "p95Millis" : 34516.763958, + "maxMillis" : 34516.763958 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 2.300791, + "p95Millis" : 2.9825, + "maxMillis" : 2.9825 + }, + "embeddingRequestCount" : 4, + "paretoCandidate" : false, + "rounds" : [ { + "chunkingMillis" : 12.084958, + "embeddingMillis" : 34516.763958, + "searchMillis" : 2.9825, + "embeddingRequestCount" : 2 + }, { + "chunkingMillis" : 2.682625, + "embeddingMillis" : 33289.058042, + "searchMillis" : 2.300791, + "embeddingRequestCount" : 2 + } ] + }, { + "profileId" : "c800-o0", + "chunkSize" : 800, + "overlap" : 0, + "chunkCount" : 36, + "originalCodePoints" : 26400, + "chunkCodePoints" : 26400, + "duplicateCodePoints" : 0, + "duplicateRatio" : 0.0, + "quality" : { + "answerCoverageRatio" : 0.5, + "hitAt1" : 0.25, + "hitAt3" : 0.3333333333333333, + "mrrAt10" : 0.31547619047619047, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 2, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6790695446941827 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 7, + "topCandidateId" : "doc12:2", + "topSimilarity" : 0.6893403650966076 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:0", + "topSimilarity" : 0.6409625259330887 + }, { + "queryId" : "q4", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6422960943853532 + }, { + "queryId" : "q5", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6482296162220844 + }, { + "queryId" : "q6", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6665407593531907 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.6852590579358906 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 7, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.7164643163641427 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.7044479036929447 + }, { + "queryId" : "q10", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc4:1", + "topSimilarity" : 0.6559190450876863 + }, { + "queryId" : "q11", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6593161185410963 + }, { + "queryId" : "q12", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6370144683768945 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.966167, + "p95Millis" : 3.862917, + "maxMillis" : 3.862917 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 28615.78275, + "p95Millis" : 30743.704083, + "maxMillis" : 30743.704083 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.932167, + "p95Millis" : 1.301417, + "maxMillis" : 1.301417 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : true, + "rounds" : [ { + "chunkingMillis" : 3.862917, + "embeddingMillis" : 30743.704083, + "searchMillis" : 0.932167, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 0.966167, + "embeddingMillis" : 28615.78275, + "searchMillis" : 1.301417, + "embeddingRequestCount" : 1 + } ] + }, { + "profileId" : "c800-o160", + "chunkSize" : 800, + "overlap" : 160, + "chunkCount" : 48, + "originalCodePoints" : 26400, + "chunkCodePoints" : 32160, + "duplicateCodePoints" : 5760, + "duplicateRatio" : 0.21818181818181817, + "quality" : { + "answerCoverageRatio" : 1.0, + "hitAt1" : 0.5, + "hitAt3" : 0.6666666666666666, + "mrrAt10" : 0.6597222222222222, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6778207321784095 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc4:1", + "topSimilarity" : 0.6510561645726254 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:0", + "topSimilarity" : 0.6409625259330887 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc4:1", + "topSimilarity" : 0.7060108112125649 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc5:1", + "topSimilarity" : 0.7208090727431955 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.7080127853398269 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 6, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6586650481938062 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6827513155016285 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6499167971165735 + }, { + "queryId" : "q10", + "answerCovered" : true, + "firstRelevantRank" : 2, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6557535458820644 + }, { + "queryId" : "q11", + "answerCovered" : true, + "firstRelevantRank" : 2, + "topCandidateId" : "doc4:1", + "topSimilarity" : 0.6634190860335112 + }, { + "queryId" : "q12", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc12:2", + "topSimilarity" : 0.6495438677852051 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 2.109666, + "p95Millis" : 2.389542, + "maxMillis" : 2.389542 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 38248.445167, + "p95Millis" : 38416.442958, + "maxMillis" : 38416.442958 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.291917, + "p95Millis" : 1.6895, + "maxMillis" : 1.6895 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : true, + "rounds" : [ { + "chunkingMillis" : 2.389542, + "embeddingMillis" : 38416.442958, + "searchMillis" : 1.291917, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 2.109666, + "embeddingMillis" : 38248.445167, + "searchMillis" : 1.6895, + "embeddingRequestCount" : 1 + } ] + }, { + "profileId" : "c1000-o0", + "chunkSize" : 1000, + "overlap" : 0, + "chunkCount" : 36, + "originalCodePoints" : 26400, + "chunkCodePoints" : 26400, + "duplicateCodePoints" : 0, + "duplicateRatio" : 0.0, + "quality" : { + "answerCoverageRatio" : 0.75, + "hitAt1" : 0.08333333333333333, + "hitAt3" : 0.16666666666666666, + "mrrAt10" : 0.23921957671957672, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 2, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.6904044091974141 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.6656396724261332 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:0", + "topSimilarity" : 0.6371132458563253 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 7, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.6425385345264588 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 9, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6377573328475455 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 6, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6573300608037055 + }, { + "queryId" : "q7", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6545689520213677 + }, { + "queryId" : "q8", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc8:1", + "topSimilarity" : 0.7040165847552031 + }, { + "queryId" : "q9", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6434803191205066 + }, { + "queryId" : "q10", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc8:1", + "topSimilarity" : 0.6431785530295476 + }, { + "queryId" : "q11", + "answerCovered" : true, + "firstRelevantRank" : 5, + "topCandidateId" : "doc8:1", + "topSimilarity" : 0.6549985842255989 + }, { + "queryId" : "q12", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.6292480225167075 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.203541, + "p95Millis" : 1.3275, + "maxMillis" : 1.3275 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 37793.176792, + "p95Millis" : 41457.951042, + "maxMillis" : 41457.951042 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.915083, + "p95Millis" : 1.831083, + "maxMillis" : 1.831083 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : true, + "rounds" : [ { + "chunkingMillis" : 1.3275, + "embeddingMillis" : 37793.176792, + "searchMillis" : 0.915083, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 1.203541, + "embeddingMillis" : 41457.951042, + "searchMillis" : 1.831083, + "embeddingRequestCount" : 1 + } ] + }, { + "profileId" : "c1000-o200", + "chunkSize" : 1000, + "overlap" : 200, + "chunkCount" : 36, + "originalCodePoints" : 26400, + "chunkCodePoints" : 31200, + "duplicateCodePoints" : 4800, + "duplicateRatio" : 0.18181818181818182, + "quality" : { + "answerCoverageRatio" : 1.0, + "hitAt1" : 0.25, + "hitAt3" : 0.3333333333333333, + "mrrAt10" : 0.30158730158730157, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 3, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6790695446941827 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 7, + "topCandidateId" : "doc12:2", + "topSimilarity" : 0.6893403650966076 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:0", + "topSimilarity" : 0.6371132458563253 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 13, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6423020703851419 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 14, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6482296162220844 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 12, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.6665407593531907 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc7:1", + "topSimilarity" : 0.689658608623764 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 7, + "topCandidateId" : "doc10:2", + "topSimilarity" : 0.7164643163641427 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6977039652276941 + }, { + "queryId" : "q10", + "answerCovered" : true, + "firstRelevantRank" : 13, + "topCandidateId" : "doc12:2", + "topSimilarity" : 0.6536859433031583 + }, { + "queryId" : "q11", + "answerCovered" : true, + "firstRelevantRank" : 11, + "topCandidateId" : "doc9:1", + "topSimilarity" : 0.6591973183194991 + }, { + "queryId" : "q12", + "answerCovered" : true, + "firstRelevantRank" : 11, + "topCandidateId" : "doc6:1", + "topSimilarity" : 0.6369197667974564 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.166667, + "p95Millis" : 1.455208, + "maxMillis" : 1.455208 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 36656.349834, + "p95Millis" : 38899.592708, + "maxMillis" : 38899.592708 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.012375, + "p95Millis" : 1.186834, + "maxMillis" : 1.186834 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : false, + "rounds" : [ { + "chunkingMillis" : 1.455208, + "embeddingMillis" : 36656.349834, + "searchMillis" : 1.012375, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 1.166667, + "embeddingMillis" : 38899.592708, + "searchMillis" : 1.186834, + "embeddingRequestCount" : 1 + } ] + }, { + "profileId" : "c1600-o0", + "chunkSize" : 1600, + "overlap" : 0, + "chunkCount" : 24, + "originalCodePoints" : 26400, + "chunkCodePoints" : 26400, + "duplicateCodePoints" : 0, + "duplicateRatio" : 0.0, + "quality" : { + "answerCoverageRatio" : 0.75, + "hitAt1" : 0.0, + "hitAt3" : 0.0, + "mrrAt10" : 0.0925925925925926, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6790695446941827 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.6893403650966076 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 4, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.621412959436665 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 11, + "topCandidateId" : "doc11:1", + "topSimilarity" : 0.6416130232835789 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 12, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6482296162220844 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 8, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6665407593531907 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 8, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6630577925625493 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 13, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.7164643163641427 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 9, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6409233550075842 + }, { + "queryId" : "q10", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.6536859433031583 + }, { + "queryId" : "q11", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.6548509647252859 + }, { + "queryId" : "q12", + "answerCovered" : false, + "firstRelevantRank" : null, + "topCandidateId" : "doc11:1", + "topSimilarity" : 0.6365183412550758 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 1.300458, + "p95Millis" : 3.021625, + "maxMillis" : 3.021625 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 42823.150167, + "p95Millis" : 47688.757333, + "maxMillis" : 47688.757333 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.549166, + "p95Millis" : 0.591667, + "maxMillis" : 0.591667 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : false, + "rounds" : [ { + "chunkingMillis" : 1.300458, + "embeddingMillis" : 47688.757333, + "searchMillis" : 0.549166, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 3.021625, + "embeddingMillis" : 42823.150167, + "searchMillis" : 0.591667, + "embeddingRequestCount" : 1 + } ] + }, { + "profileId" : "c1600-o320", + "chunkSize" : 1600, + "overlap" : 320, + "chunkCount" : 24, + "originalCodePoints" : 26400, + "chunkCodePoints" : 30240, + "duplicateCodePoints" : 3840, + "duplicateRatio" : 0.14545454545454545, + "quality" : { + "answerCoverageRatio" : 1.0, + "hitAt1" : 0.5, + "hitAt3" : 0.5, + "mrrAt10" : 0.5300925925925926, + "queries" : [ { + "queryId" : "q1", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6471420915832058 + }, { + "queryId" : "q2", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc2:0", + "topSimilarity" : 0.6007945593193365 + }, { + "queryId" : "q3", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc3:0", + "topSimilarity" : 0.6124200172407591 + }, { + "queryId" : "q4", + "answerCovered" : true, + "firstRelevantRank" : 11, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.5664043443544751 + }, { + "queryId" : "q5", + "answerCovered" : true, + "firstRelevantRank" : 12, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.5968825674609654 + }, { + "queryId" : "q6", + "answerCovered" : true, + "firstRelevantRank" : 8, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.5832690317702722 + }, { + "queryId" : "q7", + "answerCovered" : true, + "firstRelevantRank" : 8, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6063545280542689 + }, { + "queryId" : "q8", + "answerCovered" : true, + "firstRelevantRank" : 13, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6306948024727911 + }, { + "queryId" : "q9", + "answerCovered" : true, + "firstRelevantRank" : 9, + "topCandidateId" : "doc1:0", + "topSimilarity" : 0.6128615819920745 + }, { + "queryId" : "q10", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc10:1", + "topSimilarity" : 0.6522574518982497 + }, { + "queryId" : "q11", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc11:1", + "topSimilarity" : 0.6522453506144413 + }, { + "queryId" : "q12", + "answerCovered" : true, + "firstRelevantRank" : 1, + "topCandidateId" : "doc12:1", + "topSimilarity" : 0.6418130607741558 + } ] + }, + "chunkingTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.856042, + "p95Millis" : 1.125625, + "maxMillis" : 1.125625 + }, + "embeddingTiming" : { + "sampleCount" : 2, + "medianMillis" : 43325.423167, + "p95Millis" : 45104.538875, + "maxMillis" : 45104.538875 + }, + "searchTiming" : { + "sampleCount" : 2, + "medianMillis" : 0.698083, + "p95Millis" : 0.720375, + "maxMillis" : 0.720375 + }, + "embeddingRequestCount" : 2, + "paretoCandidate" : true, + "rounds" : [ { + "chunkingMillis" : 1.125625, + "embeddingMillis" : 43325.423167, + "searchMillis" : 0.698083, + "embeddingRequestCount" : 1 + }, { + "chunkingMillis" : 0.856042, + "embeddingMillis" : 45104.538875, + "searchMillis" : 0.720375, + "embeddingRequestCount" : 1 + } ] + } ] +} \ No newline at end of file diff --git a/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark.md b/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark.md new file mode 100644 index 0000000..0c7da95 --- /dev/null +++ b/docs/test-results/gimin-#147-chunk-size-overlap-quality-benchmark.md @@ -0,0 +1,135 @@ +# Chunk Size·Overlap 검색 품질 및 비용 실측 결과 + +- 관련 이슈: [#147](https://github.com/DocGrid/backend/issues/147) +- 측정 일시: 2026-08-11 +- 상태: 실제 BGE-M3 측정 완료 +- 원본 데이터: [`gimin-#147-chunk-size-overlap-quality-benchmark-data.json`](./gimin-%23147-chunk-size-overlap-quality-benchmark-data.json) + +## 1. 결론 + +결정적 경계 Corpus에서는 모든 20% Overlap Profile이 Answer Coverage 100%를 달성했다. Overlap이 +없는 Profile은 근거 문장이 Chunk 경계에서 분리돼 Coverage가 25~75%로 낮아졌다. 따라서 경계 근거 +보존에는 Overlap이 효과가 있었다. + +검색 순위 품질은 `800/160`이 Hit@3 `66.7%`, MRR@10 `0.660`으로 가장 높았다. 같은 Hit@3를 +기록한 `400/80`보다 Chunk 수가 `84 → 48`로 적고 MRR도 높았다. 현재 기본 `1000/200`은 Coverage는 +100%지만 Hit@3 `33.3%`, MRR@10 `0.302`로 합성 Corpus의 최적 Profile은 아니었다. + +다만 이 결과는 의도적으로 근거를 경계에 배치한 12개 합성 Query의 상대 비교다. 실제 사용자 문서의 +길이·질문 분포를 대표하지 않으므로 제품 기본값은 변경하지 않는다. 실제 문서 검색 평가셋으로 재검증한 +뒤 별도 설정 변경 PR에서 결정해야 한다. + +![Chunk Size·Overlap 검색 품질과 중복 비용 Trade-off](./assets/chunk-quality/chunk-quality-tradeoff.svg) + +## 2. 측정 환경 + +| 항목 | 값 | +|---|---| +| Host | macOS, Apple Silicon `aarch64` | +| Java | 17.0.18 | +| Embedding Model | 실제 `BAAI/bge-m3`, Docker CPU 추론 | +| Vector | Dense 1024차원, 모든 값 유한, 0이 아닌 Norm | +| Corpus | 12문서, 문서당 2,200 Code Point, 총 26,400 Code Point | +| Query | 400·800·1000·1600 경계마다 3개, 총 12개 | +| 검색 | 메모리 내 Exact Cosine, DB·HNSW 제외 | +| Warm-up | 1회 | +| 본 측정 | Profile별 2회, 시작 순서 회전 | +| Model Batch Size | 32 | +| HTTP 요청당 최대 Text | 64 | +| Query Embedding | 12개, 572.52ms, 1회 요청 | + +## 3. 품질·비용 비교 + +| Chunk/Overlap | Coverage | Hit@1 | Hit@3 | MRR@10 | Chunk 수 | 중복 비율 | Embedding Median | Embedding p95 | Pareto | +|---|---:|---:|---:|---:|---:|---:|---:|---:|:---:| +| `400/0` | 25.0% | 16.7% | 16.7% | 0.176 | 72 | 0.0% | 26.15s | 26.85s | | +| `400/80` | 100.0% | 50.0% | 66.7% | 0.600 | 84 | 21.8% | 33.29s | 34.52s | | +| `800/0` | 50.0% | 25.0% | 33.3% | 0.315 | 36 | 0.0% | 28.62s | 30.74s | ✓ | +| `800/160` | **100.0%** | **50.0%** | **66.7%** | **0.660** | 48 | 21.8% | 38.25s | 38.42s | ✓ | +| `1000/0` | 75.0% | 8.3% | 16.7% | 0.239 | 36 | 0.0% | 37.79s | 41.46s | ✓ | +| `1000/200` 현재 기본 | 100.0% | 25.0% | 33.3% | 0.302 | 36 | 18.2% | 36.66s | 38.90s | | +| `1600/0` | 75.0% | 0.0% | 0.0% | 0.093 | 24 | 0.0% | 42.82s | 47.69s | | +| `1600/320` | 100.0% | 50.0% | 50.0% | 0.530 | 24 | **14.5%** | 43.33s | 45.10s | ✓ | + +Pareto 표시는 Coverage·Hit@1·Hit@3·MRR@10은 높을수록 좋고 Chunk Code Point 수는 낮을수록 +좋다는 기준으로 다른 Profile에 완전히 지배되지 않은 조합이다. 실측 지연은 Host 열 상태의 영향을 +받으므로 Pareto 판정에는 결정적인 Chunk Code Point 비용만 사용했다. + +## 4. 관찰 결과 + +### 4.1 Overlap의 경계 근거 복구 + +- `400/0 → 400/80`: Coverage `25% → 100%`, MRR `0.176 → 0.600` +- `800/0 → 800/160`: Coverage `50% → 100%`, MRR `0.315 → 0.660` +- `1000/0 → 1000/200`: Coverage `75% → 100%`, MRR `0.239 → 0.302` +- `1600/0 → 1600/320`: Coverage `75% → 100%`, MRR `0.093 → 0.530` + +모든 크기에서 20% Overlap이 완전한 근거 Chunk를 복구했다. 다만 Coverage 회복이 곧 같은 순위 +개선을 뜻하지는 않았다. 큰 Chunk는 질문과 무관한 채움 Text 비율이 높아 `1600/320`의 Coverage가 +100%여도 Hit@3는 50%에 머물렀다. + +### 4.2 중복·임베딩 비용 + +- 20% 설정의 실제 중복 비율은 마지막 짧은 Chunk 영향으로 14.5~21.8%였다. +- 가장 작은 `400/80`은 84개 Chunk를 만들었고, `1600/320`은 24개를 만들었다. +- Embedding p95는 26.85~47.69초 범위였다. Text 수뿐 아니라 긴 Sequence의 CPU 추론 비용이 + 영향을 주어 큰 Chunk가 항상 빠르지 않았다. +- Exact 검색 p95는 모든 Profile에서 8.28ms 이하였지만 Candidate가 최대 84개인 Micro + Benchmark라 운영 Vector 검색 성능으로 해석하지 않는다. + +### 4.3 현재 기본값 판단 + +`1000/200`은 경계 근거 보존에는 성공했지만 이번 Corpus에서는 `800/160`보다 Hit@3가 33.4%p, +MRR@10이 0.358 낮았다. 반면 중복 Code Point는 `800/160`보다 960개 적었다. 품질–비용 Trade-off가 +있고 실제 문서 평가셋이 없으므로 현재 기본값을 유지한다. + +후속 기본값 판단에서는 실제 PDF·DOCX에서 수집한 질문–근거 쌍, 문서 제목·문단 Metadata와 Token +기준 Chunking을 함께 비교해야 한다. + +## 5. 재현 방법 + +```bash +docker compose up -d embedding-server +./gradlew chunkQualityPerformanceTest +``` + +측정 Round와 Batch Size를 늘릴 때는 다음처럼 실행한다. + +```bash +./gradlew chunkQualityPerformanceTest \ + -Dchunk.quality.performance.rounds=3 \ + -Dchunk.quality.performance.batch-size=32 \ + -Dchunk.quality.performance.output=build/reports/chunk-quality/chunk-quality.json +``` + +기본 출력은 `build/reports/chunk-quality/chunk-quality-latest.json`이다. 이 문서에 연결된 원본 JSON은 +성공한 기본 실행 결과를 그대로 보존한다. + +## 6. 검증 결과 + +| 검증 | 결과 | +|---|---| +| Corpus·Ground Truth·Hit@K·MRR 단위 테스트 | ✅ 성공 | +| 일반 회귀 테스트 | ✅ 734개 성공 | +| 실제 BGE-M3 전용 Benchmark | ✅ 9분 52초, 8 Profile × 2 Round 성공 | +| Model명·응답 개수·순서 | ✅ 모두 일치 | +| 1024차원·유한값·0이 아닌 Norm | ✅ 모두 통과 | +| HTTP·응답 계약·Vector 불변식 | ✅ 모든 검증 통과 후 JSON 생성 | +| Profile별 반복 품질 결정성 | ✅ 모두 일치 | + +일반 회귀 테스트는 로컬 PostgreSQL의 SSL 미지원과 필수 테스트 JWT를 반영해 다음 환경으로 실행했다. + +```bash +DB_SSLMODE=disable \ +JWT_SECRET=<로컬-테스트용-임의-문자열> \ +./gradlew test +``` + +## 7. 해석 한계 + +- 합성 Corpus는 Chunk 경계 손실을 의도적으로 강조한다. +- 12개 Query는 통계적으로 운영 검색 품질을 대표하지 않는다. +- Apple Silicon Docker CPU 절대 지연은 운영 GPU·Rocky Linux 환경과 직접 비교할 수 없다. +- Exact Cosine을 사용했으므로 HNSW Recall과 DB 실행 계획은 포함하지 않는다. +- Parser·OCR·페이지·섹션 Metadata와 RAG 답변 품질은 포함하지 않는다. +- 결과는 파라미터 후보를 좁히는 근거이며 운영 SLO나 기본값 변경 승인이 아니다. diff --git a/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupport.java b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupport.java new file mode 100644 index 0000000..0ce8a92 --- /dev/null +++ b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupport.java @@ -0,0 +1,448 @@ +package com.opensource.docgrid.domain.document.benchmark; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.opensource.docgrid.domain.document.config.DocumentChunkingProperties; +import com.opensource.docgrid.domain.document.service.DocumentChunkDraft; +import com.opensource.docgrid.domain.document.service.FixedSizeChunker; + +/** + * Chunk Size·Overlap Benchmark의 결정적 Corpus, Ground Truth와 품질·비용 계산 계약을 제공한다. + * + *

실제 모델이나 파일·DB I/O를 사용하지 않는 순수 계산 경계다. 외부 Benchmark와 일반 단위 + * 테스트가 같은 입력과 Relevant 판정, Exact Cosine 순위를 공유하게 해 실측 해석의 변형을 막는다. + */ +final class ChunkQualityBenchmarkSupport { + + static final int VECTOR_DIMENSION = 1024; + static final int DOCUMENT_LENGTH = 2_200; + static final int TOP_K = 10; + private static final int EVIDENCE_BOUNDARY_OFFSET = 45; + + private static final int[] BOUNDARIES = {400, 800, 1_000, 1_600}; + private static final String[] PROJECTS = { + "해오름", "은하수", "푸른샘", "노을빛", "별무리", "새벽길", + "가람", "미리내", "솔바람", "구름재", "달맞이", "바다숲" + }; + private static final String[] MARKERS = { + "청록색 솔방울", "자주색 나침반", "황금색 종이학", "은색 모래시계", + "주황색 풍차", "남색 조약돌", "연두색 등대", "분홍색 우산", + "하늘색 책갈피", "갈색 망원경", "보라색 연", "초록색 종" + }; + private static final String[] OWNERS = { + "세림", "도윤", "하린", "지후", "예린", "수현", + "태오", "민서", "유진", "현우", "서아", "준호" + }; + private static final String FILLER = + "이 문단은 일반적인 시스템 운영 배경과 절차를 설명한다. 기준 정보는 별도 문장에 기록된다. "; + + private ChunkQualityBenchmarkSupport() { + } + + /** + * 비교 대상 8개 Profile을 작은 Chunk부터 결정적인 순서로 반환한다. + */ + static List profiles() { + return List.of( + new ChunkProfile("c400-o0", 400, 0), + new ChunkProfile("c400-o80", 400, 80), + new ChunkProfile("c800-o0", 800, 0), + new ChunkProfile("c800-o160", 800, 160), + new ChunkProfile("c1000-o0", 1_000, 0), + new ChunkProfile("c1000-o200", 1_000, 200), + new ChunkProfile("c1600-o0", 1_600, 0), + new ChunkProfile("c1600-o320", 1_600, 320) + ); + } + + /** + * 네 Chunk 경계마다 3개의 고유한 한국어 근거 문서·질의를 생성한다. + */ + static List createCorpus() { + List corpus = new ArrayList<>(); + int caseIndex = 0; + + // 1. 각 비교 Chunk 크기의 첫 경계 주변에 같은 수의 Case를 배치한다. + for (int boundary : BOUNDARIES) { + for (int repetition = 0; repetition < 3; repetition++) { + String project = PROJECTS[caseIndex]; + String marker = MARKERS[caseIndex]; + String owner = OWNERS[caseIndex]; + String code = "DG-147-" + (char) ('A' + caseIndex); + int intervalSeconds = 31 + caseIndex * 2; + String evidence = "프로젝트 " + project + "의 장애 복구 표식은 " + marker + + "이고 승인 코드는 " + code + "이다. 야간 복구 담당자는 " + owner + + "이며 상태 확인 간격은 " + intervalSeconds + "초다."; + String question = "프로젝트 " + project + + "의 승인 코드, 야간 복구 담당자와 상태 확인 간격은 무엇인가?"; + int evidenceStart = boundary - EVIDENCE_BOUNDARY_OFFSET; + + // 2. 근거 시작 Offset을 먼저 고정한 뒤 문서 길이를 정확히 맞춰 경계 조건을 보존한다. + String text = padToCodePointLength(FILLER, evidenceStart) + + evidence + + padToCodePointLength(FILLER, DOCUMENT_LENGTH - evidenceStart - codePointLength(evidence)); + corpus.add(new QueryCase( + "q" + (caseIndex + 1), + "doc" + (caseIndex + 1), + question, + text, + evidence, + evidenceStart, + evidenceStart + codePointLength(evidence), + boundary + )); + caseIndex++; + } + } + return List.copyOf(corpus); + } + + /** + * 제품 FixedSizeChunker를 주어진 Profile로 실행해 검색 후보와 비용을 계산한다. + */ + static ChunkedCorpus chunk(List corpus, ChunkProfile profile) { + DocumentChunkingProperties properties = new DocumentChunkingProperties(); + properties.setChunkSize(profile.chunkSize()); + properties.setOverlap(profile.overlap()); + FixedSizeChunker chunker = new FixedSizeChunker(properties); + List candidates = new ArrayList<>(); + long originalCodePoints = 0L; + long chunkCodePoints = 0L; + + // 1. 실제 제품 Chunker를 문서별로 실행해 Offset과 Chunk Text를 그대로 사용한다. + for (QueryCase queryCase : corpus) { + originalCodePoints += codePointLength(queryCase.documentText()); + List drafts = chunker.chunk(queryCase.documentText()); + for (DocumentChunkDraft draft : drafts) { + int length = codePointLength(draft.chunkText()); + chunkCodePoints += length; + candidates.add(new ChunkCandidate( + queryCase.documentId() + ":" + draft.chunkIndex(), + queryCase.documentId(), + draft.chunkIndex(), + draft.chunkText(), + draft.charStart(), + draft.charEnd() + )); + } + } + + long duplicateCodePoints = chunkCodePoints - originalCodePoints; + double duplicateRatio = originalCodePoints == 0L + ? 0.0 + : (double) duplicateCodePoints / originalCodePoints; + return new ChunkedCorpus( + List.copyOf(candidates), + originalCodePoints, + chunkCodePoints, + duplicateCodePoints, + duplicateRatio + ); + } + + /** + * 실제 Vector의 Exact Cosine 순위와 완전한 근거 범위로 검색 품질을 계산한다. + */ + static QualityMetrics evaluate( + List corpus, + List candidates, + Map queryVectors, + Map chunkVectors + ) { + int coveredQueries = 0; + int hitAt1 = 0; + int hitAt3 = 0; + double reciprocalRankSum = 0.0; + List queryResults = new ArrayList<>(); + + for (QueryCase queryCase : corpus) { + List rankedChunks = rank( + candidates, + requiredVector(queryVectors, queryCase.queryId()), + chunkVectors + ); + boolean covered = candidates.stream().anyMatch(candidate -> isRelevant(queryCase, candidate)); + if (covered) { + coveredQueries++; + } + + Integer firstRelevantRank = null; + for (int index = 0; index < rankedChunks.size(); index++) { + if (isRelevant(queryCase, rankedChunks.get(index).candidate())) { + firstRelevantRank = index + 1; + break; + } + } + if (firstRelevantRank != null && firstRelevantRank == 1) { + hitAt1++; + } + if (firstRelevantRank != null && firstRelevantRank <= 3) { + hitAt3++; + } + if (firstRelevantRank != null && firstRelevantRank <= TOP_K) { + reciprocalRankSum += 1.0 / firstRelevantRank; + } + + RankedChunk top = rankedChunks.get(0); + queryResults.add(new QueryQualityResult( + queryCase.queryId(), + covered, + firstRelevantRank, + top.candidate().candidateId(), + top.similarity() + )); + } + + int queryCount = corpus.size(); + return new QualityMetrics( + ratio(coveredQueries, queryCount), + ratio(hitAt1, queryCount), + ratio(hitAt3, queryCount), + reciprocalRankSum / queryCount, + List.copyOf(queryResults) + ); + } + + /** + * Vector 차원과 유한값, 0이 아닌 Norm을 검증한다. + */ + static void validateVector(float[] vector) { + validatedNorm(vector); + } + + private static double validatedNorm(float[] vector) { + if (vector == null || vector.length != VECTOR_DIMENSION) { + throw new IllegalArgumentException("Embedding Vector는 1024차원이어야 합니다."); + } + double squaredNorm = 0.0; + for (float value : vector) { + if (!Float.isFinite(value)) { + throw new IllegalArgumentException("Embedding Vector는 유한값만 포함해야 합니다."); + } + squaredNorm += value * value; + } + if (squaredNorm == 0.0) { + throw new IllegalArgumentException("Embedding Vector Norm은 0보다 커야 합니다."); + } + return Math.sqrt(squaredNorm); + } + + /** + * 같은 차원의 두 Dense Vector 사이 Cosine Similarity를 계산한다. + */ + static double cosineSimilarity(float[] left, float[] right) { + return cosineSimilarity(left, validatedNorm(left), right); + } + + private static double cosineSimilarity(float[] left, double leftNorm, float[] right) { + double rightNorm = validatedNorm(right); + double dotProduct = 0.0; + for (int index = 0; index < left.length; index++) { + dotProduct += left[index] * right[index]; + } + return dotProduct / (leftNorm * rightNorm); + } + + /** + * Millisecond 표본의 Nearest-rank Median과 p95를 계산한다. + */ + static TimingSummary summarizeTimings(List samplesMillis) { + if (samplesMillis == null || samplesMillis.isEmpty()) { + throw new IllegalArgumentException("Timing 표본은 한 개 이상이어야 합니다."); + } + if (samplesMillis.stream().anyMatch(value -> value == null || !Double.isFinite(value) || value < 0.0)) { + throw new IllegalArgumentException("Timing 표본은 0 이상의 유한값이어야 합니다."); + } + List sorted = samplesMillis.stream().sorted().toList(); + return new TimingSummary( + sorted.size(), + nearestRank(sorted, 0.50), + nearestRank(sorted, 0.95), + sorted.get(sorted.size() - 1) + ); + } + + private static List rank( + List candidates, + float[] queryVector, + Map chunkVectors + ) { + double queryNorm = validatedNorm(queryVector); + List ranked = new ArrayList<>(candidates.size()); + for (ChunkCandidate candidate : candidates) { + ranked.add(new RankedChunk( + candidate, + cosineSimilarity( + queryVector, + queryNorm, + requiredVector(chunkVectors, candidate.candidateId()) + ) + )); + } + ranked.sort( + Comparator.comparingDouble(RankedChunk::similarity).reversed() + .thenComparing(result -> result.candidate().documentId()) + .thenComparingInt(result -> result.candidate().chunkIndex()) + ); + return ranked; + } + + static boolean isRelevant(QueryCase queryCase, ChunkCandidate candidate) { + return queryCase.documentId().equals(candidate.documentId()) + && candidate.charStart() <= queryCase.evidenceStart() + && candidate.charEnd() >= queryCase.evidenceEnd(); + } + + private static float[] requiredVector(Map vectors, String key) { + float[] vector = vectors.get(key); + if (vector == null) { + throw new IllegalArgumentException("Vector가 누락됐습니다: " + key); + } + return vector; + } + + private static String padToCodePointLength(String pattern, int targetLength) { + if (targetLength < 0) { + throw new IllegalArgumentException("Padding 길이는 0 이상이어야 합니다."); + } + int[] patternCodePoints = pattern.codePoints().toArray(); + int[] result = new int[targetLength]; + for (int index = 0; index < targetLength; index++) { + result[index] = patternCodePoints[index % patternCodePoints.length]; + } + return new String(result, 0, result.length); + } + + private static int codePointLength(String value) { + return value.codePointCount(0, value.length()); + } + + private static double ratio(int numerator, int denominator) { + return denominator == 0 ? 0.0 : (double) numerator / denominator; + } + + private static double nearestRank(List sorted, double percentile) { + int rank = Math.max(1, (int) Math.ceil(percentile * sorted.size())); + return sorted.get(rank - 1); + } + + /** + * 비교할 Chunk Size와 Overlap의 불변 조합이다. + */ + record ChunkProfile(String profileId, int chunkSize, int overlap) { + + ChunkProfile { + if (profileId == null || profileId.isBlank()) { + throw new IllegalArgumentException("Profile ID는 비어 있을 수 없습니다."); + } + if (chunkSize <= 0 || overlap < 0 || overlap >= chunkSize) { + throw new IllegalArgumentException("Chunk Size·Overlap 조합이 유효하지 않습니다."); + } + } + } + + /** + * 한 Query의 문서 본문과 완전한 정답 근거 범위를 결합한 Ground Truth다. + */ + record QueryCase( + String queryId, + String documentId, + String question, + String documentText, + String evidence, + int evidenceStart, + int evidenceEnd, + int targetBoundary + ) { + } + + /** + * 제품 Chunker 결과에 검색 후보 식별자와 원문 범위를 결합한다. + */ + record ChunkCandidate( + String candidateId, + String documentId, + int chunkIndex, + String text, + int charStart, + int charEnd + ) { + } + + /** + * 한 Profile이 생성한 Chunk 목록과 결정적인 중복 비용을 보관한다. + */ + record ChunkedCorpus( + List candidates, + long originalCodePoints, + long chunkCodePoints, + long duplicateCodePoints, + double duplicateRatio + ) { + + ChunkedCorpus { + candidates = List.copyOf(candidates); + } + } + + /** + * 전체 Query의 Answer Coverage, Hit@K와 MRR 결과다. + */ + record QualityMetrics( + double answerCoverageRatio, + double hitAt1, + double hitAt3, + double mrrAt10, + List queries + ) { + + QualityMetrics { + queries = List.copyOf(queries); + } + } + + /** + * 반복 실행 지연의 표본 수, Median, p95와 최댓값을 보관한다. + */ + record TimingSummary(int sampleCount, double medianMillis, double p95Millis, double maxMillis) { + } + + /** + * 한 Query의 근거 보존 여부와 Exact 검색 결과를 설명한다. + */ + record QueryQualityResult( + String queryId, + boolean answerCovered, + Integer firstRelevantRank, + String topCandidateId, + double topSimilarity + ) { + } + + /** + * Exact Ranking 내부에서 Chunk와 Cosine Similarity를 결합한다. + */ + private record RankedChunk(ChunkCandidate candidate, double similarity) { + } + + /** + * Vector Map을 입력 순서대로 만들 때 사용하는 결정적 변환 Helper다. + */ + static Map vectorMap(List ids, List vectors) { + if (ids.size() != vectors.size()) { + throw new IllegalArgumentException("ID와 Vector 개수가 일치해야 합니다."); + } + Map result = new LinkedHashMap<>(); + for (int index = 0; index < ids.size(); index++) { + validateVector(vectors.get(index)); + if (result.put(ids.get(index), vectors.get(index).clone()) != null) { + throw new IllegalArgumentException("중복된 Vector ID입니다: " + ids.get(index)); + } + } + return Map.copyOf(result); + } +} diff --git a/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupportTest.java b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupportTest.java new file mode 100644 index 0000000..c3e1b8f --- /dev/null +++ b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityBenchmarkSupportTest.java @@ -0,0 +1,230 @@ +package com.opensource.docgrid.domain.document.benchmark; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkCandidate; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkProfile; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkedCorpus; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.QualityMetrics; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.QueryCase; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.TimingSummary; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityPerformanceBenchmark.BenchmarkConfiguration; + +/** + * 실제 모델 없이 Chunk 품질 Benchmark의 Corpus, Ground Truth와 Exact 품질 계산을 검증한다. + */ +@DisplayName("Chunk Size·Overlap 품질 Benchmark 계약 테스트") +class ChunkQualityBenchmarkSupportTest { + + @Test + @DisplayName("8개 Profile과 12개 경계 Query를 결정적인 순서로 생성한다") + void createCorpus_preservesProfilesAndBoundaryCases() { + List profiles = ChunkQualityBenchmarkSupport.profiles(); + List corpus = ChunkQualityBenchmarkSupport.createCorpus(); + + assertThat(profiles).extracting(ChunkProfile::profileId) + .containsExactly( + "c400-o0", "c400-o80", "c800-o0", "c800-o160", + "c1000-o0", "c1000-o200", "c1600-o0", "c1600-o320" + ); + assertThat(corpus).hasSize(12); + assertThat(corpus).extracting(QueryCase::targetBoundary) + .containsExactly(400, 400, 400, 800, 800, 800, 1_000, 1_000, 1_000, 1_600, 1_600, 1_600); + assertThat(corpus).allSatisfy(queryCase -> { + assertThat(queryCase.documentText().codePointCount(0, queryCase.documentText().length())) + .isEqualTo(ChunkQualityBenchmarkSupport.DOCUMENT_LENGTH); + assertThat(queryCase.documentText().codePoints() + .skip(queryCase.evidenceStart()) + .limit(queryCase.evidence().codePointCount(0, queryCase.evidence().length())) + .toArray()) + .containsExactly(queryCase.evidence().codePoints().toArray()); + }); + } + + @Test + @DisplayName("20% Overlap은 대응 경계의 완전한 근거 Chunk를 복구한다") + void chunk_overlapRestoresEvidenceAcrossTargetBoundary() { + List corpus = ChunkQualityBenchmarkSupport.createCorpus(); + + assertBoundaryCoverage(corpus, new ChunkProfile("without-overlap", 1_000, 0), 1_000, false); + assertBoundaryCoverage(corpus, new ChunkProfile("with-overlap", 1_000, 200), 1_000, true); + } + + @Test + @DisplayName("완전한 근거 범위만 Relevant로 사용해 Hit@K와 MRR을 계산한다") + void evaluate_usesFullEvidenceRangeAsGroundTruth() { + QueryCase first = queryCase("q1", "doc1", 10, 20); + QueryCase second = queryCase("q2", "doc2", 10, 20); + List chunks = List.of( + candidate("doc1:partial", "doc1", 0, 10, 15), + candidate("doc1:answer", "doc1", 1, 5, 25), + candidate("doc2:answer", "doc2", 0, 5, 25) + ); + + float[] axisX = vector(1.0F, 0.0F); + float[] axisY = vector(0.0F, 1.0F); + float[] diagonal = vector(0.8F, 0.6F); + Map queryVectors = Map.of("q1", axisX, "q2", axisY); + Map chunkVectors = Map.of( + "doc1:partial", axisX, + "doc1:answer", diagonal, + "doc2:answer", axisY + ); + + QualityMetrics metrics = ChunkQualityBenchmarkSupport.evaluate( + List.of(first, second), + chunks, + queryVectors, + chunkVectors + ); + + assertThat(metrics.answerCoverageRatio()).isEqualTo(1.0); + assertThat(metrics.hitAt1()).isEqualTo(0.5); + assertThat(metrics.hitAt3()).isEqualTo(1.0); + assertThat(metrics.mrrAt10()).isEqualTo(0.75); + assertThat(metrics.queries()).extracting(result -> result.firstRelevantRank()) + .containsExactly(2, 1); + } + + @Test + @DisplayName("Vector 차원·유한값·Norm 불변식을 위반하면 거부한다") + void validateVector_rejectsInvalidDenseVectors() { + assertThatThrownBy(() -> ChunkQualityBenchmarkSupport.validateVector(new float[3])) + .isInstanceOf(IllegalArgumentException.class); + + float[] notFinite = vector(1.0F, 0.0F); + notFinite[7] = Float.NaN; + assertThatThrownBy(() -> ChunkQualityBenchmarkSupport.validateVector(notFinite)) + .isInstanceOf(IllegalArgumentException.class); + + assertThatThrownBy(() -> ChunkQualityBenchmarkSupport.validateVector(new float[1_024])) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("잘못된 Chunk Size·Overlap 조합을 생성 단계에서 거부한다") + void chunkProfile_rejectsInvalidOverlap() { + assertThatThrownBy(() -> new ChunkProfile("invalid", 400, 400)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new ChunkProfile("invalid", 400, -1)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("반복 지연의 Nearest-rank Median과 p95를 계산한다") + void summarizeTimings_usesNearestRankPercentiles() { + TimingSummary summary = ChunkQualityBenchmarkSupport.summarizeTimings( + List.of(8.0, 1.0, 4.0, 2.0, 7.0, 3.0, 6.0, 5.0) + ); + + assertThat(summary.sampleCount()).isEqualTo(8); + assertThat(summary.medianMillis()).isEqualTo(4.0); + assertThat(summary.p95Millis()).isEqualTo(8.0); + assertThat(summary.maxMillis()).isEqualTo(8.0); + } + + @Test + @DisplayName("Timing null 표본은 정렬 전에 명시적인 입력 오류로 거부한다") + void summarizeTimings_rejectsNullBeforeSorting() { + List samples = new ArrayList<>(); + samples.add(1.0); + samples.add(null); + + assertThatThrownBy(() -> ChunkQualityBenchmarkSupport.summarizeTimings(samples)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Timing 표본은 0 이상의 유한값이어야 합니다."); + } + + @Test + @DisplayName("Vector Map은 중복 ID로 앞선 Vector가 덮어써지는 것을 거부한다") + void vectorMap_rejectsDuplicateIds() { + assertThatThrownBy(() -> ChunkQualityBenchmarkSupport.vectorMap( + List.of("duplicate", "duplicate"), + List.of(vector(1.0F, 0.0F), vector(0.0F, 1.0F)) + )) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("중복된 Vector ID입니다: duplicate"); + } + + @Test + @DisplayName("Benchmark 설정은 직접 생성해도 URL과 실행 횟수 불변식을 검증한다") + void benchmarkConfiguration_validatesEveryConstructionPath() { + assertThatThrownBy(() -> new BenchmarkConfiguration( + URI.create("ftp://localhost:8000"), + 1, + 2, + 32, + Path.of("result.json") + )) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Embedding Server URL은 HTTP 또는 HTTPS여야 합니다."); + + assertThatThrownBy(() -> new BenchmarkConfiguration( + URI.create("http://localhost:8000"), + 0, + 2, + 32, + Path.of("result.json") + )) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Warm-up과 측정 Round는 각각 1 이상이어야 합니다."); + } + + private void assertBoundaryCoverage( + List corpus, + ChunkProfile profile, + int targetBoundary, + boolean expected + ) { + List targetCases = corpus.stream() + .filter(queryCase -> queryCase.targetBoundary() == targetBoundary) + .toList(); + ChunkedCorpus chunked = ChunkQualityBenchmarkSupport.chunk(targetCases, profile); + + for (QueryCase queryCase : targetCases) { + boolean covered = chunked.candidates().stream() + .anyMatch(candidate -> ChunkQualityBenchmarkSupport.isRelevant(queryCase, candidate)); + assertThat(covered).isEqualTo(expected); + } + } + + private QueryCase queryCase(String queryId, String documentId, int evidenceStart, int evidenceEnd) { + return new QueryCase( + queryId, + documentId, + "질문", + "본문", + "근거", + evidenceStart, + evidenceEnd, + 100 + ); + } + + private ChunkCandidate candidate( + String candidateId, + String documentId, + int chunkIndex, + int charStart, + int charEnd + ) { + return new ChunkCandidate(candidateId, documentId, chunkIndex, "본문", charStart, charEnd); + } + + private float[] vector(float first, float second) { + float[] vector = new float[ChunkQualityBenchmarkSupport.VECTOR_DIMENSION]; + vector[0] = first; + vector[1] = second; + return vector; + } +} diff --git a/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java new file mode 100644 index 0000000..d68f077 --- /dev/null +++ b/src/test/java/com/opensource/docgrid/domain/document/benchmark/ChunkQualityPerformanceBenchmark.java @@ -0,0 +1,515 @@ +package com.opensource.docgrid.domain.document.benchmark; + +import static com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.summarizeTimings; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.springframework.web.client.RestClient; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkCandidate; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkProfile; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.ChunkedCorpus; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.QualityMetrics; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.QueryCase; +import com.opensource.docgrid.domain.document.benchmark.ChunkQualityBenchmarkSupport.TimingSummary; +import com.opensource.docgrid.domain.embedding.client.EmbeddingClient; +import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchItemResponse; +import com.opensource.docgrid.domain.embedding.dto.response.EmbedBatchServerResponse; + +import lombok.extern.slf4j.Slf4j; + +/** + * 실제 FixedSizeChunker와 BAAI/bge-m3를 사용해 Chunk Size·Overlap별 검색 품질과 비용을 비교한다. + * + *

Chunking 효과를 격리하기 위해 DB·HNSW 대신 메모리 내 Exact Cosine Ranking을 사용한다. + * 실제 외부 모델을 호출하는 장시간 작업이므로 일반 테스트에서 제외하고 전용 Gradle Task로만 실행한다. + */ +@Slf4j +@Tag("chunk-quality-performance") +@DisplayName("Chunk Size·Overlap 검색 품질·비용 Benchmark") +class ChunkQualityPerformanceBenchmark { + + private static final String EXPECTED_MODEL = "BAAI/bge-m3"; + private static final int MAX_REQUEST_TEXTS = 64; + private static final double COMPARISON_EPSILON = 1.0E-12; + + @Test + @Timeout(value = 3_600, unit = TimeUnit.SECONDS) + @DisplayName("8개 Profile의 실제 BGE-M3 품질과 중복·임베딩 비용을 비교한다") + void compareChunkSizeAndOverlapQuality() throws IOException { + BenchmarkConfiguration configuration = BenchmarkConfiguration.fromSystemProperties(); + RestClient restClient = RestClient.builder().baseUrl(configuration.serverUri().toString()).build(); + EmbeddingClient embeddingClient = new EmbeddingClient(restClient); + List corpus = ChunkQualityBenchmarkSupport.createCorpus(); + List profiles = ChunkQualityBenchmarkSupport.profiles(); + + // 1. Health와 Warm-up을 측정 전에 끝내 Model Loading 시간을 Profile 지연에서 제외한다. + verifyHealth(restClient); + warmUp(embeddingClient, configuration, corpus); + + // 2. Query Vector를 한 번만 생성해 모든 Profile이 같은 검색 입력을 공유하게 한다. + long queryEmbeddingStartedAt = System.nanoTime(); + EmbeddedVectors queryEmbeddings = embed( + embeddingClient, + corpus.stream().map(QueryCase::queryId).toList(), + corpus.stream().map(QueryCase::question).toList(), + configuration.batchSize() + ); + double queryEmbeddingMillis = nanosToMillis(System.nanoTime() - queryEmbeddingStartedAt); + + // 3. Round마다 Profile 시작 순서를 회전해 실행 순서와 Host 열 상태 편향을 줄인다. + Map accumulators = new LinkedHashMap<>(); + profiles.forEach(profile -> accumulators.put(profile.profileId(), new ProfileAccumulator(profile))); + for (int round = 0; round < configuration.rounds(); round++) { + for (int offset = 0; offset < profiles.size(); offset++) { + ChunkProfile profile = profiles.get((round + offset) % profiles.size()); + RoundMeasurement measurement = runProfileRound( + profile, + corpus, + embeddingClient, + configuration.batchSize(), + queryEmbeddings.vectors() + ); + accumulators.get(profile.profileId()).add(measurement); + } + } + + // 4. 결정적 품질·비용과 반복 지연을 집계하고 비용 대비 비지배 Profile을 표시한다. + List preliminary = profiles.stream() + .map(profile -> accumulators.get(profile.profileId()).toResult(false)) + .toList(); + List results = preliminary.stream() + .map(result -> result.withParetoCandidate(isParetoCandidate(result, preliminary))) + .toList(); + + BenchmarkReport report = new BenchmarkReport( + Instant.now().toString(), + EXPECTED_MODEL, + sanitizedEndpoint(configuration.serverUri()), + System.getProperty("os.name"), + System.getProperty("os.arch"), + System.getProperty("java.version"), + configuration.warmUpRuns(), + configuration.rounds(), + configuration.batchSize(), + MAX_REQUEST_TEXTS, + corpus.size(), + corpus.size(), + ChunkQualityBenchmarkSupport.DOCUMENT_LENGTH * (long) corpus.size(), + queryEmbeddingMillis, + queryEmbeddings.requestCount(), + List.copyOf(results) + ); + + writeReport(configuration.outputPath(), report); + logSummary(report); + } + + private RoundMeasurement runProfileRound( + ChunkProfile profile, + List corpus, + EmbeddingClient embeddingClient, + int batchSize, + Map queryVectors + ) { + long chunkingStartedAt = System.nanoTime(); + ChunkedCorpus chunked = ChunkQualityBenchmarkSupport.chunk(corpus, profile); + double chunkingMillis = nanosToMillis(System.nanoTime() - chunkingStartedAt); + + List candidates = chunked.candidates(); + long embeddingStartedAt = System.nanoTime(); + EmbeddedVectors chunkEmbeddings = embed( + embeddingClient, + candidates.stream().map(ChunkCandidate::candidateId).toList(), + candidates.stream().map(ChunkCandidate::text).toList(), + batchSize + ); + double embeddingMillis = nanosToMillis(System.nanoTime() - embeddingStartedAt); + + long searchStartedAt = System.nanoTime(); + QualityMetrics quality = ChunkQualityBenchmarkSupport.evaluate( + corpus, + candidates, + queryVectors, + chunkEmbeddings.vectors() + ); + double searchMillis = nanosToMillis(System.nanoTime() - searchStartedAt); + + return new RoundMeasurement( + chunked, + quality, + chunkingMillis, + embeddingMillis, + searchMillis, + chunkEmbeddings.requestCount() + ); + } + + private void verifyHealth(RestClient restClient) { + restClient.get().uri("/health").retrieve().toBodilessEntity(); + } + + private void warmUp( + EmbeddingClient embeddingClient, + BenchmarkConfiguration configuration, + List corpus + ) { + ChunkedCorpus representative = ChunkQualityBenchmarkSupport.chunk( + corpus.subList(0, 1), + ChunkQualityBenchmarkSupport.profiles().get(0) + ); + List texts = List.of(corpus.get(0).question(), representative.candidates().get(0).text()); + List ids = List.of("warmup-query", "warmup-chunk"); + for (int run = 0; run < configuration.warmUpRuns(); run++) { + embed(embeddingClient, ids, texts, Math.min(configuration.batchSize(), texts.size())); + } + } + + private EmbeddedVectors embed( + EmbeddingClient embeddingClient, + List ids, + List texts, + int batchSize + ) { + if (ids.size() != texts.size() || texts.isEmpty()) { + throw new IllegalArgumentException("Embedding ID와 Text는 같은 개수의 비어 있지 않은 목록이어야 합니다."); + } + List vectors = new ArrayList<>(texts.size()); + int requestCount = 0; + + // 외부 API의 Text 수 상한과 모델 내부 Batch Size를 분리해 큰 Corpus도 순서를 보존한다. + for (int start = 0; start < texts.size(); start += MAX_REQUEST_TEXTS) { + int end = Math.min(start + MAX_REQUEST_TEXTS, texts.size()); + EmbedBatchServerResponse response = embeddingClient.embedBatch(texts.subList(start, end), batchSize); + if (!EXPECTED_MODEL.equals(response.model())) { + throw new IllegalStateException("예상하지 않은 Embedding Model입니다: " + response.model()); + } + for (EmbedBatchItemResponse item : response.embeddings()) { + float[] vector = item.vector(); + ChunkQualityBenchmarkSupport.validateVector(vector); + vectors.add(vector); + } + requestCount++; + } + return new EmbeddedVectors(ChunkQualityBenchmarkSupport.vectorMap(ids, vectors), requestCount); + } + + private boolean isParetoCandidate(ProfileResult candidate, List results) { + // record의 값 기반 equals는 지표가 같은 다른 Profile까지 제외하므로 참조로 자기 자신만 건너뛴다. + return results.stream().noneMatch(other -> other != candidate && dominates(other, candidate)); + } + + private boolean dominates(ProfileResult left, ProfileResult right) { + boolean qualityNotWorse = greaterOrEqual( + left.quality().answerCoverageRatio(), right.quality().answerCoverageRatio() + ) && greaterOrEqual(left.quality().hitAt1(), right.quality().hitAt1()) + && greaterOrEqual(left.quality().hitAt3(), right.quality().hitAt3()) + && greaterOrEqual(left.quality().mrrAt10(), right.quality().mrrAt10()); + boolean costNotWorse = left.chunkCodePoints() <= right.chunkCodePoints(); + boolean strictlyBetter = greater( + left.quality().answerCoverageRatio(), right.quality().answerCoverageRatio() + ) || greater(left.quality().hitAt1(), right.quality().hitAt1()) + || greater(left.quality().hitAt3(), right.quality().hitAt3()) + || greater(left.quality().mrrAt10(), right.quality().mrrAt10()) + || left.chunkCodePoints() < right.chunkCodePoints(); + return qualityNotWorse && costNotWorse && strictlyBetter; + } + + private boolean greaterOrEqual(double left, double right) { + return left + COMPARISON_EPSILON >= right; + } + + private boolean greater(double left, double right) { + return left > right + COMPARISON_EPSILON; + } + + private void writeReport(Path outputPath, BenchmarkReport report) throws IOException { + Path parent = outputPath.toAbsolutePath().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + new ObjectMapper().findAndRegisterModules() + .writerWithDefaultPrettyPrinter() + .writeValue(outputPath.toFile(), report); + } + + private void logSummary(BenchmarkReport report) { + for (ProfileResult profile : report.profiles()) { + log.info( + "Chunk 품질 결과 profile={}, coverage={}, hit@3={}, mrr@10={}, chunks={}, duplicateRatio={}, " + + "embeddingP95Ms={}, pareto={}", + profile.profileId(), + format(profile.quality().answerCoverageRatio()), + format(profile.quality().hitAt3()), + format(profile.quality().mrrAt10()), + profile.chunkCount(), + format(profile.duplicateRatio()), + format(profile.embeddingTiming().p95Millis()), + profile.paretoCandidate() + ); + } + } + + private String sanitizedEndpoint(URI serverUri) { + int port = serverUri.getPort(); + return serverUri.getScheme() + "://" + serverUri.getHost() + (port < 0 ? "" : ":" + port); + } + + private String format(double value) { + return String.format(Locale.ROOT, "%.4f", value); + } + + private double nanosToMillis(long nanos) { + return nanos / 1_000_000.0; + } + + /** + * 외부 Benchmark 입력을 System Property와 환경 변수에서 검증해 읽는다. + */ + record BenchmarkConfiguration( + URI serverUri, + int warmUpRuns, + int rounds, + int batchSize, + Path outputPath + ) { + + BenchmarkConfiguration { + if (serverUri == null || serverUri.getScheme() == null || serverUri.getHost() == null) { + throw new IllegalArgumentException("Embedding Server URL은 절대 HTTP URL이어야 합니다."); + } + if (!"http".equals(serverUri.getScheme()) && !"https".equals(serverUri.getScheme())) { + throw new IllegalArgumentException("Embedding Server URL은 HTTP 또는 HTTPS여야 합니다."); + } + if (serverUri.getUserInfo() != null) { + throw new IllegalArgumentException("Embedding Server URL에 인증 정보를 포함할 수 없습니다."); + } + if (warmUpRuns < 1 || rounds < 1) { + throw new IllegalArgumentException("Warm-up과 측정 Round는 각각 1 이상이어야 합니다."); + } + if (batchSize < 1 || batchSize > MAX_REQUEST_TEXTS) { + throw new IllegalArgumentException("Batch Size는 1 이상 64 이하여야 합니다."); + } + if (outputPath == null) { + throw new IllegalArgumentException("Benchmark 출력 경로가 필요합니다."); + } + } + + static BenchmarkConfiguration fromSystemProperties() { + String configuredUrl = System.getProperty("chunk.quality.performance.server-url"); + if (configuredUrl == null || configuredUrl.isBlank()) { + configuredUrl = System.getenv().getOrDefault("EMBEDDING_SERVER_URL", "http://localhost:8000"); + } + return new BenchmarkConfiguration( + URI.create(configuredUrl), + integerProperty("chunk.quality.performance.warm-up-runs", 1), + integerProperty("chunk.quality.performance.rounds", 2), + integerProperty("chunk.quality.performance.batch-size", 32), + Path.of(System.getProperty( + "chunk.quality.performance.output", + "build/reports/chunk-quality/chunk-quality-latest.json" + )) + ); + } + + private static int integerProperty(String name, int defaultValue) { + try { + return Integer.parseInt(System.getProperty(name, Integer.toString(defaultValue))); + } catch (NumberFormatException exception) { + throw new IllegalArgumentException("정수 System Property가 필요합니다: " + name, exception); + } + } + + } + + /** + * 한 Embedding 단계에서 ID별 Vector와 실제 HTTP 요청 수를 보관한다. + */ + private record EmbeddedVectors(Map vectors, int requestCount) { + } + + /** + * 한 Profile의 한 Round에서 수집한 결정적 품질·비용과 구간별 지연이다. + */ + private record RoundMeasurement( + ChunkedCorpus chunked, + QualityMetrics quality, + double chunkingMillis, + double embeddingMillis, + double searchMillis, + int embeddingRequestCount + ) { + } + + /** + * 한 Profile의 반복 측정이 같은 품질·비용 계약을 지키는지 검증하고 통계를 집계한다. + */ + private static final class ProfileAccumulator { + + private final ChunkProfile profile; + private final List rounds = new ArrayList<>(); + + private ProfileAccumulator(ChunkProfile profile) { + this.profile = profile; + } + + private void add(RoundMeasurement measurement) { + if (!rounds.isEmpty()) { + RoundMeasurement baseline = rounds.get(0); + if (!sameDeterministicResult(baseline, measurement)) { + throw new IllegalStateException("Profile 반복 결과가 결정적이지 않습니다: " + profile.profileId()); + } + } + rounds.add(measurement); + } + + private boolean sameDeterministicResult(RoundMeasurement left, RoundMeasurement right) { + return left.chunked().originalCodePoints() == right.chunked().originalCodePoints() + && left.chunked().chunkCodePoints() == right.chunked().chunkCodePoints() + && left.chunked().duplicateCodePoints() == right.chunked().duplicateCodePoints() + && left.chunked().candidates().size() == right.chunked().candidates().size() + && sameQuality(left.quality(), right.quality()); + } + + private boolean sameQuality(QualityMetrics left, QualityMetrics right) { + return Double.compare(left.answerCoverageRatio(), right.answerCoverageRatio()) == 0 + && Double.compare(left.hitAt1(), right.hitAt1()) == 0 + && Double.compare(left.hitAt3(), right.hitAt3()) == 0 + && Double.compare(left.mrrAt10(), right.mrrAt10()) == 0 + && left.queries().stream().map(result -> result.firstRelevantRank()).toList() + .equals(right.queries().stream().map(result -> result.firstRelevantRank()).toList()); + } + + private ProfileResult toResult(boolean paretoCandidate) { + if (rounds.isEmpty()) { + throw new IllegalStateException("Profile 측정 결과가 없습니다: " + profile.profileId()); + } + RoundMeasurement baseline = rounds.get(0); + return new ProfileResult( + profile.profileId(), + profile.chunkSize(), + profile.overlap(), + baseline.chunked().candidates().size(), + baseline.chunked().originalCodePoints(), + baseline.chunked().chunkCodePoints(), + baseline.chunked().duplicateCodePoints(), + baseline.chunked().duplicateRatio(), + baseline.quality(), + summarizeTimings(rounds.stream().map(RoundMeasurement::chunkingMillis).toList()), + summarizeTimings(rounds.stream().map(RoundMeasurement::embeddingMillis).toList()), + summarizeTimings(rounds.stream().map(RoundMeasurement::searchMillis).toList()), + rounds.stream().mapToInt(RoundMeasurement::embeddingRequestCount).sum(), + paretoCandidate, + rounds.stream().map(RoundResult::from).toList() + ); + } + } + + /** + * 외부 결과에 노출하는 한 Round의 지연과 요청 수다. + */ + record RoundResult( + double chunkingMillis, + double embeddingMillis, + double searchMillis, + int embeddingRequestCount + ) { + + private static RoundResult from(RoundMeasurement measurement) { + return new RoundResult( + measurement.chunkingMillis(), + measurement.embeddingMillis(), + measurement.searchMillis(), + measurement.embeddingRequestCount() + ); + } + } + + /** + * 한 Chunk Profile의 품질, 결정적 비용, 반복 지연과 Pareto 여부를 결합한다. + */ + record ProfileResult( + String profileId, + int chunkSize, + int overlap, + int chunkCount, + long originalCodePoints, + long chunkCodePoints, + long duplicateCodePoints, + double duplicateRatio, + QualityMetrics quality, + TimingSummary chunkingTiming, + TimingSummary embeddingTiming, + TimingSummary searchTiming, + int embeddingRequestCount, + boolean paretoCandidate, + List rounds + ) { + + ProfileResult { + rounds = List.copyOf(rounds); + } + + private ProfileResult withParetoCandidate(boolean value) { + return new ProfileResult( + profileId, + chunkSize, + overlap, + chunkCount, + originalCodePoints, + chunkCodePoints, + duplicateCodePoints, + duplicateRatio, + quality, + chunkingTiming, + embeddingTiming, + searchTiming, + embeddingRequestCount, + value, + rounds + ); + } + } + + /** + * 재현 환경과 모든 Profile 결과를 담는 Chunk 품질 Benchmark JSON 계약이다. + */ + record BenchmarkReport( + String generatedAt, + String model, + String embeddingEndpoint, + String osName, + String osArchitecture, + String javaVersion, + int warmUpRuns, + int measuredRounds, + int modelBatchSize, + int maxTextsPerRequest, + int documentCount, + int queryCount, + long originalCodePoints, + double queryEmbeddingMillis, + int queryEmbeddingRequestCount, + List profiles + ) { + + BenchmarkReport { + profiles = List.copyOf(profiles); + } + } +}