From 6e1d57414e0e08636ebbc2f3a77823e6099230d3 Mon Sep 17 00:00:00 2001 From: kric_jspark Date: Wed, 11 Nov 2026 10:37:13 +0900 Subject: [PATCH 1/2] =?UTF-8?q?261111=5F01=5FENH:=20Kiwi=20=ED=98=95?= =?UTF-8?q?=ED=83=9C=EC=86=8C=20=EB=B6=84=EC=84=9D=20=EC=A0=95=ED=99=95?= =?UTF-8?q?=EB=8F=84=20=EA=B0=9C=EC=84=A0=20=EB=B0=8F=20=ED=92=88=EC=82=AC?= =?UTF-8?q?=20=ED=95=A9=EC=82=B0=C2=B7=EB=B3=B5=ED=95=A9=EB=AA=85=EC=82=AC?= =?UTF-8?q?=C2=B7=EC=B9=98=ED=99=98=C2=B7=EB=B6=88=EC=9A=A9=EC=96=B4=20?= =?UTF-8?q?=EC=98=B5=EC=85=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .agent/references/repository-rules.md | 13 +- README.md | 12 +- docs/statistics-flow.md | 23 +- gradle.properties | 3 + .../server/common/consts/KartConstant.java | 2 + .../server/common/enums/WordMergeMode.java | 32 ++ .../dto/statistics/NlpDictionaryWordDto.java | 5 + .../server/dto/statistics/NlpSynonymDto.java | 14 + .../dto/statistics/StatisticsOptionsDto.java | 10 + .../server/service/StatisticsService.java | 8 +- .../service/statistics/KiwiNlpService.java | 366 +++++++++++++++++- .../statistics/StatisticsTextService.java | 111 +++++- .../resources/mybatis/mapper/NlpMapper.xml | 9 +- 13 files changed, 572 insertions(+), 36 deletions(-) create mode 100644 src/main/java/com/kri/kart/server/common/enums/WordMergeMode.java create mode 100644 src/main/java/com/kri/kart/server/dto/statistics/NlpSynonymDto.java diff --git a/.agent/references/repository-rules.md b/.agent/references/repository-rules.md index 014305f..6553597 100644 --- a/.agent/references/repository-rules.md +++ b/.agent/references/repository-rules.md @@ -112,7 +112,7 @@ - Follow existing question organization. Return N and percent together, including Total. Preserve repeated rank/matrix occurrences, original numeric missing/mean/statistic rules, scale/reverse/NPS, grouped values and TABULATION denominators. Frontend chooses PRTMODE presentation without requery. - Keep original empty-base semantics: ordinary cell/Total percentages are 0; undefined group/statistic values are null with '-' display. Use 15-significant-digit C# double-format behavior and midpoint-away rounding, validated by legacy fixtures. - StatisticsMapper uses survey sessions, parameterized complete/TEST/date conditions, original distinct derived/profile joins and selected storage-group projection. Filter data is private and separate from projected aggregation data. AST filters never inject user SQL. -- User approved the latest Kiwi model after finding the original DLL was 0.15.2. Use matching official KiwiJava/model 0.24.0 CONG instead of pinning SBG. Model-result changes are allowed; preserve Report NLP dictionary/tag scopes, word scores 10/50/100, tag/length/SL selection and stable top-100 frequency logic. Add no new stopword rules. +- User approved the latest Kiwi model after finding the original DLL was 0.15.2. Use matching official KiwiJava/model 0.24.0 CONG instead of pinning SBG. Model-result changes are allowed; preserve Report NLP dictionary/tag scopes, word scores 10/50/100, tag/length/SL selection and stable top-100 frequency logic. Stopwords exist only as the REPORT_NLP_WORD STOP rows described under Kiwi NLP accuracy; add no other stopword rules. - Configure kri.kart.nlp.model-directory as ./nlp_model/v0.24.0, install model separately, ignore runtime models in Git, and choose the platform-specific official jar at build time. Do not claim Ubuntu execution from Windows tests. Missing native/model infrastructure is explicit 503, not empty successful NLP output. - Per user decision, the legacy external-Referer condition is removed: every general statistics batch writes Get Report Data once through LogService. Quota results write QUOTA VIEW only, never Get Report Data. Do not add a separate logging endpoint. - Validate through original C# synthetic calculation fixtures, authenticated API/Swagger tests, source-bound MyBatis tests and opt-in actual native analysis with -PkiwiModelDirectory. Actual SQL Server equivalence and deployment performance remain separate checks. @@ -187,3 +187,14 @@ - Apply team/assignment before candidate TOP 50 and ordering, never after Java merge/TOP 6. Must-view does not override authorization. Linked list retains its business query team and adds separately named authenticated-team/role/assignment constraints. - Derived management list: role 1001 sees all project definitions; others see matching GKEY only. Keep normal questions/report metadata unfiltered by this management rule. - Scope is these read conditions only. Do not add generic per-project/mutation/owner authorization, change original aggregations/Excel, or weaken External scope. Validate actual bound mapper SQL and H2 results on both sources, empty assignments, precise id routing, trusted JWT/API Key context versus forged query fields; no shared DB execution. + +## Kiwi NLP accuracy (user decisions 2026-11-10) + +- The legacy C# report services are retired; REPORT_NLP_WORD/TAG are no longer read by them. +- KiwiNlpService builds with BuildOption.default_ and analyzes with Match.allWithNormalizing | oovChrModel plus basicTypoSetWithContinual typo correction (threshold 2.5). Do not revert to AnalyzeOption(0). +- User dictionary: register each (text, tag) once with its highest scope score, because Kiwi keeps the last duplicate score and the query has no order. Trim/uppercase tags. Log only registered/skipped/rejected counts, never words. +- Statistics options: wordMergeMode 1=ALL, 2=NOUN (default; NNG/NNP/NNB/NR/NP/SL/SH), 3=NONE sums the same text across tags, keeping the most frequent tag (first on ties) and first-appearance order. compoundNounMaxTokens 1-5 (default 3; 1 disables) joins noun prefixes/suffixes and merges NNG/NNP/SL runs attached within one word when they contain a Korean noun; NNP if any part is NNP; longer runs stay separate. +- Synonyms (user decision 2026-11-11, replacing the separate REPORT_NLP_SYNONYM table): a REPORT_NLP_WORD row with SOURCE_TEXT (NVARCHAR(128), nullable; WORD_TEXT is NVARCHAR(128)) replaces SOURCE_TEXT with WORD_TEXT/WORD_TAG. WORD_TEXT always means the result word. Scope priority question > project > ALL uses the 100/50/10 scores. The column must exist before deployment. Sources are pre-analyzed (score 200, above every user word) after registering the target as a word with score 0 unless a plain row for the same target already registered it with its scope score, because Kiwi rejects a target morpheme missing from its dictionary such as LG/NNP; a failing row is skipped, never a 503; a spaced source also covers its spaceless form and replaces whole token runs only when no noun of the same word continues the run. +- Stopwords (불용어): REPORT_NLP_WORD rows with WORD_TAG STOP (case-insensitive) are never registered in Kiwi. Any scope excludes the final word (after synonyms and compound merging) for every tag, comparing trimmed, space-collapsed text case-insensitively; SOURCE_TEXT is ignored on STOP rows. +- User dictionary words and synonym targets take precedence over everything: they are never merged with neighbors. +- The original analyze(project, qid, responses) and tokenize(texts, dictionary) overloads keep the previous behavior (synonym rows ignored; no stopwords or merging). Only the statistics API uses the new options. diff --git a/README.md b/README.md index 7a61440..2cadd0c 100644 --- a/README.md +++ b/README.md @@ -148,13 +148,19 @@ ProjectMapper는 list/summary/mustViewPkeys를 유지하고 ProgressMapper/Progr 필터는 `filter`의 AST로 입력한다. 그룹 AND/OR/NOT는 `children`, 조건 EQ/NE/GT/GE/LT/LE/IN/NOT_IN/BETWEEN/IS_ANSWERED/IS_UNANSWERED는 `dimensionId`, `reference=QID|VARIABLE`, `values`를 사용한다. JSON 숫자는 수치 비교, 문자열은 기존 SQL collation처럼 대소문자 무시 비교다. 부정 조건은 일치 응답이 하나라도 있으면 제외하고 해당 키가 없는 응답은 포함하는 기존 NOT EXISTS 의미를 유지한다. IS_ANSWERED는 기존 IS NOT NULL 의미여서 빈 문자열도 포함한다. 필터 문자열을 SQL에 삽입하지 않는다. 필터 깊이는 32, 그룹당 자식은 128까지 허용한다. -주관식 문자열은 `indexes=["QID_입력칸"]`으로 같은 API를 호출한다. `type=TEXT_ANALYSIS`와 `words[{text,tag,count}]`를 반환하며 원문·IDKEY와 기존의 가짜 제목 데이터 행을 노출하지 않는다. 기존 REPORT_NLP_TAG/WORD의 전체·프로젝트·문항 범위 및 사전 점수 10/50/100, SL 포함·품사/길이 조건, 같은 단어의 품사별 별도 빈도와 빈도 내림차순/동률 최초 등장 순서/상위 100개를 유지한다. 문장별 분리는 토큰을 빠뜨리지 않으므로 결과를 전체 토큰 빈도로 집계한다. 별도의 불용어 규칙을 추가하지 않는다. +주관식 문자열은 `indexes=["QID_입력칸"]`으로 같은 API를 호출한다. `type=TEXT_ANALYSIS`와 `words[{text,tag,count}]`를 반환하며 원문·IDKEY와 기존의 가짜 제목 데이터 행을 노출하지 않는다. 기존 REPORT_NLP_TAG/WORD의 전체·프로젝트·문항 범위 및 사전 점수 10/50/100, SL 포함·품사/길이 조건, 빈도 내림차순/동률 최초 등장 순서/상위 100개를 유지한다. 문장별 분리는 토큰을 빠뜨리지 않으므로 결과를 전체 토큰 빈도로 집계한다. -사용자 결정에 따라 최신 공식 KiwiJava·모델 **0.24.0**을 사용한다. 기존 DLL은 실제 실행에서 0.15.2로 확인되었으며 기존 모델은 Java 0.20.4에서도 로딩되지 않았다. 최신 모델 적용에 따른 형태소 결과 변화는 허용하되 Report DB 사전·품사 선택·빈도 집계 규칙은 유지한다. 0.24.0의 CONG 모델, default/typo dictionary 로딩, 이형태 통합 비활성, 오타 교정 비활성, match=0을 사용한다. 모델은 `kri.kart.nlp.model-directory`(기본 `./nlp_model/v0.24.0`)에 별도로 설치한다. `scripts/setup-kiwi-model.ps1`은 공식 모델 다운로드·SHA-256 검증 후 비어 있는 폴더에만 설치한다. 모델은 Git에서 제외한다. Kiwi 라이브러리/모델이 없거나 분석에 실패하면 해당 문항에 status=503, code=KIWI_NLP_UNAVAILABLE인 error 이벤트를 반환하고 다음 문항을 계속한다. +텍스트 분석 요청 옵션: `wordMergeMode`는 같은 글자의 품사별 빈도 합산 방식으로 1=모두 합치기, 2=명사 계열(NNG·NNP·NNB·NR·NP·SL·SH)만 합치기(기본값), 3=합치지 않기이다. 합친 단어는 가장 많이 나온 품사(동률이면 먼저 나온 품사)를 사용한다. `compoundNounMaxTokens`(1~5, 기본값 3, 1이면 끔)는 명사 접두사·접미사를 붙이고, 한 어절 안에서 공백 없이 붙은 NNG·NNP·SL 토큰(한글 명사 1개 이상 포함)을 복합명사 하나로 합친다. 하나라도 NNP면 NNP이고, 상한을 넘는 묶음은 합치지 않는다. 사용자 사전 단어와 치환 결과는 합치지 않는다. + +불용어: REPORT_NLP_WORD에서 WORD_TAG가 `STOP`인 행은 불용어이며 Kiwi 사전에 등록하지 않는다. 치환·복합명사 합치기 후의 최종 단어를 품사와 관계없이 제외하며, 앞뒤·연속 공백과 대소문자를 구분하지 않는다. 전체·프로젝트·문항 중 어느 범위에 있어도 적용한다. 그 밖의 불용어 규칙은 두지 않는다. + +치환: REPORT_NLP_WORD에서 `SOURCE_TEXT`가 있는 행은 `SOURCE_TEXT`를 `WORD_TEXT`/`WORD_TAG`로 바꿔 집계한다(예: SOURCE_TEXT 엘지 → WORD_TEXT LG). `WORD_TEXT`는 모든 행에서 결과에 나오는 단어이며, 치환 행의 `WORD_TEXT`는 점수 0으로만 등록한다(같은 단어의 일반 행이 있으면 그 범위 점수 유지). 우선순위는 문항 > 프로젝트 > 전체이고 사용자 사전보다 우선한다. 띄어 쓴 원표기는 붙여 쓴 형태도 함께 처리하며, 같은 어절의 명사가 이어지면 치환하지 않는다. 짧은 원표기는 긴 단어 안에서도 바뀌므로(엘지전자 → LG + 전자) 필요하면 긴 표기 행을 함께 둔다. `SOURCE_TEXT` 컬럼(NVARCHAR(128), NULL 허용)이 없는 DB에서는 사전 조회가 실패하므로 컬럼 추가 후 배포한다. + +사용자 결정에 따라 최신 공식 KiwiJava·모델 **0.24.0**을 사용한다. 기존 DLL은 실제 실행에서 0.15.2로 확인되었으며 기존 모델은 Java 0.20.4에서도 로딩되지 않았다. 최신 모델 적용에 따른 형태소 결과 변화는 허용하되 Report DB 사전·품사 선택·빈도 집계 규칙은 유지한다. 0.24.0의 CONG 모델과 기본 빌드 옵션(default·typo·multi-word dictionary, 이형태 통합), `Match.allWithNormalizing | oovChrModel`, 오타 교정(basicTypoSetWithContinual, 기준값 2.5)을 사용한다. 같은 사용자 사전 단어는 범위 중 가장 높은 점수로 한 번만 등록한다. 모델은 `kri.kart.nlp.model-directory`(기본 `./nlp_model/v0.24.0`)에 별도로 설치한다. `scripts/setup-kiwi-model.ps1`은 공식 모델 다운로드·SHA-256 검증 후 비어 있는 폴더에만 설치한다. 모델은 Git에서 제외한다. Kiwi 라이브러리/모델이 없거나 분석에 실패하면 해당 문항에 status=503, code=KIWI_NLP_UNAVAILABLE인 error 이벤트를 반환하고 다음 문항을 계속한다. Gradle은 Windows에서 win-x64, Linux에서 lnx-x86-64 또는 lnx-aarch64 JAR을 선택한다. Windows에서 Linux 배포물을 만들 때 `./gradlew.bat bootJar -PkiwiPlatform=lnx-x86-64`로 명시한다. 배포 JAR과 JVM 플랫폼이 일치해야 하고 모델도 운영에 별도로 배치한다. Kiwi 및 모델의 라이선스는 공식 저장소의 LGPL-2.1 조건을 확인한다: https://github.com/bab2min/Kiwi/tree/v0.24.0/bindings/java . -검증: 원본 C# Report.cs를 실행한 합성 비교 기준 25개, 필터/좌표/조회 분리/인증·Swagger/API와 실제 Windows Kiwi 0.24.0 분석을 테스트한다. 네이티브 검증은 `./gradlew.bat test '-PkiwiModelDirectory=nlp_model/v0.24.0'`로 실행한다. 이 속성이 없으면 해당 네이티브 테스트 1개만 생략한다. `build/phase5-openapi.json`은 API 테스트에서 생성된다. 실제 SQL Server 결과, Ubuntu 네이티브 실행·운영 성능은 별도 검수 대상이며 합성 테스트 통과로 운영 동등성을 주장하지 않는다. +검증: 원본 C# Report.cs를 실행한 합성 비교 기준 25개, 필터/좌표/조회 분리/인증·Swagger/API와 실제 Windows Kiwi 0.24.0 분석을 테스트한다. 네이티브 검증은 `./gradlew.bat test '-PkiwiModelDirectory=nlp_model/v0.24.0'`로 실행한다. 이 속성이 없으면 실제 모델을 사용하는 네이티브 테스트만 생략한다. `build/phase5-openapi.json`은 API 테스트에서 생성된다. 실제 SQL Server 결과, Ubuntu 네이티브 실행·운영 성능은 별도 검수 대상이며 합성 테스트 통과로 운영 동등성을 주장하지 않는다. ### 배열 집계와 NDJSON 스트리밍 diff --git a/docs/statistics-flow.md b/docs/statistics-flow.md index 12d2caf..5a24a97 100644 --- a/docs/statistics-flow.md +++ b/docs/statistics-flow.md @@ -173,23 +173,32 @@ flowchart TD flowchart TD INPUT["StatisticsService.textAnalysis()
StatisticsTextService.analyze()"] TEXT["StatisticsTextService.answerTexts()
해당 QID와 QID_ 하위 키의 문자열 수집
HTML 문자 복원"] - DICT[("Report DB
REPORT_NLP_WORD
REPORT_NLP_TAG")] + DICT[("Report DB
REPORT_NLP_WORD (STOP 행 = 불용어, SOURCE_TEXT 행 = 치환)
REPORT_NLP_TAG")] REPO["NlpRepository.words() / tags()
NlpMapper.words() / tags()
kartSqlSessionTemplate 사용"] MODEL[("로컬 Kiwi 모델 파일
kri.kart.nlp.model-directory")] - TOKEN["KiwiNlpService.tokenize()
응답 문자열 + 사용자 사전 → 형태소 목록"] - FILTER["StatisticsTextService.isCounted()
선택 품사 중 명사 또는 2글자 이상
SL은 선택 품사·길이와 관계없이 포함"] - COUNT["StatisticsTextService.analyze()
단어+품사별 빈도 합산
빈도 내림차순, 동률은 첫 등장 순서
상위 100개 선택"] + TOKEN["KiwiNlpService.tokenize()
응답 문자열 + 사용자 사전 + 치환 → 형태소 목록
오타 교정·구어체 정규화
compoundNounMaxTokens에 따라 복합명사 합치기"] + FILTER["StatisticsTextService.analyze() / isCounted()
STOP 불용어 제외
선택 품사 중 명사 또는 2글자 이상
SL은 선택 품사·길이와 관계없이 포함"] + COUNT["StatisticsTextService.analyze() / merge()
단어+품사별 빈도 합산
wordMergeMode에 따라 같은 글자 합산
빈도 내림차순, 동률은 첫 등장 순서
상위 100개 선택"] OUT["StatisticsResponseDto, type=TEXT_ANALYSIS
words: 단어·품사·빈도
문항별 result 이벤트로 반환"] INPUT --> TEXT --> TOKEN DICT --> REPO - REPO -->|사전 단어| TOKEN + REPO -->|사전 단어·치환| TOKEN MODEL --> TOKEN TOKEN --> FILTER --> COUNT --> OUT - REPO -->|선택 품사| FILTER + REPO -->|선택 품사·불용어| FILTER ``` -사전은 전체 공통(`PKEY=ALL`), 프로젝트 공통(`QID=ALL`), 해당 문항 범위를 합쳐 읽는다. 사용자 단어의 점수는 각각 10·50·100이다. 이 점수는 형태소 분석 사전에 사용하는 값이며 최종 빈도에 곱하는 통계 가중치가 아니다. 모델이나 native 실행 환경이 없으면 오류를 반환한다. +사전은 전체 공통(`PKEY=ALL`), 프로젝트 공통(`QID=ALL`), 해당 문항 범위를 합쳐 읽는다. 사용자 단어의 점수는 각각 10·50·100이며, 같은 단어는 가장 높은 점수로 한 번만 등록한다. 이 점수는 형태소 분석 사전에 사용하는 값이며 최종 빈도에 곱하는 통계 가중치가 아니다. 모델이나 native 실행 환경이 없으면 오류를 반환한다. + +텍스트 분석 처리 순서는 다음과 같다. + +1. **치환**: `REPORT_NLP_WORD`에서 `SOURCE_TEXT`가 있는 행은 `SOURCE_TEXT`를 `WORD_TEXT`/`WORD_TAG`로 바꾼다. 우선순위는 문항 > 프로젝트 > 전체이고, 사용자 사전보다 우선한다. +2. **복합명사 합치기**: `compoundNounMaxTokens`(기본 3)에 따라 한 어절 안에서 붙은 명사를 합친다. 사용자 사전 단어와 치환 결과는 합치지 않는다. +3. **불용어 제외**: `REPORT_NLP_WORD`에서 WORD_TAG가 `STOP`인 단어를 품사와 관계없이 제외한다. 대소문자와 공백 차이는 무시한다. +4. **품사 합산**: `wordMergeMode`(기본 2=명사 계열)에 따라 같은 글자의 품사별 빈도를 합친다. + +기존 `analyze(project, qid, responses)`와 `tokenize(texts, dictionary)` 오버로드는 치환·불용어·합치기를 적용하지 않는 이전 동작을 유지하며, 통계 API만 새 옵션을 사용한다. ### 2-4. 쿼터는 어디서 갈라지는가 diff --git a/gradle.properties b/gradle.properties index 0e829de..270ebc3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,3 +3,6 @@ org.gradle.jvmargs=-Xmx2g org.gradle.parallel=true org.gradle.caching=true + +# Enabled parallel sync for Gradle 9.4+ +org.gradle.tooling.parallel=true diff --git a/src/main/java/com/kri/kart/server/common/consts/KartConstant.java b/src/main/java/com/kri/kart/server/common/consts/KartConstant.java index a9435a5..e613b21 100644 --- a/src/main/java/com/kri/kart/server/common/consts/KartConstant.java +++ b/src/main/java/com/kri/kart/server/common/consts/KartConstant.java @@ -100,6 +100,8 @@ public final class KartConstant { // StatisticsTextService public static final int TOP_WORD_LIMIT = 100; public static final String FOREIGN_WORD_TAG = "SL"; + // 불용어: REPORT_NLP_WORD rows with this WORD_TAG are excluded from word counts. + public static final String STOPWORD_TAG = "STOP"; // StatisticsController public static final String CROSSTAB_PAGE = "/Report/Crosstab"; diff --git a/src/main/java/com/kri/kart/server/common/enums/WordMergeMode.java b/src/main/java/com/kri/kart/server/common/enums/WordMergeMode.java new file mode 100644 index 0000000..467547d --- /dev/null +++ b/src/main/java/com/kri/kart/server/common/enums/WordMergeMode.java @@ -0,0 +1,32 @@ +package com.kri.kart.server.common.enums; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.kri.common.feature.enums.CodeEnum; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter(onMethod_ = @JsonValue) +@RequiredArgsConstructor +@Schema(description = "같은 단어의 품사별 빈도 합산: 1=모두 합치기, 2=명사 계열만 합치기, 3=합치지 않기") +public enum WordMergeMode implements CodeEnum { + ALL(1), + NOUN(2), + NONE(3); + + private final Integer code; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static WordMergeMode fromCode(Integer code) { + if (code == null) { + return null; + } + for (WordMergeMode s : values()) { + if (s.code.equals(code)) { + return s; + } + } + throw new IllegalArgumentException("Unknown WordMergeMode code: " + code); + } +} diff --git a/src/main/java/com/kri/kart/server/dto/statistics/NlpDictionaryWordDto.java b/src/main/java/com/kri/kart/server/dto/statistics/NlpDictionaryWordDto.java index 60a43f7..a4af3e9 100644 --- a/src/main/java/com/kri/kart/server/dto/statistics/NlpDictionaryWordDto.java +++ b/src/main/java/com/kri/kart/server/dto/statistics/NlpDictionaryWordDto.java @@ -3,10 +3,15 @@ package com.kri.kart.server.dto.statistics; import lombok.Getter; import lombok.Setter; +/** + * REPORT_NLP_WORD row. WORD_TAG STOP marks a 불용어 (stopword); a row with SOURCE_TEXT is a synonym + * that replaces SOURCE_TEXT with WORD_TEXT/WORD_TAG; any other row is a Kiwi user word. + */ @Getter @Setter public class NlpDictionaryWordDto { private String wordText; private String wordTag; private Integer wordScore; + private String sourceText; } diff --git a/src/main/java/com/kri/kart/server/dto/statistics/NlpSynonymDto.java b/src/main/java/com/kri/kart/server/dto/statistics/NlpSynonymDto.java new file mode 100644 index 0000000..04b1d69 --- /dev/null +++ b/src/main/java/com/kri/kart/server/dto/statistics/NlpSynonymDto.java @@ -0,0 +1,14 @@ +package com.kri.kart.server.dto.statistics; + +import lombok.Getter; +import lombok.Setter; + +/** Text-analysis replacement such as 엘지전자 → LG전자, with 1=ALL, 2=project, 3=question scope. */ +@Getter +@Setter +public class NlpSynonymDto { + private String sourceText; + private String targetText; + private String targetTag; + private Integer scopeRank; +} diff --git a/src/main/java/com/kri/kart/server/dto/statistics/StatisticsOptionsDto.java b/src/main/java/com/kri/kart/server/dto/statistics/StatisticsOptionsDto.java index d4b66db..0d7bc4f 100644 --- a/src/main/java/com/kri/kart/server/dto/statistics/StatisticsOptionsDto.java +++ b/src/main/java/com/kri/kart/server/dto/statistics/StatisticsOptionsDto.java @@ -1,6 +1,7 @@ package com.kri.kart.server.dto.statistics; import com.fasterxml.jackson.annotation.JsonFormat; +import com.kri.kart.server.common.enums.WordMergeMode; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import jakarta.validation.constraints.Max; @@ -55,4 +56,13 @@ public class StatisticsOptionsDto { private boolean showNumericSum; @Valid @NotNull private List valueGroups = new ArrayList<>(); + + @NotNull + @Schema(description = "텍스트 분석에서 같은 단어의 품사별 빈도 합산 방식. 생략하면 2(명사 계열만 합치기)") + private WordMergeMode wordMergeMode = WordMergeMode.NOUN; + + @Min(1) + @Max(5) + @Schema(description = "텍스트 분석에서 공백 없이 붙은 명사를 합칠 최대 토큰 수. 1이면 합치지 않음. 생략하면 3") + private int compoundNounMaxTokens = 3; } diff --git a/src/main/java/com/kri/kart/server/service/StatisticsService.java b/src/main/java/com/kri/kart/server/service/StatisticsService.java index 52b6988..7480258 100644 --- a/src/main/java/com/kri/kart/server/service/StatisticsService.java +++ b/src/main/java/com/kri/kart/server/service/StatisticsService.java @@ -249,7 +249,13 @@ public class StatisticsService { .columns(List.of()) .headerRows(List.of()) .rows(List.of()) - .words(statisticsTextService.analyze(project, index.getDimensionId(), responses)) + .words( + statisticsTextService.analyze( + project, + index.getDimensionId(), + responses, + request.getWordMergeMode(), + request.getCompoundNounMaxTokens())) .appliedOptions(request) .build(); } diff --git a/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java b/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java index c387ad7..4de07eb 100644 --- a/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java +++ b/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java @@ -1,22 +1,39 @@ package com.kri.kart.server.service.statistics; +import com.kri.common.feature.util.StringUtil; import com.kri.common.feature.web.BusinessException; import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto; import com.kri.kart.server.dto.statistics.NlpTokenDto; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; import kr.pe.bab2min.Kiwi; import kr.pe.bab2min.KiwiBuilder; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +@Slf4j @Service public class KiwiNlpService { + // Same as kiwipiepy's default typo cost threshold. + private static final float TYPO_COST_THRESHOLD = 2.5f; + private static final Set COMPOUND_NOUN_TAGS = Set.of("NNG", "NNP", "SL"); + private static final Set NOUN_AFFIX_TAGS = Set.of("SL", "SH", "XPN", "XSN"); + // Above every user dictionary score (10/50/100) so a synonym always wins. + private static final float SYNONYM_SCORE = 200f; + private static final Pattern WHITESPACE = Pattern.compile("\\s+"); + private final Path modelDirectory; public KiwiNlpService( @@ -24,34 +41,76 @@ public class KiwiNlpService { this.modelDirectory = Path.of(modelDirectory); } + /** + * Keeps the existing behavior: synonym rows (with SOURCE_TEXT) are ignored and there is no + * compound-noun or noun-affix joining. + */ public List tokenize(List texts, List dictionary) { + List words = + dictionary.stream().filter(word -> !isSynonymRow(word)).toList(); + return tokenize(texts, words, List.of(), 1); + } + + /** + * Tokenizes texts with REPORT_NLP_WORD rows: a row with SOURCE_TEXT is a synonym that replaces + * SOURCE_TEXT with WORD_TEXT/WORD_TAG before anything else, any other row is a user word. When + * {@code compoundNounMaxTokens} is 2 or more, noun prefixes/suffixes are joined and adjacent + * nouns are merged into one compound noun (see {@link #joinCompoundNouns}). + */ + public List tokenize( + List texts, List dictionary, int compoundNounMaxTokens) { + List words = new ArrayList<>(); + List synonyms = new ArrayList<>(); + for (NlpDictionaryWordDto word : dictionary) { + (isSynonymRow(word) ? synonyms : words).add(word); + } + return tokenize(texts, words, synonyms, compoundNounMaxTokens); + } + + private static boolean isSynonymRow(NlpDictionaryWordDto word) { + return StringUtil.normalizeBlank(word.getSourceText()) != null; + } + + private List tokenize( + List texts, + List dictionary, + List synonyms, + int compoundNounMaxTokens) { if (texts.isEmpty()) { return List.of(); } if (!Files.isRegularFile(modelDirectory.resolve("cong.mdl"))) { throw unavailable(); } - Map tags = tags(); List result = new ArrayList<>(); - int options = KiwiBuilder.BuildOption.loadDefaultDict | KiwiBuilder.BuildOption.loadTypoDict; - // Use the user-approved latest model; retain dictionary, allomorph and typo-processing rules. + // Kiwi defaults: allomorph integration plus default, typo and multi-word dictionaries. + int options = KiwiBuilder.BuildOption.default_; try (KiwiBuilder kiwiBuilder = - new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong)) { - for (NlpDictionaryWordDto word : dictionary) { - Byte tag = tags.get(word.getWordTag()); - if (tag != null && word.getWordText() != null && word.getWordScore() != null) { - try { - kiwiBuilder.addWord(word.getWordText(), tag, word.getWordScore()); - } catch (RuntimeException ignored) { - // Existing dictionary registration ignores an individually invalid word. - } - } + new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong); + KiwiBuilder.PreparedTypoTransformer typoTransformer = + KiwiBuilder.basicTypoSetWithContinual.prepare()) { + // User words and synonym targets are never merged with their neighbors. + Set userWords = registerWords(kiwiBuilder, dictionary); + List spacedSynonyms = registerSynonyms(kiwiBuilder, synonyms, userWords); + boolean joinNouns = compoundNounMaxTokens > 1; + // Recognize URL/email/hashtag/mention/serial spans, normalize informal codas, + // analyze unknown words with the character model and correct common typos. + int match = Kiwi.Match.allWithNormalizing | Kiwi.Match.oovChrModel; + if (joinNouns) { + match |= Kiwi.Match.joinNounPrefix | Kiwi.Match.joinNounSuffix; } + Kiwi.AnalyzeOption analyzeOption = + new Kiwi.AnalyzeOption( + match, null, (short) 0, 0f, typoTransformer, TYPO_COST_THRESHOLD); try (Kiwi kiwi = kiwiBuilder.build()) { for (String text : texts) { - for (Kiwi.Token token : kiwi.tokenize(text, new Kiwi.AnalyzeOption(0))) { - String tag = Kiwi.POSTag.toString(token.tag); - result.add(new NlpTokenDto(token.form, tag == null ? "UNK" : tag)); + List tokens = + replaceSpacedSynonyms( + text, List.of(kiwi.tokenize(text, analyzeOption)), spacedSynonyms); + if (joinNouns) { + result.addAll(joinCompoundNouns(tokens, userWords, compoundNounMaxTokens)); + } else { + tokens.forEach(token -> result.add(toToken(token))); } } } @@ -62,6 +121,277 @@ public class KiwiNlpService { return result; } + /** + * Registers each word once with its highest scope score (10/50/100). Kiwi keeps the last score + * of a duplicate registration, and the dictionary query has no defined order. + */ + private Set registerWords( + KiwiBuilder kiwiBuilder, List dictionary) { + Map tags = tags(); + Map scores = new LinkedHashMap<>(); + int skipped = 0; + for (NlpDictionaryWordDto word : dictionary) { + String text = StringUtil.normalizeBlank(word.getWordText()); + String tagName = StringUtil.normalizeBlank(word.getWordTag()); + Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT)); + if (text == null || tag == null || word.getWordScore() == null) { + skipped++; + continue; + } + scores.merge(new DictionaryWord(text, tag), word.getWordScore(), Math::max); + } + int rejected = 0; + for (Map.Entry entry : scores.entrySet()) { + try { + if (!kiwiBuilder.addWord(entry.getKey().text(), entry.getKey().tag(), entry.getValue())) { + rejected++; + } + } catch (Exception ignored) { + // The native call throws checked exceptions; an invalid word does not stop the analysis. + rejected++; + } + } + if (skipped > 0 || rejected > 0) { + // Counts only: never log dictionary words. + log.warn( + "Kiwi user dictionary: registered={}, skipped={}, rejected={}", + scores.size() - rejected, + skipped, + rejected); + } + return new HashSet<>(scores.keySet()); + } + + /** + * Registers synonym rows above every user word: SOURCE_TEXT is analyzed directly as + * WORD_TEXT/WORD_TAG, and a spaced source also registers its spaceless form. The most specific + * scope (score 100 > 50 > 10) wins for the same source, and an explicit spaceless source wins over + * a derived one. Returns spaced sources, longest first, for {@link #replaceSpacedSynonyms}; adds + * every target to {@code protectedWords}. + */ + private List registerSynonyms( + KiwiBuilder kiwiBuilder, + List synonyms, + Set protectedWords) { + Map tags = tags(); + // Targets already registered as plain user words keep their scope score. + Set registeredTargets = new HashSet<>(protectedWords); + Map sources = new LinkedHashMap<>(); + Map sourceRanks = new HashMap<>(); + int skipped = 0; + for (NlpDictionaryWordDto synonym : synonyms) { + String source = normalizeSpaces(synonym.getSourceText()); + String target = StringUtil.normalizeBlank(synonym.getWordText()); + String tagName = StringUtil.normalizeBlank(synonym.getWordTag()); + Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT)); + if (source == null || target == null || tag == null) { + skipped++; + continue; + } + int rank = synonym.getWordScore() == null ? 0 : synonym.getWordScore(); + if (!sources.containsKey(source) || rank > sourceRanks.get(source)) { + sources.put(source, new Synonym(source, target, tag)); + sourceRanks.put(source, rank); + } + } + Map attached = new LinkedHashMap<>(); + Map attachedWeights = new HashMap<>(); + List spaced = new ArrayList<>(); + for (Synonym synonym : sources.values()) { + boolean explicit = !synonym.source().contains(" "); + String form = synonym.source().replace(" ", ""); + int weight = sourceRanks.get(synonym.source()) * 2 + (explicit ? 1 : 0); + Integer current = attachedWeights.get(form); + if (current == null || weight > current) { + attached.put(form, new Synonym(form, synonym.target(), synonym.tag())); + attachedWeights.put(form, weight); + } + if (!explicit) { + spaced.add(synonym); + } + protectedWords.add(new DictionaryWord(synonym.target(), synonym.tag())); + } + int rejected = 0; + for (Synonym synonym : attached.values()) { + KiwiBuilder.AnalyzedMorph[] morphs = { + new KiwiBuilder.AnalyzedMorph( + synonym.target(), synonym.tag(), 0, synonym.source().length()) + }; + try { + // Kiwi requires the target morpheme to exist, e.g. "LG" is not in the default dictionary. + // Score 0 keeps a short target from being forced inside longer words. + if (registeredTargets.add(new DictionaryWord(synonym.target(), synonym.tag()))) { + kiwiBuilder.addWord(synonym.target(), synonym.tag(), 0f); + } + if (!kiwiBuilder.addPreAnalyzedWord(synonym.source(), morphs, SYNONYM_SCORE)) { + rejected++; + } + } catch (Exception ignored) { + // The native call throws checked exceptions; an invalid synonym does not stop the analysis. + rejected++; + } + } + if (skipped > 0 || rejected > 0) { + // Counts only: never log synonym texts. + log.warn( + "Kiwi synonyms: registered={}, skipped={}, rejected={}", + attached.size() - rejected, + skipped, + rejected); + } + spaced.sort(Comparator.comparingInt((Synonym synonym) -> synonym.source().length()).reversed()); + return spaced; + } + + /** + * Replaces a run of tokens whose original text, with spaces collapsed, equals a spaced synonym + * source. The run must start and end on token boundaries and must not continue into a noun of + * the same word, so "엘지 전자레인지" (엘지/전자/레인지) is not "엘지 전자". + */ + static List replaceSpacedSynonyms( + String text, List tokens, List spacedSynonyms) { + if (spacedSynonyms.isEmpty()) { + return tokens; + } + List result = new ArrayList<>(); + int start = 0; + while (start < tokens.size()) { + int end = -1; + Synonym matched = null; + for (Synonym synonym : spacedSynonyms) { + end = matchEnd(text, tokens, start, synonym.source()); + if (end > start && !continuesNoun(tokens, start, end)) { + matched = synonym; + break; + } + } + if (matched == null) { + result.add(tokens.get(start)); + start++; + continue; + } + Kiwi.Token first = tokens.get(start); + Kiwi.Token last = tokens.get(end - 1); + Kiwi.Token replaced = new Kiwi.Token(); + replaced.form = matched.target(); + replaced.tag = matched.tag(); + replaced.position = first.position; + replaced.length = (short) (last.position + last.length - first.position); + replaced.wordPosition = first.wordPosition; + replaced.sentPosition = first.sentPosition; + replaced.lineNumber = first.lineNumber; + result.add(replaced); + start = end; + } + return result; + } + + /** Whether a noun is attached right before or after the run inside the same word. */ + private static boolean continuesNoun(List tokens, int start, int end) { + boolean before = + start > 0 + && isAttached(tokens.get(start - 1), tokens.get(start)) + && isNounLike(tokens.get(start - 1)); + boolean after = + end < tokens.size() + && isAttached(tokens.get(end - 1), tokens.get(end)) + && isNounLike(tokens.get(end)); + return before || after; + } + + private static boolean isNounLike(Kiwi.Token token) { + String tag = tagName(token); + return tag.startsWith("N") || NOUN_AFFIX_TAGS.contains(tag); + } + + /** Exclusive end index of the run starting at {@code start} that spells {@code source}, or -1. */ + private static int matchEnd(String text, List tokens, int start, String source) { + int from = tokens.get(start).position; + for (int end = start; end < tokens.size(); end++) { + Kiwi.Token last = tokens.get(end); + int to = last.position + last.length; + if (from < 0 || to > text.length() || to < from) { + return -1; + } + String span = WHITESPACE.matcher(text.substring(from, to)).replaceAll(" "); + if (span.equals(source)) { + return end + 1; + } + if (span.length() >= source.length()) { + return -1; + } + } + return -1; + } + + private static String normalizeSpaces(String raw) { + String normalized = StringUtil.normalizeBlank(raw); + return normalized == null ? null : WHITESPACE.matcher(normalized).replaceAll(" "); + } + + /** + * Merges a run of 2..maxTokens NNG/NNP/SL tokens written without a space inside one word into a + * single noun, when the run contains a Korean noun. The merged tag is NNP if any part is NNP, + * otherwise NNG. User dictionary words always stay as they are, and longer runs are kept apart. + */ + static List joinCompoundNouns( + List tokens, Set userWords, int maxTokens) { + List result = new ArrayList<>(); + int start = 0; + while (start < tokens.size()) { + if (!isCompoundPart(tokens.get(start), userWords)) { + result.add(toToken(tokens.get(start))); + start++; + continue; + } + int end = start + 1; + while (end < tokens.size() + && isCompoundPart(tokens.get(end), userWords) + && isAttached(tokens.get(end - 1), tokens.get(end))) { + end++; + } + List run = tokens.subList(start, end); + boolean hasKoreanNoun = run.stream().anyMatch(token -> !isForeignWord(token)); + if (run.size() > 1 && run.size() <= maxTokens && hasKoreanNoun) { + StringBuilder form = new StringBuilder(); + boolean properNoun = false; + for (Kiwi.Token token : run) { + form.append(token.form); + properNoun |= "NNP".equals(tagName(token)); + } + result.add(new NlpTokenDto(form.toString(), properNoun ? "NNP" : "NNG")); + } else { + run.forEach(token -> result.add(toToken(token))); + } + start = end; + } + return result; + } + + private static boolean isCompoundPart(Kiwi.Token token, Set userWords) { + return COMPOUND_NOUN_TAGS.contains(tagName(token)) + && !userWords.contains(new DictionaryWord(token.form, token.tag)); + } + + private static boolean isForeignWord(Kiwi.Token token) { + return "SL".equals(tagName(token)); + } + + /** Same word and no space or other character between the two tokens. */ + private static boolean isAttached(Kiwi.Token previous, Kiwi.Token next) { + return previous.wordPosition == next.wordPosition + && previous.position + previous.length == next.position; + } + + private static NlpTokenDto toToken(Kiwi.Token token) { + return new NlpTokenDto(token.form, tagName(token)); + } + + private static String tagName(Kiwi.Token token) { + String tag = Kiwi.POSTag.toString(token.tag); + return tag == null ? "UNK" : tag; + } + private Map tags() { Map result = new LinkedHashMap<>(); for (int code = Byte.MIN_VALUE; code <= Byte.MAX_VALUE; code++) { @@ -77,4 +407,8 @@ public class KiwiNlpService { return BusinessException.of( HttpStatus.SERVICE_UNAVAILABLE, "KIWI_NLP_UNAVAILABLE", "Kiwi NLP 라이브러리와 모델 설정을 확인해 주세요."); } + + record DictionaryWord(String text, byte tag) {} + + record Synonym(String source, String target, byte tag) {} } diff --git a/src/main/java/com/kri/kart/server/service/statistics/StatisticsTextService.java b/src/main/java/com/kri/kart/server/service/statistics/StatisticsTextService.java index 27f1ea9..07afbf5 100644 --- a/src/main/java/com/kri/kart/server/service/statistics/StatisticsTextService.java +++ b/src/main/java/com/kri/kart/server/service/statistics/StatisticsTextService.java @@ -1,19 +1,25 @@ package com.kri.kart.server.service.statistics; +import com.kri.common.feature.util.StringUtil; import com.kri.kart.server.common.consts.KartConstant; +import com.kri.kart.server.common.enums.WordMergeMode; import com.kri.kart.server.dto.project.ProjectRefDto; +import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto; import com.kri.kart.server.dto.statistics.NlpTokenDto; import com.kri.kart.server.dto.statistics.StatisticsWordDto; import com.kri.kart.server.dto.statistics.SurveyResponseDto; import com.kri.kart.server.persistence.repository.NlpRepository; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; +import java.util.regex.Pattern; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -24,31 +30,128 @@ import org.springframework.stereotype.Service; @ConditionalOnProperty(prefix = "kri.kart.database", name = "enabled", havingValue = "true") public class StatisticsTextService { + // Noun family for WordMergeMode.NOUN: nouns, foreign words and Chinese characters. + private static final Set NOUN_FAMILY_TAGS = + Set.of("NNG", "NNP", "NNB", "NR", "NP", "SL", "SH"); + private static final Pattern WHITESPACE = Pattern.compile("\\s+"); + private final NlpRepository nlpRepository; private final KiwiNlpService kiwiNlpService; private final StatisticsSourceService statisticsSourceService; + /** + * Keeps the existing behavior: no synonyms or stopwords, the same word with a different tag is + * counted separately and compound nouns are not merged. + */ public List analyze( ProjectRefDto project, String qid, List responses) { + return analyze(project, qid, responses, false, WordMergeMode.NONE, 1); + } + + /** + * Applies the REPORT_NLP_WORD synonyms (rows with SOURCE_TEXT), the stopwords (불용어, rows + * tagged STOP) and the given merge options. + */ + public List analyze( + ProjectRefDto project, + String qid, + List responses, + WordMergeMode mode, + int compoundNounMaxTokens) { + return analyze(project, qid, responses, true, mode, compoundNounMaxTokens); + } + + private List analyze( + ProjectRefDto project, + String qid, + List responses, + boolean extended, + WordMergeMode mode, + int compoundNounMaxTokens) { List texts = answerTexts(qid, responses); Set selectedTags = new LinkedHashSet<>(nlpRepository.tags(project.pkey(), qid)); + // 불용어 (STOP) rows are not dictionary words: they are never registered in Kiwi. + List dictionary = new ArrayList<>(); + Set stopwords = new HashSet<>(); + for (NlpDictionaryWordDto word : nlpRepository.words(project.pkey(), qid)) { + if (isStopwordRow(word)) { + String stopword = stopwordKey(word.getWordText()); + if (stopword != null) { + stopwords.add(stopword); + } + } else { + dictionary.add(word); + } + } + // The original overload ignores synonym rows and does not merge compound nouns. List tokens = - kiwiNlpService.tokenize(texts, nlpRepository.words(project.pkey(), qid)); - // The same word with a different tag is counted separately, in first-appearance order. + extended + ? kiwiNlpService.tokenize(texts, dictionary, compoundNounMaxTokens) + : kiwiNlpService.tokenize(texts, dictionary); + // Count each word and tag pair in first-appearance order. LinkedHashMap counts = new LinkedHashMap<>(); for (NlpTokenDto token : tokens) { - if (isCounted(token, selectedTags)) { + // Stopwords apply to the final word after synonyms and compound merging, for every tag. + boolean stopword = extended && stopwords.contains(stopwordKey(token.text())); + if (!stopword && isCounted(token, selectedTags)) { counts.merge(token, 1L, Long::sum); } } // Stable sort: equal counts keep their first-appearance order. - return counts.entrySet().stream() + return merge(counts, mode == null ? WordMergeMode.NONE : mode).entrySet().stream() .sorted(Map.Entry.comparingByValue().reversed()) .limit(KartConstant.TOP_WORD_LIMIT) .map(entry -> toWord(entry.getKey(), entry.getValue())) .toList(); } + /** + * Sums the same text across mergeable tags. The merged word keeps the tag with the highest count + * (first appearance on ties) and the position of its first appearance. + */ + private LinkedHashMap merge( + LinkedHashMap counts, WordMergeMode mode) { + if (mode == WordMergeMode.NONE) { + return counts; + } + LinkedHashMap> groups = new LinkedHashMap<>(); + for (Map.Entry entry : counts.entrySet()) { + NlpTokenDto token = entry.getKey(); + boolean mergeable = mode == WordMergeMode.ALL || NOUN_FAMILY_TAGS.contains(token.tag()); + NlpTokenDto key = mergeable ? new NlpTokenDto(token.text(), null) : token; + groups.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).put(token.tag(), entry.getValue()); + } + LinkedHashMap result = new LinkedHashMap<>(); + for (Map.Entry> group : groups.entrySet()) { + String tag = null; + long best = 0; + long total = 0; + for (Map.Entry tagCount : group.getValue().entrySet()) { + total += tagCount.getValue(); + if (tag == null || tagCount.getValue() > best) { + tag = tagCount.getKey(); + best = tagCount.getValue(); + } + } + result.put(new NlpTokenDto(group.getKey().text(), tag), total); + } + return result; + } + + /** 불용어 row: WORD_TAG is STOP, ignoring case and surrounding spaces. */ + private boolean isStopwordRow(NlpDictionaryWordDto word) { + String tag = StringUtil.normalizeBlank(word.getWordTag()); + return KartConstant.STOPWORD_TAG.equalsIgnoreCase(tag); + } + + /** Stopword comparison key: trimmed, spaces collapsed and case-insensitive. */ + private String stopwordKey(String text) { + String normalized = StringUtil.normalizeBlank(text); + return normalized == null + ? null + : WHITESPACE.matcher(normalized).replaceAll(" ").toLowerCase(Locale.ROOT); + } + private StatisticsWordDto toWord(NlpTokenDto token, long count) { return new StatisticsWordDto(token.text(), token.tag(), count); } diff --git a/src/main/resources/mybatis/mapper/NlpMapper.xml b/src/main/resources/mybatis/mapper/NlpMapper.xml index 6f65588..c25c0e3 100644 --- a/src/main/resources/mybatis/mapper/NlpMapper.xml +++ b/src/main/resources/mybatis/mapper/NlpMapper.xml @@ -11,13 +11,14 @@ ) TAGS From 55d068a17fa123704c8a48b35793c0c7f74b62eb Mon Sep 17 00:00:00 2001 From: kric_jspark Date: Wed, 11 Nov 2026 12:29:38 +0900 Subject: [PATCH 2/2] =?UTF-8?q?261111=5F02=5FENH:=20Kiwi=20=EC=82=AC?= =?UTF-8?q?=EC=A0=84=C2=B7=EC=B9=98=ED=99=98=20=EB=93=B1=EB=A1=9D=20?= =?UTF-8?q?=EA=B2=BD=EA=B3=A0=20=EB=A1=9C=EA=B7=B8=EC=9D=98=20=EC=9D=B8?= =?UTF-8?q?=EC=9E=90=20=EA=B3=84=EC=82=B0=EC=9D=84=20=EB=B3=80=EC=88=98?= =?UTF-8?q?=EB=A1=9C=20=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kri/kart/server/service/statistics/KiwiNlpService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java b/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java index 4de07eb..deb754a 100644 --- a/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java +++ b/src/main/java/com/kri/kart/server/service/statistics/KiwiNlpService.java @@ -152,10 +152,11 @@ public class KiwiNlpService { } } if (skipped > 0 || rejected > 0) { + int registered = scores.size() - rejected; // Counts only: never log dictionary words. log.warn( "Kiwi user dictionary: registered={}, skipped={}, rejected={}", - scores.size() - rejected, + registered, skipped, rejected); } @@ -232,10 +233,11 @@ public class KiwiNlpService { } } if (skipped > 0 || rejected > 0) { + int registered = attached.size() - rejected; // Counts only: never log synonym texts. log.warn( "Kiwi synonyms: registered={}, skipped={}, rejected={}", - attached.size() - rejected, + registered, skipped, rejected); }