261111_01_ENH: Kiwi 형태소 분석 정확도 개선 및 품사 합산·복합명사·치환·불용어 옵션 추가

This commit is contained in:
2026-11-11 10:37:13 +09:00
parent fb76c9dcb3
commit 6e1d57414e
13 changed files with 572 additions and 36 deletions

View File

@@ -100,6 +100,8 @@ public final class KartConstant {
// StatisticsTextService
public static final int TOP_WORD_LIMIT = 100;
public static final String FOREIGN_WORD_TAG = "SL";
// 불용어: REPORT_NLP_WORD rows with this WORD_TAG are excluded from word counts.
public static final String STOPWORD_TAG = "STOP";
// StatisticsController
public static final String CROSSTAB_PAGE = "/Report/Crosstab";

View File

@@ -0,0 +1,32 @@
package com.kri.kart.server.common.enums;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
import com.kri.common.feature.enums.CodeEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Getter(onMethod_ = @JsonValue)
@RequiredArgsConstructor
@Schema(description = "같은 단어의 품사별 빈도 합산: 1=모두 합치기, 2=명사 계열만 합치기, 3=합치지 않기")
public enum WordMergeMode implements CodeEnum<Integer> {
ALL(1),
NOUN(2),
NONE(3);
private final Integer code;
@JsonCreator(mode = JsonCreator.Mode.DELEGATING)
public static WordMergeMode fromCode(Integer code) {
if (code == null) {
return null;
}
for (WordMergeMode s : values()) {
if (s.code.equals(code)) {
return s;
}
}
throw new IllegalArgumentException("Unknown WordMergeMode code: " + code);
}
}

View File

@@ -3,10 +3,15 @@ package com.kri.kart.server.dto.statistics;
import lombok.Getter;
import lombok.Setter;
/**
* REPORT_NLP_WORD row. WORD_TAG STOP marks a 불용어 (stopword); a row with SOURCE_TEXT is a synonym
* that replaces SOURCE_TEXT with WORD_TEXT/WORD_TAG; any other row is a Kiwi user word.
*/
@Getter
@Setter
public class NlpDictionaryWordDto {
private String wordText;
private String wordTag;
private Integer wordScore;
private String sourceText;
}

View File

@@ -0,0 +1,14 @@
package com.kri.kart.server.dto.statistics;
import lombok.Getter;
import lombok.Setter;
/** Text-analysis replacement such as 엘지전자 → LG전자, with 1=ALL, 2=project, 3=question scope. */
@Getter
@Setter
public class NlpSynonymDto {
private String sourceText;
private String targetText;
private String targetTag;
private Integer scopeRank;
}

View File

@@ -1,6 +1,7 @@
package com.kri.kart.server.dto.statistics;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.kri.kart.server.common.enums.WordMergeMode;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.Max;
@@ -55,4 +56,13 @@ public class StatisticsOptionsDto {
private boolean showNumericSum;
@Valid @NotNull private List<StatisticsValueGroupDto> valueGroups = new ArrayList<>();
@NotNull
@Schema(description = "텍스트 분석에서 같은 단어의 품사별 빈도 합산 방식. 생략하면 2(명사 계열만 합치기)")
private WordMergeMode wordMergeMode = WordMergeMode.NOUN;
@Min(1)
@Max(5)
@Schema(description = "텍스트 분석에서 공백 없이 붙은 명사를 합칠 최대 토큰 수. 1이면 합치지 않음. 생략하면 3")
private int compoundNounMaxTokens = 3;
}

View File

@@ -249,7 +249,13 @@ public class StatisticsService {
.columns(List.of())
.headerRows(List.of())
.rows(List.of())
.words(statisticsTextService.analyze(project, index.getDimensionId(), responses))
.words(
statisticsTextService.analyze(
project,
index.getDimensionId(),
responses,
request.getWordMergeMode(),
request.getCompoundNounMaxTokens()))
.appliedOptions(request)
.build();
}

View File

@@ -1,22 +1,39 @@
package com.kri.kart.server.service.statistics;
import com.kri.common.feature.util.StringUtil;
import com.kri.common.feature.web.BusinessException;
import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto;
import com.kri.kart.server.dto.statistics.NlpTokenDto;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import kr.pe.bab2min.Kiwi;
import kr.pe.bab2min.KiwiBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class KiwiNlpService {
// Same as kiwipiepy's default typo cost threshold.
private static final float TYPO_COST_THRESHOLD = 2.5f;
private static final Set<String> COMPOUND_NOUN_TAGS = Set.of("NNG", "NNP", "SL");
private static final Set<String> NOUN_AFFIX_TAGS = Set.of("SL", "SH", "XPN", "XSN");
// Above every user dictionary score (10/50/100) so a synonym always wins.
private static final float SYNONYM_SCORE = 200f;
private static final Pattern WHITESPACE = Pattern.compile("\\s+");
private final Path modelDirectory;
public KiwiNlpService(
@@ -24,34 +41,76 @@ public class KiwiNlpService {
this.modelDirectory = Path.of(modelDirectory);
}
/**
* Keeps the existing behavior: synonym rows (with SOURCE_TEXT) are ignored and there is no
* compound-noun or noun-affix joining.
*/
public List<NlpTokenDto> tokenize(List<String> texts, List<NlpDictionaryWordDto> dictionary) {
List<NlpDictionaryWordDto> words =
dictionary.stream().filter(word -> !isSynonymRow(word)).toList();
return tokenize(texts, words, List.of(), 1);
}
/**
* Tokenizes texts with REPORT_NLP_WORD rows: a row with SOURCE_TEXT is a synonym that replaces
* SOURCE_TEXT with WORD_TEXT/WORD_TAG before anything else, any other row is a user word. When
* {@code compoundNounMaxTokens} is 2 or more, noun prefixes/suffixes are joined and adjacent
* nouns are merged into one compound noun (see {@link #joinCompoundNouns}).
*/
public List<NlpTokenDto> tokenize(
List<String> texts, List<NlpDictionaryWordDto> dictionary, int compoundNounMaxTokens) {
List<NlpDictionaryWordDto> words = new ArrayList<>();
List<NlpDictionaryWordDto> synonyms = new ArrayList<>();
for (NlpDictionaryWordDto word : dictionary) {
(isSynonymRow(word) ? synonyms : words).add(word);
}
return tokenize(texts, words, synonyms, compoundNounMaxTokens);
}
private static boolean isSynonymRow(NlpDictionaryWordDto word) {
return StringUtil.normalizeBlank(word.getSourceText()) != null;
}
private List<NlpTokenDto> tokenize(
List<String> texts,
List<NlpDictionaryWordDto> dictionary,
List<NlpDictionaryWordDto> synonyms,
int compoundNounMaxTokens) {
if (texts.isEmpty()) {
return List.of();
}
if (!Files.isRegularFile(modelDirectory.resolve("cong.mdl"))) {
throw unavailable();
}
Map<String, Byte> tags = tags();
List<NlpTokenDto> result = new ArrayList<>();
int options = KiwiBuilder.BuildOption.loadDefaultDict | KiwiBuilder.BuildOption.loadTypoDict;
// Use the user-approved latest model; retain dictionary, allomorph and typo-processing rules.
// Kiwi defaults: allomorph integration plus default, typo and multi-word dictionaries.
int options = KiwiBuilder.BuildOption.default_;
try (KiwiBuilder kiwiBuilder =
new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong)) {
for (NlpDictionaryWordDto word : dictionary) {
Byte tag = tags.get(word.getWordTag());
if (tag != null && word.getWordText() != null && word.getWordScore() != null) {
try {
kiwiBuilder.addWord(word.getWordText(), tag, word.getWordScore());
} catch (RuntimeException ignored) {
// Existing dictionary registration ignores an individually invalid word.
}
}
new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong);
KiwiBuilder.PreparedTypoTransformer typoTransformer =
KiwiBuilder.basicTypoSetWithContinual.prepare()) {
// User words and synonym targets are never merged with their neighbors.
Set<DictionaryWord> userWords = registerWords(kiwiBuilder, dictionary);
List<Synonym> spacedSynonyms = registerSynonyms(kiwiBuilder, synonyms, userWords);
boolean joinNouns = compoundNounMaxTokens > 1;
// Recognize URL/email/hashtag/mention/serial spans, normalize informal codas,
// analyze unknown words with the character model and correct common typos.
int match = Kiwi.Match.allWithNormalizing | Kiwi.Match.oovChrModel;
if (joinNouns) {
match |= Kiwi.Match.joinNounPrefix | Kiwi.Match.joinNounSuffix;
}
Kiwi.AnalyzeOption analyzeOption =
new Kiwi.AnalyzeOption(
match, null, (short) 0, 0f, typoTransformer, TYPO_COST_THRESHOLD);
try (Kiwi kiwi = kiwiBuilder.build()) {
for (String text : texts) {
for (Kiwi.Token token : kiwi.tokenize(text, new Kiwi.AnalyzeOption(0))) {
String tag = Kiwi.POSTag.toString(token.tag);
result.add(new NlpTokenDto(token.form, tag == null ? "UNK" : tag));
List<Kiwi.Token> tokens =
replaceSpacedSynonyms(
text, List.of(kiwi.tokenize(text, analyzeOption)), spacedSynonyms);
if (joinNouns) {
result.addAll(joinCompoundNouns(tokens, userWords, compoundNounMaxTokens));
} else {
tokens.forEach(token -> result.add(toToken(token)));
}
}
}
@@ -62,6 +121,277 @@ public class KiwiNlpService {
return result;
}
/**
* Registers each word once with its highest scope score (10/50/100). Kiwi keeps the last score
* of a duplicate registration, and the dictionary query has no defined order.
*/
private Set<DictionaryWord> registerWords(
KiwiBuilder kiwiBuilder, List<NlpDictionaryWordDto> dictionary) {
Map<String, Byte> tags = tags();
Map<DictionaryWord, Integer> scores = new LinkedHashMap<>();
int skipped = 0;
for (NlpDictionaryWordDto word : dictionary) {
String text = StringUtil.normalizeBlank(word.getWordText());
String tagName = StringUtil.normalizeBlank(word.getWordTag());
Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT));
if (text == null || tag == null || word.getWordScore() == null) {
skipped++;
continue;
}
scores.merge(new DictionaryWord(text, tag), word.getWordScore(), Math::max);
}
int rejected = 0;
for (Map.Entry<DictionaryWord, Integer> entry : scores.entrySet()) {
try {
if (!kiwiBuilder.addWord(entry.getKey().text(), entry.getKey().tag(), entry.getValue())) {
rejected++;
}
} catch (Exception ignored) {
// The native call throws checked exceptions; an invalid word does not stop the analysis.
rejected++;
}
}
if (skipped > 0 || rejected > 0) {
// Counts only: never log dictionary words.
log.warn(
"Kiwi user dictionary: registered={}, skipped={}, rejected={}",
scores.size() - rejected,
skipped,
rejected);
}
return new HashSet<>(scores.keySet());
}
/**
* Registers synonym rows above every user word: SOURCE_TEXT is analyzed directly as
* WORD_TEXT/WORD_TAG, and a spaced source also registers its spaceless form. The most specific
* scope (score 100 > 50 > 10) wins for the same source, and an explicit spaceless source wins over
* a derived one. Returns spaced sources, longest first, for {@link #replaceSpacedSynonyms}; adds
* every target to {@code protectedWords}.
*/
private List<Synonym> registerSynonyms(
KiwiBuilder kiwiBuilder,
List<NlpDictionaryWordDto> synonyms,
Set<DictionaryWord> protectedWords) {
Map<String, Byte> tags = tags();
// Targets already registered as plain user words keep their scope score.
Set<DictionaryWord> registeredTargets = new HashSet<>(protectedWords);
Map<String, Synonym> sources = new LinkedHashMap<>();
Map<String, Integer> sourceRanks = new HashMap<>();
int skipped = 0;
for (NlpDictionaryWordDto synonym : synonyms) {
String source = normalizeSpaces(synonym.getSourceText());
String target = StringUtil.normalizeBlank(synonym.getWordText());
String tagName = StringUtil.normalizeBlank(synonym.getWordTag());
Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT));
if (source == null || target == null || tag == null) {
skipped++;
continue;
}
int rank = synonym.getWordScore() == null ? 0 : synonym.getWordScore();
if (!sources.containsKey(source) || rank > sourceRanks.get(source)) {
sources.put(source, new Synonym(source, target, tag));
sourceRanks.put(source, rank);
}
}
Map<String, Synonym> attached = new LinkedHashMap<>();
Map<String, Integer> attachedWeights = new HashMap<>();
List<Synonym> spaced = new ArrayList<>();
for (Synonym synonym : sources.values()) {
boolean explicit = !synonym.source().contains(" ");
String form = synonym.source().replace(" ", "");
int weight = sourceRanks.get(synonym.source()) * 2 + (explicit ? 1 : 0);
Integer current = attachedWeights.get(form);
if (current == null || weight > current) {
attached.put(form, new Synonym(form, synonym.target(), synonym.tag()));
attachedWeights.put(form, weight);
}
if (!explicit) {
spaced.add(synonym);
}
protectedWords.add(new DictionaryWord(synonym.target(), synonym.tag()));
}
int rejected = 0;
for (Synonym synonym : attached.values()) {
KiwiBuilder.AnalyzedMorph[] morphs = {
new KiwiBuilder.AnalyzedMorph(
synonym.target(), synonym.tag(), 0, synonym.source().length())
};
try {
// Kiwi requires the target morpheme to exist, e.g. "LG" is not in the default dictionary.
// Score 0 keeps a short target from being forced inside longer words.
if (registeredTargets.add(new DictionaryWord(synonym.target(), synonym.tag()))) {
kiwiBuilder.addWord(synonym.target(), synonym.tag(), 0f);
}
if (!kiwiBuilder.addPreAnalyzedWord(synonym.source(), morphs, SYNONYM_SCORE)) {
rejected++;
}
} catch (Exception ignored) {
// The native call throws checked exceptions; an invalid synonym does not stop the analysis.
rejected++;
}
}
if (skipped > 0 || rejected > 0) {
// Counts only: never log synonym texts.
log.warn(
"Kiwi synonyms: registered={}, skipped={}, rejected={}",
attached.size() - rejected,
skipped,
rejected);
}
spaced.sort(Comparator.comparingInt((Synonym synonym) -> synonym.source().length()).reversed());
return spaced;
}
/**
* Replaces a run of tokens whose original text, with spaces collapsed, equals a spaced synonym
* source. The run must start and end on token boundaries and must not continue into a noun of
* the same word, so "엘지 전자레인지" (엘지/전자/레인지) is not "엘지 전자".
*/
static List<Kiwi.Token> replaceSpacedSynonyms(
String text, List<Kiwi.Token> tokens, List<Synonym> spacedSynonyms) {
if (spacedSynonyms.isEmpty()) {
return tokens;
}
List<Kiwi.Token> result = new ArrayList<>();
int start = 0;
while (start < tokens.size()) {
int end = -1;
Synonym matched = null;
for (Synonym synonym : spacedSynonyms) {
end = matchEnd(text, tokens, start, synonym.source());
if (end > start && !continuesNoun(tokens, start, end)) {
matched = synonym;
break;
}
}
if (matched == null) {
result.add(tokens.get(start));
start++;
continue;
}
Kiwi.Token first = tokens.get(start);
Kiwi.Token last = tokens.get(end - 1);
Kiwi.Token replaced = new Kiwi.Token();
replaced.form = matched.target();
replaced.tag = matched.tag();
replaced.position = first.position;
replaced.length = (short) (last.position + last.length - first.position);
replaced.wordPosition = first.wordPosition;
replaced.sentPosition = first.sentPosition;
replaced.lineNumber = first.lineNumber;
result.add(replaced);
start = end;
}
return result;
}
/** Whether a noun is attached right before or after the run inside the same word. */
private static boolean continuesNoun(List<Kiwi.Token> tokens, int start, int end) {
boolean before =
start > 0
&& isAttached(tokens.get(start - 1), tokens.get(start))
&& isNounLike(tokens.get(start - 1));
boolean after =
end < tokens.size()
&& isAttached(tokens.get(end - 1), tokens.get(end))
&& isNounLike(tokens.get(end));
return before || after;
}
private static boolean isNounLike(Kiwi.Token token) {
String tag = tagName(token);
return tag.startsWith("N") || NOUN_AFFIX_TAGS.contains(tag);
}
/** Exclusive end index of the run starting at {@code start} that spells {@code source}, or -1. */
private static int matchEnd(String text, List<Kiwi.Token> tokens, int start, String source) {
int from = tokens.get(start).position;
for (int end = start; end < tokens.size(); end++) {
Kiwi.Token last = tokens.get(end);
int to = last.position + last.length;
if (from < 0 || to > text.length() || to < from) {
return -1;
}
String span = WHITESPACE.matcher(text.substring(from, to)).replaceAll(" ");
if (span.equals(source)) {
return end + 1;
}
if (span.length() >= source.length()) {
return -1;
}
}
return -1;
}
private static String normalizeSpaces(String raw) {
String normalized = StringUtil.normalizeBlank(raw);
return normalized == null ? null : WHITESPACE.matcher(normalized).replaceAll(" ");
}
/**
* Merges a run of 2..maxTokens NNG/NNP/SL tokens written without a space inside one word into a
* single noun, when the run contains a Korean noun. The merged tag is NNP if any part is NNP,
* otherwise NNG. User dictionary words always stay as they are, and longer runs are kept apart.
*/
static List<NlpTokenDto> joinCompoundNouns(
List<Kiwi.Token> tokens, Set<DictionaryWord> userWords, int maxTokens) {
List<NlpTokenDto> result = new ArrayList<>();
int start = 0;
while (start < tokens.size()) {
if (!isCompoundPart(tokens.get(start), userWords)) {
result.add(toToken(tokens.get(start)));
start++;
continue;
}
int end = start + 1;
while (end < tokens.size()
&& isCompoundPart(tokens.get(end), userWords)
&& isAttached(tokens.get(end - 1), tokens.get(end))) {
end++;
}
List<Kiwi.Token> run = tokens.subList(start, end);
boolean hasKoreanNoun = run.stream().anyMatch(token -> !isForeignWord(token));
if (run.size() > 1 && run.size() <= maxTokens && hasKoreanNoun) {
StringBuilder form = new StringBuilder();
boolean properNoun = false;
for (Kiwi.Token token : run) {
form.append(token.form);
properNoun |= "NNP".equals(tagName(token));
}
result.add(new NlpTokenDto(form.toString(), properNoun ? "NNP" : "NNG"));
} else {
run.forEach(token -> result.add(toToken(token)));
}
start = end;
}
return result;
}
private static boolean isCompoundPart(Kiwi.Token token, Set<DictionaryWord> userWords) {
return COMPOUND_NOUN_TAGS.contains(tagName(token))
&& !userWords.contains(new DictionaryWord(token.form, token.tag));
}
private static boolean isForeignWord(Kiwi.Token token) {
return "SL".equals(tagName(token));
}
/** Same word and no space or other character between the two tokens. */
private static boolean isAttached(Kiwi.Token previous, Kiwi.Token next) {
return previous.wordPosition == next.wordPosition
&& previous.position + previous.length == next.position;
}
private static NlpTokenDto toToken(Kiwi.Token token) {
return new NlpTokenDto(token.form, tagName(token));
}
private static String tagName(Kiwi.Token token) {
String tag = Kiwi.POSTag.toString(token.tag);
return tag == null ? "UNK" : tag;
}
private Map<String, Byte> tags() {
Map<String, Byte> result = new LinkedHashMap<>();
for (int code = Byte.MIN_VALUE; code <= Byte.MAX_VALUE; code++) {
@@ -77,4 +407,8 @@ public class KiwiNlpService {
return BusinessException.of(
HttpStatus.SERVICE_UNAVAILABLE, "KIWI_NLP_UNAVAILABLE", "Kiwi NLP 라이브러리와 모델 설정을 확인해 주세요.");
}
record DictionaryWord(String text, byte tag) {}
record Synonym(String source, String target, byte tag) {}
}

View File

@@ -1,19 +1,25 @@
package com.kri.kart.server.service.statistics;
import com.kri.common.feature.util.StringUtil;
import com.kri.kart.server.common.consts.KartConstant;
import com.kri.kart.server.common.enums.WordMergeMode;
import com.kri.kart.server.dto.project.ProjectRefDto;
import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto;
import com.kri.kart.server.dto.statistics.NlpTokenDto;
import com.kri.kart.server.dto.statistics.StatisticsWordDto;
import com.kri.kart.server.dto.statistics.SurveyResponseDto;
import com.kri.kart.server.persistence.repository.NlpRepository;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@@ -24,31 +30,128 @@ import org.springframework.stereotype.Service;
@ConditionalOnProperty(prefix = "kri.kart.database", name = "enabled", havingValue = "true")
public class StatisticsTextService {
// Noun family for WordMergeMode.NOUN: nouns, foreign words and Chinese characters.
private static final Set<String> NOUN_FAMILY_TAGS =
Set.of("NNG", "NNP", "NNB", "NR", "NP", "SL", "SH");
private static final Pattern WHITESPACE = Pattern.compile("\\s+");
private final NlpRepository nlpRepository;
private final KiwiNlpService kiwiNlpService;
private final StatisticsSourceService statisticsSourceService;
/**
* Keeps the existing behavior: no synonyms or stopwords, the same word with a different tag is
* counted separately and compound nouns are not merged.
*/
public List<StatisticsWordDto> analyze(
ProjectRefDto project, String qid, List<SurveyResponseDto> responses) {
return analyze(project, qid, responses, false, WordMergeMode.NONE, 1);
}
/**
* Applies the REPORT_NLP_WORD synonyms (rows with SOURCE_TEXT), the stopwords (불용어, rows
* tagged STOP) and the given merge options.
*/
public List<StatisticsWordDto> analyze(
ProjectRefDto project,
String qid,
List<SurveyResponseDto> responses,
WordMergeMode mode,
int compoundNounMaxTokens) {
return analyze(project, qid, responses, true, mode, compoundNounMaxTokens);
}
private List<StatisticsWordDto> analyze(
ProjectRefDto project,
String qid,
List<SurveyResponseDto> responses,
boolean extended,
WordMergeMode mode,
int compoundNounMaxTokens) {
List<String> texts = answerTexts(qid, responses);
Set<String> selectedTags = new LinkedHashSet<>(nlpRepository.tags(project.pkey(), qid));
// 불용어 (STOP) rows are not dictionary words: they are never registered in Kiwi.
List<NlpDictionaryWordDto> dictionary = new ArrayList<>();
Set<String> stopwords = new HashSet<>();
for (NlpDictionaryWordDto word : nlpRepository.words(project.pkey(), qid)) {
if (isStopwordRow(word)) {
String stopword = stopwordKey(word.getWordText());
if (stopword != null) {
stopwords.add(stopword);
}
} else {
dictionary.add(word);
}
}
// The original overload ignores synonym rows and does not merge compound nouns.
List<NlpTokenDto> tokens =
kiwiNlpService.tokenize(texts, nlpRepository.words(project.pkey(), qid));
// The same word with a different tag is counted separately, in first-appearance order.
extended
? kiwiNlpService.tokenize(texts, dictionary, compoundNounMaxTokens)
: kiwiNlpService.tokenize(texts, dictionary);
// Count each word and tag pair in first-appearance order.
LinkedHashMap<NlpTokenDto, Long> counts = new LinkedHashMap<>();
for (NlpTokenDto token : tokens) {
if (isCounted(token, selectedTags)) {
// Stopwords apply to the final word after synonyms and compound merging, for every tag.
boolean stopword = extended && stopwords.contains(stopwordKey(token.text()));
if (!stopword && isCounted(token, selectedTags)) {
counts.merge(token, 1L, Long::sum);
}
}
// Stable sort: equal counts keep their first-appearance order.
return counts.entrySet().stream()
return merge(counts, mode == null ? WordMergeMode.NONE : mode).entrySet().stream()
.sorted(Map.Entry.<NlpTokenDto, Long>comparingByValue().reversed())
.limit(KartConstant.TOP_WORD_LIMIT)
.map(entry -> toWord(entry.getKey(), entry.getValue()))
.toList();
}
/**
* Sums the same text across mergeable tags. The merged word keeps the tag with the highest count
* (first appearance on ties) and the position of its first appearance.
*/
private LinkedHashMap<NlpTokenDto, Long> merge(
LinkedHashMap<NlpTokenDto, Long> counts, WordMergeMode mode) {
if (mode == WordMergeMode.NONE) {
return counts;
}
LinkedHashMap<NlpTokenDto, LinkedHashMap<String, Long>> groups = new LinkedHashMap<>();
for (Map.Entry<NlpTokenDto, Long> entry : counts.entrySet()) {
NlpTokenDto token = entry.getKey();
boolean mergeable = mode == WordMergeMode.ALL || NOUN_FAMILY_TAGS.contains(token.tag());
NlpTokenDto key = mergeable ? new NlpTokenDto(token.text(), null) : token;
groups.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).put(token.tag(), entry.getValue());
}
LinkedHashMap<NlpTokenDto, Long> result = new LinkedHashMap<>();
for (Map.Entry<NlpTokenDto, LinkedHashMap<String, Long>> group : groups.entrySet()) {
String tag = null;
long best = 0;
long total = 0;
for (Map.Entry<String, Long> tagCount : group.getValue().entrySet()) {
total += tagCount.getValue();
if (tag == null || tagCount.getValue() > best) {
tag = tagCount.getKey();
best = tagCount.getValue();
}
}
result.put(new NlpTokenDto(group.getKey().text(), tag), total);
}
return result;
}
/** 불용어 row: WORD_TAG is STOP, ignoring case and surrounding spaces. */
private boolean isStopwordRow(NlpDictionaryWordDto word) {
String tag = StringUtil.normalizeBlank(word.getWordTag());
return KartConstant.STOPWORD_TAG.equalsIgnoreCase(tag);
}
/** Stopword comparison key: trimmed, spaces collapsed and case-insensitive. */
private String stopwordKey(String text) {
String normalized = StringUtil.normalizeBlank(text);
return normalized == null
? null
: WHITESPACE.matcher(normalized).replaceAll(" ").toLowerCase(Locale.ROOT);
}
private StatisticsWordDto toWord(NlpTokenDto token, long count) {
return new StatisticsWordDto(token.text(), token.tag(), count);
}

View File

@@ -11,13 +11,14 @@
) TAGS
</select>
<select id="words" databaseId="kart" resultType="com.kri.kart.server.dto.statistics.NlpDictionaryWordDto">
SELECT DISTINCT WORD_TAG, WORD_TEXT, WORD_SCORE FROM (
SELECT WORD_TAG, WORD_TEXT, 10 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD WHERE PKEY = 'ALL'
SELECT DISTINCT WORD_TAG, WORD_TEXT, SOURCE_TEXT, WORD_SCORE FROM (
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 10 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
WHERE PKEY = 'ALL'
UNION
SELECT WORD_TAG, WORD_TEXT, 50 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 50 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
WHERE PKEY = #{pkey} AND QID = 'ALL'
UNION
SELECT WORD_TAG, WORD_TEXT, 100 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 100 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
WHERE PKEY = #{pkey} AND QID = #{qid}
) WORDS
</select>