261111_01_ENH: Kiwi 형태소 분석 정확도 개선 및 품사 합산·복합명사·치환·불용어 옵션 추가
This commit is contained in:
@@ -100,6 +100,8 @@ public final class KartConstant {
|
||||
// StatisticsTextService
|
||||
public static final int TOP_WORD_LIMIT = 100;
|
||||
public static final String FOREIGN_WORD_TAG = "SL";
|
||||
// 불용어: REPORT_NLP_WORD rows with this WORD_TAG are excluded from word counts.
|
||||
public static final String STOPWORD_TAG = "STOP";
|
||||
|
||||
// StatisticsController
|
||||
public static final String CROSSTAB_PAGE = "/Report/Crosstab";
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.kri.kart.server.common.enums;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
import com.kri.common.feature.enums.CodeEnum;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Getter(onMethod_ = @JsonValue)
|
||||
@RequiredArgsConstructor
|
||||
@Schema(description = "같은 단어의 품사별 빈도 합산: 1=모두 합치기, 2=명사 계열만 합치기, 3=합치지 않기")
|
||||
public enum WordMergeMode implements CodeEnum<Integer> {
|
||||
ALL(1),
|
||||
NOUN(2),
|
||||
NONE(3);
|
||||
|
||||
private final Integer code;
|
||||
|
||||
@JsonCreator(mode = JsonCreator.Mode.DELEGATING)
|
||||
public static WordMergeMode fromCode(Integer code) {
|
||||
if (code == null) {
|
||||
return null;
|
||||
}
|
||||
for (WordMergeMode s : values()) {
|
||||
if (s.code.equals(code)) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown WordMergeMode code: " + code);
|
||||
}
|
||||
}
|
||||
@@ -3,10 +3,15 @@ package com.kri.kart.server.dto.statistics;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* REPORT_NLP_WORD row. WORD_TAG STOP marks a 불용어 (stopword); a row with SOURCE_TEXT is a synonym
|
||||
* that replaces SOURCE_TEXT with WORD_TEXT/WORD_TAG; any other row is a Kiwi user word.
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class NlpDictionaryWordDto {
|
||||
private String wordText;
|
||||
private String wordTag;
|
||||
private Integer wordScore;
|
||||
private String sourceText;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.kri.kart.server.dto.statistics;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/** Text-analysis replacement such as 엘지전자 → LG전자, with 1=ALL, 2=project, 3=question scope. */
|
||||
@Getter
|
||||
@Setter
|
||||
public class NlpSynonymDto {
|
||||
private String sourceText;
|
||||
private String targetText;
|
||||
private String targetTag;
|
||||
private Integer scopeRank;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.kri.kart.server.dto.statistics;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.kri.kart.server.common.enums.WordMergeMode;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Max;
|
||||
@@ -55,4 +56,13 @@ public class StatisticsOptionsDto {
|
||||
private boolean showNumericSum;
|
||||
|
||||
@Valid @NotNull private List<StatisticsValueGroupDto> valueGroups = new ArrayList<>();
|
||||
|
||||
@NotNull
|
||||
@Schema(description = "텍스트 분석에서 같은 단어의 품사별 빈도 합산 방식. 생략하면 2(명사 계열만 합치기)")
|
||||
private WordMergeMode wordMergeMode = WordMergeMode.NOUN;
|
||||
|
||||
@Min(1)
|
||||
@Max(5)
|
||||
@Schema(description = "텍스트 분석에서 공백 없이 붙은 명사를 합칠 최대 토큰 수. 1이면 합치지 않음. 생략하면 3")
|
||||
private int compoundNounMaxTokens = 3;
|
||||
}
|
||||
|
||||
@@ -249,7 +249,13 @@ public class StatisticsService {
|
||||
.columns(List.of())
|
||||
.headerRows(List.of())
|
||||
.rows(List.of())
|
||||
.words(statisticsTextService.analyze(project, index.getDimensionId(), responses))
|
||||
.words(
|
||||
statisticsTextService.analyze(
|
||||
project,
|
||||
index.getDimensionId(),
|
||||
responses,
|
||||
request.getWordMergeMode(),
|
||||
request.getCompoundNounMaxTokens()))
|
||||
.appliedOptions(request)
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -1,22 +1,39 @@
|
||||
package com.kri.kart.server.service.statistics;
|
||||
|
||||
import com.kri.common.feature.util.StringUtil;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto;
|
||||
import com.kri.kart.server.dto.statistics.NlpTokenDto;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import kr.pe.bab2min.Kiwi;
|
||||
import kr.pe.bab2min.KiwiBuilder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KiwiNlpService {
|
||||
// Same as kiwipiepy's default typo cost threshold.
|
||||
private static final float TYPO_COST_THRESHOLD = 2.5f;
|
||||
private static final Set<String> COMPOUND_NOUN_TAGS = Set.of("NNG", "NNP", "SL");
|
||||
private static final Set<String> NOUN_AFFIX_TAGS = Set.of("SL", "SH", "XPN", "XSN");
|
||||
// Above every user dictionary score (10/50/100) so a synonym always wins.
|
||||
private static final float SYNONYM_SCORE = 200f;
|
||||
private static final Pattern WHITESPACE = Pattern.compile("\\s+");
|
||||
|
||||
private final Path modelDirectory;
|
||||
|
||||
public KiwiNlpService(
|
||||
@@ -24,34 +41,76 @@ public class KiwiNlpService {
|
||||
this.modelDirectory = Path.of(modelDirectory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps the existing behavior: synonym rows (with SOURCE_TEXT) are ignored and there is no
|
||||
* compound-noun or noun-affix joining.
|
||||
*/
|
||||
public List<NlpTokenDto> tokenize(List<String> texts, List<NlpDictionaryWordDto> dictionary) {
|
||||
List<NlpDictionaryWordDto> words =
|
||||
dictionary.stream().filter(word -> !isSynonymRow(word)).toList();
|
||||
return tokenize(texts, words, List.of(), 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenizes texts with REPORT_NLP_WORD rows: a row with SOURCE_TEXT is a synonym that replaces
|
||||
* SOURCE_TEXT with WORD_TEXT/WORD_TAG before anything else, any other row is a user word. When
|
||||
* {@code compoundNounMaxTokens} is 2 or more, noun prefixes/suffixes are joined and adjacent
|
||||
* nouns are merged into one compound noun (see {@link #joinCompoundNouns}).
|
||||
*/
|
||||
public List<NlpTokenDto> tokenize(
|
||||
List<String> texts, List<NlpDictionaryWordDto> dictionary, int compoundNounMaxTokens) {
|
||||
List<NlpDictionaryWordDto> words = new ArrayList<>();
|
||||
List<NlpDictionaryWordDto> synonyms = new ArrayList<>();
|
||||
for (NlpDictionaryWordDto word : dictionary) {
|
||||
(isSynonymRow(word) ? synonyms : words).add(word);
|
||||
}
|
||||
return tokenize(texts, words, synonyms, compoundNounMaxTokens);
|
||||
}
|
||||
|
||||
private static boolean isSynonymRow(NlpDictionaryWordDto word) {
|
||||
return StringUtil.normalizeBlank(word.getSourceText()) != null;
|
||||
}
|
||||
|
||||
private List<NlpTokenDto> tokenize(
|
||||
List<String> texts,
|
||||
List<NlpDictionaryWordDto> dictionary,
|
||||
List<NlpDictionaryWordDto> synonyms,
|
||||
int compoundNounMaxTokens) {
|
||||
if (texts.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
if (!Files.isRegularFile(modelDirectory.resolve("cong.mdl"))) {
|
||||
throw unavailable();
|
||||
}
|
||||
Map<String, Byte> tags = tags();
|
||||
List<NlpTokenDto> result = new ArrayList<>();
|
||||
int options = KiwiBuilder.BuildOption.loadDefaultDict | KiwiBuilder.BuildOption.loadTypoDict;
|
||||
// Use the user-approved latest model; retain dictionary, allomorph and typo-processing rules.
|
||||
// Kiwi defaults: allomorph integration plus default, typo and multi-word dictionaries.
|
||||
int options = KiwiBuilder.BuildOption.default_;
|
||||
try (KiwiBuilder kiwiBuilder =
|
||||
new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong)) {
|
||||
for (NlpDictionaryWordDto word : dictionary) {
|
||||
Byte tag = tags.get(word.getWordTag());
|
||||
if (tag != null && word.getWordText() != null && word.getWordScore() != null) {
|
||||
try {
|
||||
kiwiBuilder.addWord(word.getWordText(), tag, word.getWordScore());
|
||||
} catch (RuntimeException ignored) {
|
||||
// Existing dictionary registration ignores an individually invalid word.
|
||||
}
|
||||
}
|
||||
new KiwiBuilder(modelDirectory.toString(), 1, options, KiwiBuilder.ModelType.cong);
|
||||
KiwiBuilder.PreparedTypoTransformer typoTransformer =
|
||||
KiwiBuilder.basicTypoSetWithContinual.prepare()) {
|
||||
// User words and synonym targets are never merged with their neighbors.
|
||||
Set<DictionaryWord> userWords = registerWords(kiwiBuilder, dictionary);
|
||||
List<Synonym> spacedSynonyms = registerSynonyms(kiwiBuilder, synonyms, userWords);
|
||||
boolean joinNouns = compoundNounMaxTokens > 1;
|
||||
// Recognize URL/email/hashtag/mention/serial spans, normalize informal codas,
|
||||
// analyze unknown words with the character model and correct common typos.
|
||||
int match = Kiwi.Match.allWithNormalizing | Kiwi.Match.oovChrModel;
|
||||
if (joinNouns) {
|
||||
match |= Kiwi.Match.joinNounPrefix | Kiwi.Match.joinNounSuffix;
|
||||
}
|
||||
Kiwi.AnalyzeOption analyzeOption =
|
||||
new Kiwi.AnalyzeOption(
|
||||
match, null, (short) 0, 0f, typoTransformer, TYPO_COST_THRESHOLD);
|
||||
try (Kiwi kiwi = kiwiBuilder.build()) {
|
||||
for (String text : texts) {
|
||||
for (Kiwi.Token token : kiwi.tokenize(text, new Kiwi.AnalyzeOption(0))) {
|
||||
String tag = Kiwi.POSTag.toString(token.tag);
|
||||
result.add(new NlpTokenDto(token.form, tag == null ? "UNK" : tag));
|
||||
List<Kiwi.Token> tokens =
|
||||
replaceSpacedSynonyms(
|
||||
text, List.of(kiwi.tokenize(text, analyzeOption)), spacedSynonyms);
|
||||
if (joinNouns) {
|
||||
result.addAll(joinCompoundNouns(tokens, userWords, compoundNounMaxTokens));
|
||||
} else {
|
||||
tokens.forEach(token -> result.add(toToken(token)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +121,277 @@ public class KiwiNlpService {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers each word once with its highest scope score (10/50/100). Kiwi keeps the last score
|
||||
* of a duplicate registration, and the dictionary query has no defined order.
|
||||
*/
|
||||
private Set<DictionaryWord> registerWords(
|
||||
KiwiBuilder kiwiBuilder, List<NlpDictionaryWordDto> dictionary) {
|
||||
Map<String, Byte> tags = tags();
|
||||
Map<DictionaryWord, Integer> scores = new LinkedHashMap<>();
|
||||
int skipped = 0;
|
||||
for (NlpDictionaryWordDto word : dictionary) {
|
||||
String text = StringUtil.normalizeBlank(word.getWordText());
|
||||
String tagName = StringUtil.normalizeBlank(word.getWordTag());
|
||||
Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT));
|
||||
if (text == null || tag == null || word.getWordScore() == null) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
scores.merge(new DictionaryWord(text, tag), word.getWordScore(), Math::max);
|
||||
}
|
||||
int rejected = 0;
|
||||
for (Map.Entry<DictionaryWord, Integer> entry : scores.entrySet()) {
|
||||
try {
|
||||
if (!kiwiBuilder.addWord(entry.getKey().text(), entry.getKey().tag(), entry.getValue())) {
|
||||
rejected++;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// The native call throws checked exceptions; an invalid word does not stop the analysis.
|
||||
rejected++;
|
||||
}
|
||||
}
|
||||
if (skipped > 0 || rejected > 0) {
|
||||
// Counts only: never log dictionary words.
|
||||
log.warn(
|
||||
"Kiwi user dictionary: registered={}, skipped={}, rejected={}",
|
||||
scores.size() - rejected,
|
||||
skipped,
|
||||
rejected);
|
||||
}
|
||||
return new HashSet<>(scores.keySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers synonym rows above every user word: SOURCE_TEXT is analyzed directly as
|
||||
* WORD_TEXT/WORD_TAG, and a spaced source also registers its spaceless form. The most specific
|
||||
* scope (score 100 > 50 > 10) wins for the same source, and an explicit spaceless source wins over
|
||||
* a derived one. Returns spaced sources, longest first, for {@link #replaceSpacedSynonyms}; adds
|
||||
* every target to {@code protectedWords}.
|
||||
*/
|
||||
private List<Synonym> registerSynonyms(
|
||||
KiwiBuilder kiwiBuilder,
|
||||
List<NlpDictionaryWordDto> synonyms,
|
||||
Set<DictionaryWord> protectedWords) {
|
||||
Map<String, Byte> tags = tags();
|
||||
// Targets already registered as plain user words keep their scope score.
|
||||
Set<DictionaryWord> registeredTargets = new HashSet<>(protectedWords);
|
||||
Map<String, Synonym> sources = new LinkedHashMap<>();
|
||||
Map<String, Integer> sourceRanks = new HashMap<>();
|
||||
int skipped = 0;
|
||||
for (NlpDictionaryWordDto synonym : synonyms) {
|
||||
String source = normalizeSpaces(synonym.getSourceText());
|
||||
String target = StringUtil.normalizeBlank(synonym.getWordText());
|
||||
String tagName = StringUtil.normalizeBlank(synonym.getWordTag());
|
||||
Byte tag = tagName == null ? null : tags.get(tagName.toUpperCase(Locale.ROOT));
|
||||
if (source == null || target == null || tag == null) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
int rank = synonym.getWordScore() == null ? 0 : synonym.getWordScore();
|
||||
if (!sources.containsKey(source) || rank > sourceRanks.get(source)) {
|
||||
sources.put(source, new Synonym(source, target, tag));
|
||||
sourceRanks.put(source, rank);
|
||||
}
|
||||
}
|
||||
Map<String, Synonym> attached = new LinkedHashMap<>();
|
||||
Map<String, Integer> attachedWeights = new HashMap<>();
|
||||
List<Synonym> spaced = new ArrayList<>();
|
||||
for (Synonym synonym : sources.values()) {
|
||||
boolean explicit = !synonym.source().contains(" ");
|
||||
String form = synonym.source().replace(" ", "");
|
||||
int weight = sourceRanks.get(synonym.source()) * 2 + (explicit ? 1 : 0);
|
||||
Integer current = attachedWeights.get(form);
|
||||
if (current == null || weight > current) {
|
||||
attached.put(form, new Synonym(form, synonym.target(), synonym.tag()));
|
||||
attachedWeights.put(form, weight);
|
||||
}
|
||||
if (!explicit) {
|
||||
spaced.add(synonym);
|
||||
}
|
||||
protectedWords.add(new DictionaryWord(synonym.target(), synonym.tag()));
|
||||
}
|
||||
int rejected = 0;
|
||||
for (Synonym synonym : attached.values()) {
|
||||
KiwiBuilder.AnalyzedMorph[] morphs = {
|
||||
new KiwiBuilder.AnalyzedMorph(
|
||||
synonym.target(), synonym.tag(), 0, synonym.source().length())
|
||||
};
|
||||
try {
|
||||
// Kiwi requires the target morpheme to exist, e.g. "LG" is not in the default dictionary.
|
||||
// Score 0 keeps a short target from being forced inside longer words.
|
||||
if (registeredTargets.add(new DictionaryWord(synonym.target(), synonym.tag()))) {
|
||||
kiwiBuilder.addWord(synonym.target(), synonym.tag(), 0f);
|
||||
}
|
||||
if (!kiwiBuilder.addPreAnalyzedWord(synonym.source(), morphs, SYNONYM_SCORE)) {
|
||||
rejected++;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// The native call throws checked exceptions; an invalid synonym does not stop the analysis.
|
||||
rejected++;
|
||||
}
|
||||
}
|
||||
if (skipped > 0 || rejected > 0) {
|
||||
// Counts only: never log synonym texts.
|
||||
log.warn(
|
||||
"Kiwi synonyms: registered={}, skipped={}, rejected={}",
|
||||
attached.size() - rejected,
|
||||
skipped,
|
||||
rejected);
|
||||
}
|
||||
spaced.sort(Comparator.comparingInt((Synonym synonym) -> synonym.source().length()).reversed());
|
||||
return spaced;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces a run of tokens whose original text, with spaces collapsed, equals a spaced synonym
|
||||
* source. The run must start and end on token boundaries and must not continue into a noun of
|
||||
* the same word, so "엘지 전자레인지" (엘지/전자/레인지) is not "엘지 전자".
|
||||
*/
|
||||
static List<Kiwi.Token> replaceSpacedSynonyms(
|
||||
String text, List<Kiwi.Token> tokens, List<Synonym> spacedSynonyms) {
|
||||
if (spacedSynonyms.isEmpty()) {
|
||||
return tokens;
|
||||
}
|
||||
List<Kiwi.Token> result = new ArrayList<>();
|
||||
int start = 0;
|
||||
while (start < tokens.size()) {
|
||||
int end = -1;
|
||||
Synonym matched = null;
|
||||
for (Synonym synonym : spacedSynonyms) {
|
||||
end = matchEnd(text, tokens, start, synonym.source());
|
||||
if (end > start && !continuesNoun(tokens, start, end)) {
|
||||
matched = synonym;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matched == null) {
|
||||
result.add(tokens.get(start));
|
||||
start++;
|
||||
continue;
|
||||
}
|
||||
Kiwi.Token first = tokens.get(start);
|
||||
Kiwi.Token last = tokens.get(end - 1);
|
||||
Kiwi.Token replaced = new Kiwi.Token();
|
||||
replaced.form = matched.target();
|
||||
replaced.tag = matched.tag();
|
||||
replaced.position = first.position;
|
||||
replaced.length = (short) (last.position + last.length - first.position);
|
||||
replaced.wordPosition = first.wordPosition;
|
||||
replaced.sentPosition = first.sentPosition;
|
||||
replaced.lineNumber = first.lineNumber;
|
||||
result.add(replaced);
|
||||
start = end;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Whether a noun is attached right before or after the run inside the same word. */
|
||||
private static boolean continuesNoun(List<Kiwi.Token> tokens, int start, int end) {
|
||||
boolean before =
|
||||
start > 0
|
||||
&& isAttached(tokens.get(start - 1), tokens.get(start))
|
||||
&& isNounLike(tokens.get(start - 1));
|
||||
boolean after =
|
||||
end < tokens.size()
|
||||
&& isAttached(tokens.get(end - 1), tokens.get(end))
|
||||
&& isNounLike(tokens.get(end));
|
||||
return before || after;
|
||||
}
|
||||
|
||||
private static boolean isNounLike(Kiwi.Token token) {
|
||||
String tag = tagName(token);
|
||||
return tag.startsWith("N") || NOUN_AFFIX_TAGS.contains(tag);
|
||||
}
|
||||
|
||||
/** Exclusive end index of the run starting at {@code start} that spells {@code source}, or -1. */
|
||||
private static int matchEnd(String text, List<Kiwi.Token> tokens, int start, String source) {
|
||||
int from = tokens.get(start).position;
|
||||
for (int end = start; end < tokens.size(); end++) {
|
||||
Kiwi.Token last = tokens.get(end);
|
||||
int to = last.position + last.length;
|
||||
if (from < 0 || to > text.length() || to < from) {
|
||||
return -1;
|
||||
}
|
||||
String span = WHITESPACE.matcher(text.substring(from, to)).replaceAll(" ");
|
||||
if (span.equals(source)) {
|
||||
return end + 1;
|
||||
}
|
||||
if (span.length() >= source.length()) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static String normalizeSpaces(String raw) {
|
||||
String normalized = StringUtil.normalizeBlank(raw);
|
||||
return normalized == null ? null : WHITESPACE.matcher(normalized).replaceAll(" ");
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges a run of 2..maxTokens NNG/NNP/SL tokens written without a space inside one word into a
|
||||
* single noun, when the run contains a Korean noun. The merged tag is NNP if any part is NNP,
|
||||
* otherwise NNG. User dictionary words always stay as they are, and longer runs are kept apart.
|
||||
*/
|
||||
static List<NlpTokenDto> joinCompoundNouns(
|
||||
List<Kiwi.Token> tokens, Set<DictionaryWord> userWords, int maxTokens) {
|
||||
List<NlpTokenDto> result = new ArrayList<>();
|
||||
int start = 0;
|
||||
while (start < tokens.size()) {
|
||||
if (!isCompoundPart(tokens.get(start), userWords)) {
|
||||
result.add(toToken(tokens.get(start)));
|
||||
start++;
|
||||
continue;
|
||||
}
|
||||
int end = start + 1;
|
||||
while (end < tokens.size()
|
||||
&& isCompoundPart(tokens.get(end), userWords)
|
||||
&& isAttached(tokens.get(end - 1), tokens.get(end))) {
|
||||
end++;
|
||||
}
|
||||
List<Kiwi.Token> run = tokens.subList(start, end);
|
||||
boolean hasKoreanNoun = run.stream().anyMatch(token -> !isForeignWord(token));
|
||||
if (run.size() > 1 && run.size() <= maxTokens && hasKoreanNoun) {
|
||||
StringBuilder form = new StringBuilder();
|
||||
boolean properNoun = false;
|
||||
for (Kiwi.Token token : run) {
|
||||
form.append(token.form);
|
||||
properNoun |= "NNP".equals(tagName(token));
|
||||
}
|
||||
result.add(new NlpTokenDto(form.toString(), properNoun ? "NNP" : "NNG"));
|
||||
} else {
|
||||
run.forEach(token -> result.add(toToken(token)));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static boolean isCompoundPart(Kiwi.Token token, Set<DictionaryWord> userWords) {
|
||||
return COMPOUND_NOUN_TAGS.contains(tagName(token))
|
||||
&& !userWords.contains(new DictionaryWord(token.form, token.tag));
|
||||
}
|
||||
|
||||
private static boolean isForeignWord(Kiwi.Token token) {
|
||||
return "SL".equals(tagName(token));
|
||||
}
|
||||
|
||||
/** Same word and no space or other character between the two tokens. */
|
||||
private static boolean isAttached(Kiwi.Token previous, Kiwi.Token next) {
|
||||
return previous.wordPosition == next.wordPosition
|
||||
&& previous.position + previous.length == next.position;
|
||||
}
|
||||
|
||||
private static NlpTokenDto toToken(Kiwi.Token token) {
|
||||
return new NlpTokenDto(token.form, tagName(token));
|
||||
}
|
||||
|
||||
private static String tagName(Kiwi.Token token) {
|
||||
String tag = Kiwi.POSTag.toString(token.tag);
|
||||
return tag == null ? "UNK" : tag;
|
||||
}
|
||||
|
||||
private Map<String, Byte> tags() {
|
||||
Map<String, Byte> result = new LinkedHashMap<>();
|
||||
for (int code = Byte.MIN_VALUE; code <= Byte.MAX_VALUE; code++) {
|
||||
@@ -77,4 +407,8 @@ public class KiwiNlpService {
|
||||
return BusinessException.of(
|
||||
HttpStatus.SERVICE_UNAVAILABLE, "KIWI_NLP_UNAVAILABLE", "Kiwi NLP 라이브러리와 모델 설정을 확인해 주세요.");
|
||||
}
|
||||
|
||||
record DictionaryWord(String text, byte tag) {}
|
||||
|
||||
record Synonym(String source, String target, byte tag) {}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
package com.kri.kart.server.service.statistics;
|
||||
|
||||
import com.kri.common.feature.util.StringUtil;
|
||||
import com.kri.kart.server.common.consts.KartConstant;
|
||||
import com.kri.kart.server.common.enums.WordMergeMode;
|
||||
import com.kri.kart.server.dto.project.ProjectRefDto;
|
||||
import com.kri.kart.server.dto.statistics.NlpDictionaryWordDto;
|
||||
import com.kri.kart.server.dto.statistics.NlpTokenDto;
|
||||
import com.kri.kart.server.dto.statistics.StatisticsWordDto;
|
||||
import com.kri.kart.server.dto.statistics.SurveyResponseDto;
|
||||
import com.kri.kart.server.persistence.repository.NlpRepository;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.regex.Pattern;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -24,31 +30,128 @@ import org.springframework.stereotype.Service;
|
||||
@ConditionalOnProperty(prefix = "kri.kart.database", name = "enabled", havingValue = "true")
|
||||
public class StatisticsTextService {
|
||||
|
||||
// Noun family for WordMergeMode.NOUN: nouns, foreign words and Chinese characters.
|
||||
private static final Set<String> NOUN_FAMILY_TAGS =
|
||||
Set.of("NNG", "NNP", "NNB", "NR", "NP", "SL", "SH");
|
||||
private static final Pattern WHITESPACE = Pattern.compile("\\s+");
|
||||
|
||||
private final NlpRepository nlpRepository;
|
||||
private final KiwiNlpService kiwiNlpService;
|
||||
private final StatisticsSourceService statisticsSourceService;
|
||||
|
||||
/**
|
||||
* Keeps the existing behavior: no synonyms or stopwords, the same word with a different tag is
|
||||
* counted separately and compound nouns are not merged.
|
||||
*/
|
||||
public List<StatisticsWordDto> analyze(
|
||||
ProjectRefDto project, String qid, List<SurveyResponseDto> responses) {
|
||||
return analyze(project, qid, responses, false, WordMergeMode.NONE, 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the REPORT_NLP_WORD synonyms (rows with SOURCE_TEXT), the stopwords (불용어, rows
|
||||
* tagged STOP) and the given merge options.
|
||||
*/
|
||||
public List<StatisticsWordDto> analyze(
|
||||
ProjectRefDto project,
|
||||
String qid,
|
||||
List<SurveyResponseDto> responses,
|
||||
WordMergeMode mode,
|
||||
int compoundNounMaxTokens) {
|
||||
return analyze(project, qid, responses, true, mode, compoundNounMaxTokens);
|
||||
}
|
||||
|
||||
private List<StatisticsWordDto> analyze(
|
||||
ProjectRefDto project,
|
||||
String qid,
|
||||
List<SurveyResponseDto> responses,
|
||||
boolean extended,
|
||||
WordMergeMode mode,
|
||||
int compoundNounMaxTokens) {
|
||||
List<String> texts = answerTexts(qid, responses);
|
||||
Set<String> selectedTags = new LinkedHashSet<>(nlpRepository.tags(project.pkey(), qid));
|
||||
// 불용어 (STOP) rows are not dictionary words: they are never registered in Kiwi.
|
||||
List<NlpDictionaryWordDto> dictionary = new ArrayList<>();
|
||||
Set<String> stopwords = new HashSet<>();
|
||||
for (NlpDictionaryWordDto word : nlpRepository.words(project.pkey(), qid)) {
|
||||
if (isStopwordRow(word)) {
|
||||
String stopword = stopwordKey(word.getWordText());
|
||||
if (stopword != null) {
|
||||
stopwords.add(stopword);
|
||||
}
|
||||
} else {
|
||||
dictionary.add(word);
|
||||
}
|
||||
}
|
||||
// The original overload ignores synonym rows and does not merge compound nouns.
|
||||
List<NlpTokenDto> tokens =
|
||||
kiwiNlpService.tokenize(texts, nlpRepository.words(project.pkey(), qid));
|
||||
// The same word with a different tag is counted separately, in first-appearance order.
|
||||
extended
|
||||
? kiwiNlpService.tokenize(texts, dictionary, compoundNounMaxTokens)
|
||||
: kiwiNlpService.tokenize(texts, dictionary);
|
||||
// Count each word and tag pair in first-appearance order.
|
||||
LinkedHashMap<NlpTokenDto, Long> counts = new LinkedHashMap<>();
|
||||
for (NlpTokenDto token : tokens) {
|
||||
if (isCounted(token, selectedTags)) {
|
||||
// Stopwords apply to the final word after synonyms and compound merging, for every tag.
|
||||
boolean stopword = extended && stopwords.contains(stopwordKey(token.text()));
|
||||
if (!stopword && isCounted(token, selectedTags)) {
|
||||
counts.merge(token, 1L, Long::sum);
|
||||
}
|
||||
}
|
||||
// Stable sort: equal counts keep their first-appearance order.
|
||||
return counts.entrySet().stream()
|
||||
return merge(counts, mode == null ? WordMergeMode.NONE : mode).entrySet().stream()
|
||||
.sorted(Map.Entry.<NlpTokenDto, Long>comparingByValue().reversed())
|
||||
.limit(KartConstant.TOP_WORD_LIMIT)
|
||||
.map(entry -> toWord(entry.getKey(), entry.getValue()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sums the same text across mergeable tags. The merged word keeps the tag with the highest count
|
||||
* (first appearance on ties) and the position of its first appearance.
|
||||
*/
|
||||
private LinkedHashMap<NlpTokenDto, Long> merge(
|
||||
LinkedHashMap<NlpTokenDto, Long> counts, WordMergeMode mode) {
|
||||
if (mode == WordMergeMode.NONE) {
|
||||
return counts;
|
||||
}
|
||||
LinkedHashMap<NlpTokenDto, LinkedHashMap<String, Long>> groups = new LinkedHashMap<>();
|
||||
for (Map.Entry<NlpTokenDto, Long> entry : counts.entrySet()) {
|
||||
NlpTokenDto token = entry.getKey();
|
||||
boolean mergeable = mode == WordMergeMode.ALL || NOUN_FAMILY_TAGS.contains(token.tag());
|
||||
NlpTokenDto key = mergeable ? new NlpTokenDto(token.text(), null) : token;
|
||||
groups.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).put(token.tag(), entry.getValue());
|
||||
}
|
||||
LinkedHashMap<NlpTokenDto, Long> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<NlpTokenDto, LinkedHashMap<String, Long>> group : groups.entrySet()) {
|
||||
String tag = null;
|
||||
long best = 0;
|
||||
long total = 0;
|
||||
for (Map.Entry<String, Long> tagCount : group.getValue().entrySet()) {
|
||||
total += tagCount.getValue();
|
||||
if (tag == null || tagCount.getValue() > best) {
|
||||
tag = tagCount.getKey();
|
||||
best = tagCount.getValue();
|
||||
}
|
||||
}
|
||||
result.put(new NlpTokenDto(group.getKey().text(), tag), total);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 불용어 row: WORD_TAG is STOP, ignoring case and surrounding spaces. */
|
||||
private boolean isStopwordRow(NlpDictionaryWordDto word) {
|
||||
String tag = StringUtil.normalizeBlank(word.getWordTag());
|
||||
return KartConstant.STOPWORD_TAG.equalsIgnoreCase(tag);
|
||||
}
|
||||
|
||||
/** Stopword comparison key: trimmed, spaces collapsed and case-insensitive. */
|
||||
private String stopwordKey(String text) {
|
||||
String normalized = StringUtil.normalizeBlank(text);
|
||||
return normalized == null
|
||||
? null
|
||||
: WHITESPACE.matcher(normalized).replaceAll(" ").toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private StatisticsWordDto toWord(NlpTokenDto token, long count) {
|
||||
return new StatisticsWordDto(token.text(), token.tag(), count);
|
||||
}
|
||||
|
||||
@@ -11,13 +11,14 @@
|
||||
) TAGS
|
||||
</select>
|
||||
<select id="words" databaseId="kart" resultType="com.kri.kart.server.dto.statistics.NlpDictionaryWordDto">
|
||||
SELECT DISTINCT WORD_TAG, WORD_TEXT, WORD_SCORE FROM (
|
||||
SELECT WORD_TAG, WORD_TEXT, 10 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD WHERE PKEY = 'ALL'
|
||||
SELECT DISTINCT WORD_TAG, WORD_TEXT, SOURCE_TEXT, WORD_SCORE FROM (
|
||||
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 10 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
|
||||
WHERE PKEY = 'ALL'
|
||||
UNION
|
||||
SELECT WORD_TAG, WORD_TEXT, 50 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
|
||||
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 50 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
|
||||
WHERE PKEY = #{pkey} AND QID = 'ALL'
|
||||
UNION
|
||||
SELECT WORD_TAG, WORD_TEXT, 100 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
|
||||
SELECT WORD_TAG, WORD_TEXT, SOURCE_TEXT, 100 AS WORD_SCORE FROM dbo.REPORT_NLP_WORD
|
||||
WHERE PKEY = #{pkey} AND QID = #{qid}
|
||||
) WORDS
|
||||
</select>
|
||||
|
||||
Reference in New Issue
Block a user