Files
KART_KON/REPORT_API_ver6/Services/NLP.cs
2025-10-24 10:25:54 +09:00

366 lines
12 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Newtonsoft.Json.Linq;
using System.Collections.ObjectModel;
namespace REPORT_API_ver6.Services
{
public class NLP
{
static readonly log4net.ILog logger = log4net.LogManager.GetLogger(typeof(NLP));
public class AnalyzeResult
{
public int Id { get; set; }
public string Input { get; set; }
public List<List<Token>> Result { get; set; }
public AnalyzeResult(int Id, string Input, List<List<Token>> Result)
{
this.Id = Id;
this.Input = Input;
this.Result = Result;
}
public AnalyzeResult(int Id, string Input, List<Token> Result)
{
this.Id = Id;
this.Input = Input;
this.Result = new List<List<Token>>();
this.Result.Add(Result);
}
}
public class WordTag
{
public string Text { get; set; } = string.Empty;
public string Tag { get; set; } = string.Empty;
public WordTag(string inText, string inTag)
{
Text = inText;
Tag = inTag;
}
}
public class WordCloudTag
{
public string Text { get; set; } = string.Empty;
//public string Tag { get; set; } = string.Empty;
public string Count { get; set; } = string.Empty;
public WordCloudTag(string inText, string inTag, string inCount)
{
Text = inText;
//Tag = inTag;
Count = inCount;
}
}
public string getWordTag(string pkey, string qid, JArray dataJson)
{
string jsonData = String.Empty;
try
{
List<string> keyList = dataJson.Children<JObject>().Properties().Select(x => x.Name).Distinct().Where(x => x.StartsWith(qid + "_") || x == qid).ToList<string>();
List<string> textDataList = new List<string>();
foreach (var key in keyList)
{
textDataList.AddRange(dataJson.Where(x => x.Value<string>(key) != null).Select(x => Common.StripHTMLChracters(x.Value<string>(key).Replace("¸", ","))).ToList<string>());
}
List<AnalyzeResult> resultData = WordNLP(textDataList, pkey, qid);
string[] tags = getTags(pkey, qid);
//var positive = resultData.Select(x => x.Result.Where(y => y.Where(z => z.tag == "VCP").Count() > 0).ToList()).Where(x => x.Count > 0).ToList();
//var negative = resultData.Select(x => x.Result.Where(y => y.Where(z => z.tag == "VCN").Count() > 0).ToList()).Where(x => x.Count > 0).ToList();
//////여기 손봐서 업그레이드 해야함
List<WordTag> wordList =new List<WordTag>();
foreach (AnalyzeResult result in resultData)
{
foreach(List<Token> tokenList in result.Result)
{
foreach (Token token in tokenList)
{
if(Array.IndexOf(tags, token.tag) > -1 && (token.tag.StartsWith("N") || token.form.Length > 1) || token.tag == "SL")
{
wordList.Add(new WordTag(token.form, token.tag));
}
}
}
}
//////여기 손봐서 업그레이드 해야함
List<WordCloudTag> cloudTagList = makeTagTable(wordList, 100);
cloudTagList.Insert(0, new WordCloudTag("Text", "Tag", "Count"));
jsonData = Common.GetJsonFromObject(cloudTagList);
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
return jsonData;
}
public List<AnalyzeResult> WordNLP(List<string> textDataList, string pkey = "", string qid = "")
{
ObservableCollection<AnalyzeResult> resultData = new ObservableCollection<AnalyzeResult>();//결과???
try
{
Kiwi instKiwi;//키위 라이브러리
ModelType modelType = ModelType.SBG;//분석 모델(근거리 관계까지 추론 - 느림)
//KiwiCS.ModelType modelType = KiwiCS.ModelType.KNLM;//분석 모델(근거리 관계만 추론 - 빠름)
bool useTypoCorrection = false;//오타 수정(오래 걸림 잘 안되는듯?)
var builder = new KiwiBuilder(Common.MapPath("nlp_model/"), 0, Option.LoadDefaultDict | Option.LoadTypoDict, modelType);
builder = AddWords(builder, pkey, qid); ////단어 추가
if (useTypoCorrection)
{
instKiwi = builder.Build(new TypoTransformer(true));////오타 수정 세팅
}
else
{
instKiwi = builder.Build();////오타 수정 세팅
}
int mode = 1;//어절:0/문장:1/줄:2/전체:3
int topN = 1;
instKiwi.IntegrateAllomorph = false;////이형태 통합
////아/어, 았/었 등의 이형태를 통합할 지 선택합니다.
MatchWord match = 0;
if (false) match |= MatchWord.NormalizeCoda;////종성우선 현상 정규화
////'먹었엌ㅋㅋ'처럼 연속하는 종성이 따라붙어 오분석이 나는 경우를 정규화합니다.
if (false) match |= MatchWord.Url;////url을 포함
if (false) match |= MatchWord.Email;////이메일을 포함
if (false) match |= MatchWord.Hashtag;////해시태그를 포함(#)
if (false) match |= MatchWord.Mention;////맨션을 포함(@+영어)
if (false) match |= MatchWord.Serial;////시리얼(일련번호)
if (false) match |= MatchWord.JoinNounPrefix;////명사+접두사
if (false) match |= MatchWord.JoinNounSuffix;////명사+접두사
if (false) match |= MatchWord.JoinVerbSuffix;////동사 파생접사
if (false) match |= MatchWord.JoinAdjSuffix;////형용사 파생 접사
foreach (string textData in textDataList)
{
foreach (var r in AnalyzeText(instKiwi, textData, mode, match, topN))
{
resultData.Add(r);
}
}
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
return resultData.ToList();
}
private KiwiBuilder AddWords(KiwiBuilder builder, string pkey, string qid)
{
//builder.AddWord("사랑", "NNP", 90);//// 이런식으로 추가
DataTable addWordTable = getAddWords(pkey, qid);
foreach(DataRow addWordRow in addWordTable.Rows)
{
try
{
builder.AddWord(addWordRow["WORD_TEXT"].ToString(), addWordRow["WORD_TAG"].ToString(), Convert.ToInt32(addWordRow["WORD_SCORE"]));
}
catch(Exception ex)
{
}
}
// <단일 형태소를 추가하는 경우>
// (형태) \t (품사태그) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 키위 NNP -5.0
//
// <이미 존재하는 형태소의 이형태를 추가하는 경우>
// (이형태) \t (원형태소/품사태그) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 기위 키위/NNG -3.0
//
// <기분석 형태를 추가하는 경우>
// (형태) \t (원형태소/품사태그 + 원형태소/품사태그 + ...) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 사겼다 사귀/VV + 었/EP + 다/EF -1.0
//
// 현재는 공백을 포함하는 다어절 형태를 등록할 수 없습니다.
//
// <규칙 기반의 변형된 이형태를 추가하는 경우>
// (형태 규칙)$ \t (변형된 형태/품사태그) \t (점수)
// 예) 요$ 용/EF -5
return builder;
}
/// <summary>
/// 텍스트 분석기
/// </summary>
/// <param name="text">input</param>
/// <param name="mode">어절/문장/줄/전체</param>
/// <param name="match">옵션</param>
/// <param name="topN">결과 개수 몇개 까지?</param>
/// <returns>?</returns>
private IEnumerable<AnalyzeResult> AnalyzeText(Kiwi instKiwi, string text, int mode, MatchWord match, int topN = 1)
{
if (mode == 2)
{
int rid = 0;
foreach (var line in text.Trim().Split('\n'))
{
var trimmed = line.Trim();
var res = instKiwi.Analyze(trimmed, topN, match);
List<List<Token>> s = new List<List<Token>>();
foreach (var r in res)
{
if (r.morphs.Length == 0) continue;
s.Add(new List<Token>(r.morphs));
}
if (trimmed.Length == 0) continue;
yield return new AnalyzeResult(++rid, trimmed, s);
}
}
else if (mode == 3)
{
var res = instKiwi.Analyze(text, topN, match);
List<List<Token>> s = new List<List<Token>>();
foreach (var r in res)
{
if (r.morphs.Length == 0) continue;
s.Add(new List<Token>(r.morphs));
}
yield return new AnalyzeResult(1, text.Trim(), s);
}
else
{
var r = instKiwi.Analyze(text, 1, match)[0];
int wp = 0, sp = 0, cp = 0, rid = 0, c = 0;
List<Token> s = new List<Token>();
foreach (var m in r.morphs)
{
if (mode == 0 ? (wp != m.wordPosition || sp != m.sentPosition) : (sp != m.sentPosition))
{
yield return new AnalyzeResult(++rid, text.Substring(cp, (int)m.chrPosition - cp).Trim(), s);
cp = (int)m.chrPosition;
c = 0;
s = new List<Token>();
}
s.Add(m);
wp = (int)m.wordPosition;
sp = (int)m.sentPosition;
}
if (s.Count > 0)
{
yield return new AnalyzeResult(++rid, text.Substring(cp), s);
}
}
}
public string[] getTags(string pkey, string qid)
{
List<string> tags = new List<string>();
DAL dal = new DAL();
var connection = dal.ConnectReport();
try
{
string sql = string.Empty;
sql += " SELECT DISTINCT [WORD_TAG] " + Environment.NewLine;
sql += " FROM( " + Environment.NewLine;
sql += " SELECT[WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE PKEY = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT[WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE PKEY = '" + pkey + "' AND QID = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT[WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE PKEY = '" + pkey + "' AND QID = '" + qid + "' " + Environment.NewLine;
sql += " ) TAGS " + Environment.NewLine;
DataTable tagTable = dal.GetDataTable(sql, connection);
foreach (DataRow row in tagTable.Rows)
{
tags.Add(row["WORD_TAG"].ToString());
}
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
finally
{
dal.Disconnect(connection);
}
return tags.ToArray();
}
public DataTable getAddWords(string pkey, string qid)
{
DataTable addWordsTable = new DataTable();
DAL dal = new DAL();
var connection = dal.ConnectReport();
try
{
string sql = string.Empty;
sql += " SELECT DISTINCT [WORD_TAG], [WORD_TEXT], [WORD_SCORE] " + Environment.NewLine;
sql += " FROM( " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 10 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE PKEY = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 50 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE PKEY = '" + pkey + "' AND QID = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 100 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE PKEY = '" + pkey + "' AND QID = '" + qid + "' " + Environment.NewLine;
sql += " ) WORDS " + Environment.NewLine;
addWordsTable = dal.GetDataTable(sql, connection);
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
finally
{
dal.Disconnect(connection);
}
return addWordsTable;
}
/// <summary>
/// 단어 사전을 태그화 시키는 모듈
/// </summary>
/// <param name="wordDic">단어 사전</param>
/// <returns>단어, 카운트 테이블</returns>
public List<WordCloudTag> makeTagTable(List<WordTag> wordDic, int maxCount)
{
List<WordCloudTag> cloudTagList = new List<WordCloudTag>();
var distinctDic = wordDic.Select(x => new { x.Text, x.Tag}).Distinct().ToList();
foreach (var word in distinctDic)
{
int tagCount = wordDic.Where(x => x.Text == word.Text && x.Tag == word.Tag).Count();
cloudTagList.Add(new WordCloudTag(word.Text, word.Tag, tagCount.ToString()));
}
cloudTagList = cloudTagList.OrderByDescending(x => Convert.ToInt32(x.Count)).Take(maxCount).ToList();
return cloudTagList;
}
}
}