firts commit

This commit is contained in:
2025-10-24 10:25:29 +09:00
commit ec77d8bc29
49 changed files with 134191 additions and 0 deletions

View File

@@ -0,0 +1,367 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Newtonsoft.Json.Linq;
using System.Collections.ObjectModel;
using KART_API.Utils;
using Microsoft.IdentityModel.Tokens;
namespace KART_API.Services
{
public class NlpService
{
static readonly log4net.ILog logger = log4net.LogManager.GetLogger(typeof(NlpService));
public class AnalyzeResult
{
public int Id { get; set; }
public string Input { get; set; }
public List<List<Token>> Result { get; set; }
public AnalyzeResult(int Id, string Input, List<List<Token>> Result)
{
this.Id = Id;
this.Input = Input;
this.Result = Result;
}
public AnalyzeResult(int Id, string Input, List<Token> Result)
{
this.Id = Id;
this.Input = Input;
this.Result = new List<List<Token>>();
this.Result.Add(Result);
}
}
public class WordTag
{
public string Text { get; set; } = string.Empty;
public string Tag { get; set; } = string.Empty;
public WordTag(string inText, string inTag)
{
Text = inText;
Tag = inTag;
}
}
public class WordCloudTag
{
public string Text { get; set; } = string.Empty;
//public string Tag { get; set; } = string.Empty;
public string Count { get; set; } = string.Empty;
public WordCloudTag(string inText, string inTag, string inCount)
{
Text = inText;
//Tag = inTag;
Count = inCount;
}
}
public string getWordTag(string projectCode, string qid, JArray dataJson)
{
string jsonData = String.Empty;
try
{
List<string> textDataList = new List<string>();
var vArrayList = dataJson.Where(x => x.Value<JArray>(qid).IsNullOrEmpty() == false).Select(x => x.Value<JArray>(qid).Children<JObject>().Values<string>("o"));
foreach (var vArray in vArrayList)
{
textDataList.AddRange(vArray);
}
List<AnalyzeResult> resultData = WordNLP(textDataList, projectCode, qid);
string[] tags = getTags(projectCode, qid);
//var positive = resultData.Select(x => x.Result.Where(y => y.Where(z => z.tag == "VCP").Count() > 0).ToList()).Where(x => x.Count > 0).ToList();
//var negative = resultData.Select(x => x.Result.Where(y => y.Where(z => z.tag == "VCN").Count() > 0).ToList()).Where(x => x.Count > 0).ToList();
//////여기 손봐서 업그레이드 해야함
List<WordTag> wordList =new List<WordTag>();
foreach (AnalyzeResult result in resultData)
{
foreach(List<Token> tokenList in result.Result)
{
foreach (Token token in tokenList)
{
if(Array.IndexOf(tags, token.tag) > -1 && (token.tag.StartsWith("N") || token.form.Length > 1) || token.tag == "SL")
{
wordList.Add(new WordTag(token.form, token.tag));
}
}
}
}
//////여기 손봐서 업그레이드 해야함
List<WordCloudTag> cloudTagList = makeTagTable(wordList, 100);
cloudTagList.Insert(0, new WordCloudTag("Text", "Tag", "Count"));
jsonData = Common.GetJsonFromObject(cloudTagList);
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
return jsonData;
}
public List<AnalyzeResult> WordNLP(List<string> textDataList, string projectCode = "", string qid = "")
{
ObservableCollection<AnalyzeResult> resultData = new ObservableCollection<AnalyzeResult>();//결과???
try
{
Kiwi instKiwi;//키위 라이브러리
ModelType modelType = ModelType.SBG;//분석 모델(근거리 관계까지 추론 - 느림)
//KiwiCS.ModelType modelType = KiwiCS.ModelType.KNLM;//분석 모델(근거리 관계만 추론 - 빠름)
bool useTypoCorrection = false;//오타 수정(오래 걸림 잘 안되는듯?)
var builder = new KiwiBuilder(Common.MapPath("NlpModel/"), 0, Option.LoadDefaultDict | Option.LoadTypoDict, modelType);
builder = AddWords(builder, projectCode, qid); ////단어 추가
if (useTypoCorrection)
{
instKiwi = builder.Build(new TypoTransformer(true));////오타 수정 세팅
}
else
{
instKiwi = builder.Build();////오타 수정 세팅
}
int mode = 1;//어절:0/문장:1/줄:2/전체:3
int topN = 1;
instKiwi.IntegrateAllomorph = false;////이형태 통합
////아/어, 았/었 등의 이형태를 통합할 지 선택합니다.
MatchWord match = 0;
if (false) match |= MatchWord.NormalizeCoda;////종성우선 현상 정규화
////'먹었엌ㅋㅋ'처럼 연속하는 종성이 따라붙어 오분석이 나는 경우를 정규화합니다.
if (false) match |= MatchWord.Url;////url을 포함
if (false) match |= MatchWord.Email;////이메일을 포함
if (false) match |= MatchWord.Hashtag;////해시태그를 포함(#)
if (false) match |= MatchWord.Mention;////맨션을 포함(@+영어)
if (false) match |= MatchWord.Serial;////시리얼(일련번호)
if (false) match |= MatchWord.JoinNounPrefix;////명사+접두사
if (false) match |= MatchWord.JoinNounSuffix;////명사+접두사
if (false) match |= MatchWord.JoinVerbSuffix;////동사 파생접사
if (false) match |= MatchWord.JoinAdjSuffix;////형용사 파생 접사
foreach (string textData in textDataList)
{
foreach (var r in AnalyzeText(instKiwi, textData, mode, match, topN))
{
resultData.Add(r);
}
}
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
return resultData.ToList();
}
private KiwiBuilder AddWords(KiwiBuilder builder, string projectCode, string qid)
{
//builder.AddWord("사랑", "NNP", 90);//// 이런식으로 추가
DataTable addWordTable = getAddWords(projectCode, qid);
foreach(DataRow addWordRow in addWordTable.Rows)
{
try
{
builder.AddWord(addWordRow["WORD_TEXT"].ToString(), addWordRow["WORD_TAG"].ToString(), Convert.ToInt32(addWordRow["WORD_SCORE"]));
}
catch(Exception ex)
{
}
}
// <단일 형태소를 추가하는 경우>
// (형태) \t (품사태그) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 키위 NNP -5.0
//
// <이미 존재하는 형태소의 이형태를 추가하는 경우>
// (이형태) \t (원형태소/품사태그) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 기위 키위/NNG -3.0
//
// <기분석 형태를 추가하는 경우>
// (형태) \t (원형태소/품사태그 + 원형태소/품사태그 + ...) \t (점수)
// * (점수)는 생략시 0으로 처리됩니다.
// 예) 사겼다 사귀/VV + 었/EP + 다/EF -1.0
//
// 현재는 공백을 포함하는 다어절 형태를 등록할 수 없습니다.
//
// <규칙 기반의 변형된 이형태를 추가하는 경우>
// (형태 규칙)$ \t (변형된 형태/품사태그) \t (점수)
// 예) 요$ 용/EF -5
return builder;
}
/// <summary>
/// 텍스트 분석기
/// </summary>
/// <param name="text">input</param>
/// <param name="mode">어절/문장/줄/전체</param>
/// <param name="match">옵션</param>
/// <param name="topN">결과 개수 몇개 까지?</param>
/// <returns>?</returns>
private IEnumerable<AnalyzeResult> AnalyzeText(Kiwi instKiwi, string text, int mode, MatchWord match, int topN = 1)
{
if (mode == 2)
{
int rid = 0;
foreach (var line in text.Trim().Split('\n'))
{
var trimmed = line.Trim();
var res = instKiwi.Analyze(trimmed, topN, match);
List<List<Token>> s = new List<List<Token>>();
foreach (var r in res)
{
if (r.morphs.Length == 0) continue;
s.Add(new List<Token>(r.morphs));
}
if (trimmed.Length == 0) continue;
yield return new AnalyzeResult(++rid, trimmed, s);
}
}
else if (mode == 3)
{
var res = instKiwi.Analyze(text, topN, match);
List<List<Token>> s = new List<List<Token>>();
foreach (var r in res)
{
if (r.morphs.Length == 0) continue;
s.Add(new List<Token>(r.morphs));
}
yield return new AnalyzeResult(1, text.Trim(), s);
}
else
{
var r = instKiwi.Analyze(text, 1, match)[0];
int wp = 0, sp = 0, cp = 0, rid = 0, c = 0;
List<Token> s = new List<Token>();
foreach (var m in r.morphs)
{
if (mode == 0 ? (wp != m.wordPosition || sp != m.sentPosition) : (sp != m.sentPosition))
{
yield return new AnalyzeResult(++rid, text.Substring(cp, (int)m.chrPosition - cp).Trim(), s);
cp = (int)m.chrPosition;
c = 0;
s = new List<Token>();
}
s.Add(m);
wp = (int)m.wordPosition;
sp = (int)m.sentPosition;
}
if (s.Count > 0)
{
yield return new AnalyzeResult(++rid, text.Substring(cp), s);
}
}
}
public string[] getTags(string projectCode, string qid)
{
List<string> tags = new List<string>();
MSDAL msdal = new MSDAL();
var connection = msdal.ConnectReport();
try
{
string sql = string.Empty;
sql += " SELECT DISTINCT [WORD_TAG] " + Environment.NewLine;
sql += " FROM( " + Environment.NewLine;
sql += " SELECT [WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE projectCode = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE projectCode = '" + projectCode + "' AND QID = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG] FROM[REPORT_KON].[dbo].[REPORT_NLP_TAG] WHERE projectCode = '" + projectCode + "' AND QID = '" + qid + "' " + Environment.NewLine;
sql += " ) TAGS " + Environment.NewLine;
DataTable tagTable = msdal.GetDataTable(sql, connection);
foreach (DataRow row in tagTable.Rows)
{
tags.Add(row["WORD_TAG"].ToString());
}
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
finally
{
msdal.Disconnect(connection);
}
return tags.ToArray();
}
public DataTable getAddWords(string projectCode, string qid)
{
DataTable addWordsTable = new DataTable();
MSDAL msdal = new MSDAL();
var connection = msdal.ConnectReport();
try
{
string sql = string.Empty;
sql += " SELECT DISTINCT [WORD_TAG], [WORD_TEXT], [WORD_SCORE] " + Environment.NewLine;
sql += " FROM( " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 10 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE projectCode = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 50 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE projectCode = '" + projectCode + "' AND QID = 'ALL' " + Environment.NewLine;
sql += " UNION " + Environment.NewLine;
sql += " SELECT [WORD_TAG], [WORD_TEXT], 100 as [WORD_SCORE] FROM [REPORT_KON].[dbo].[REPORT_NLP_WORD] WHERE projectCode = '" + projectCode + "' AND QID = '" + qid + "' " + Environment.NewLine;
sql += " ) WORDS " + Environment.NewLine;
addWordsTable = msdal.GetDataTable(sql, connection);
}
catch (Exception ex)
{
logger.Fatal(this.GetType().Name + " > " + System.Reflection.MethodBase.GetCurrentMethod().Name + " : " + ex.Message);
}
finally
{
msdal.Disconnect(connection);
}
return addWordsTable;
}
/// <summary>
/// 단어 사전을 태그화 시키는 모듈
/// </summary>
/// <param name="wordDic">단어 사전</param>
/// <returns>단어, 카운트 테이블</returns>
public List<WordCloudTag> makeTagTable(List<WordTag> wordDic, int maxCount)
{
List<WordCloudTag> cloudTagList = new List<WordCloudTag>();
var distinctDic = wordDic.Select(x => new { x.Text, x.Tag}).Distinct().ToList();
foreach (var word in distinctDic)
{
int tagCount = wordDic.Where(x => x.Text == word.Text && x.Tag == word.Tag).Count();
cloudTagList.Add(new WordCloudTag(word.Text, word.Tag, tagCount.ToString()));
}
cloudTagList = cloudTagList.OrderByDescending(x => Convert.ToInt32(x.Count)).Take(maxCount).ToList();
return cloudTagList;
}
}
}