develop #1
@@ -32,7 +32,7 @@ repositories {
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation "com.kri.common:starter:beta.1.0.66"
|
||||
implementation "com.kri.common:starter:beta.1.1.03"
|
||||
|
||||
implementation 'org.springframework.boot:spring-boot-starter-web'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-security'
|
||||
|
||||
@@ -4,11 +4,13 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
@Slf4j
|
||||
@SpringBootApplication
|
||||
@ConfigurationPropertiesScan
|
||||
public class LgePenetrationBackApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.common.cache;
|
||||
|
||||
import com.kri.common.cache.AbstractMapCache;
|
||||
import com.kri.common.feature.cache.AbstractMapCache;
|
||||
import com.kri.lge.penetration.dto.VariableDto;
|
||||
import com.kri.lge.penetration.persistence.entity.TVariableEntity;
|
||||
import com.kri.lge.penetration.persistence.repository.VariableRepository;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.common.cache;
|
||||
|
||||
import com.kri.common.cache.AbstractListCache;
|
||||
import com.kri.common.feature.cache.AbstractListCache;
|
||||
import com.kri.lge.penetration.persistence.repository.YearRepository;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -2,7 +2,7 @@ package com.kri.lge.penetration.common.consts;
|
||||
|
||||
public final class LgePenetrationConst {
|
||||
|
||||
public static final int BATCH_SIZE = 1000;
|
||||
public static final int INSERT_CHUNK_SIZE = 1000;
|
||||
public static final int ADMIN_ROLE = 9999;
|
||||
|
||||
public static final String EXCEL_ID = "ID";
|
||||
|
||||
@@ -2,7 +2,7 @@ package com.kri.lge.penetration.common.enums;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
import com.kri.common.enums.CodeEnum;
|
||||
import com.kri.common.feature.enums.CodeEnum;
|
||||
import java.util.Objects;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.kri.lge.penetration.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@ConfigurationProperties(prefix = "kri.file")
|
||||
public class FileStorageProperties {
|
||||
private String dataBaseDir;
|
||||
}
|
||||
15
src/main/java/com/kri/lge/penetration/config/JdbcConfig.java
Normal file
15
src/main/java/com/kri/lge/penetration/config/JdbcConfig.java
Normal file
@@ -0,0 +1,15 @@
|
||||
package com.kri.lge.penetration.config;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
@Configuration
|
||||
public class JdbcConfig {
|
||||
|
||||
@Bean
|
||||
public JdbcTemplate jdbcTemplate(DataSource dataSource) {
|
||||
return new JdbcTemplate(dataSource);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.kri.lge.penetration.controller;
|
||||
|
||||
import com.kri.common.datetime.DateTimeUtil;
|
||||
import com.kri.common.webmvc.RequestUserContext;
|
||||
import com.kri.common.feature.datetime.DateTimeUtil;
|
||||
import com.kri.common.feature.web.RequestUserContext;
|
||||
import com.kri.lge.penetration.dto.LoginRequestDto;
|
||||
import com.kri.lge.penetration.dto.LoginResponseDto;
|
||||
import com.kri.lge.penetration.service.AuthService;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.controller;
|
||||
|
||||
import com.kri.common.web.BusinessException;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.lge.penetration.common.cache.StatisticsDbCache;
|
||||
import com.kri.lge.penetration.common.cache.VariableCache;
|
||||
import com.kri.lge.penetration.common.cache.YearCache;
|
||||
@@ -45,7 +45,6 @@ public class CacheAdminController {
|
||||
throw BusinessException.unauthorized("IS_NOT_ADMIN", "권한이 없습니다.");
|
||||
}
|
||||
|
||||
statisticsDbCache.clearStatisticsCache();
|
||||
return yearCache.reload();
|
||||
}
|
||||
|
||||
@@ -56,7 +55,16 @@ public class CacheAdminController {
|
||||
throw BusinessException.unauthorized("IS_NOT_ADMIN", "권한이 없습니다.");
|
||||
}
|
||||
|
||||
statisticsDbCache.clearStatisticsCache();
|
||||
return variableCache.reload();
|
||||
}
|
||||
|
||||
@Operation(summary = "통계처리 캐시 클리어")
|
||||
@PostMapping("/statistics-db/clear/{adminPw}")
|
||||
public void clearStatisticsDbCache(@PathVariable String adminPw) {
|
||||
if (!authService.isAdmin(adminPw)) {
|
||||
throw BusinessException.unauthorized("IS_NOT_ADMIN", "권한이 없습니다.");
|
||||
}
|
||||
|
||||
statisticsDbCache.clearStatisticsCache();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.dto;
|
||||
|
||||
import com.kri.common.datetime.DateTimeUtil;
|
||||
import com.kri.common.feature.datetime.DateTimeUtil;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import java.time.LocalDateTime;
|
||||
import lombok.Getter;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.dto;
|
||||
|
||||
import com.kri.common.auth.dto.JwtUserClaimsDto;
|
||||
import com.kri.common.feature.auth.dto.JwtUserClaimsDto;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.kri.lge.penetration.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@@ -12,7 +13,10 @@ import lombok.Setter;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@Schema(description = "변수 보기 정보 Dto")
|
||||
public class VariableItemInfoDto {
|
||||
public class VariableItemInfoDto implements Serializable {
|
||||
|
||||
@Builder.Default
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Integer order;
|
||||
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package com.kri.lge.penetration.persistence.jdbc;
|
||||
|
||||
import java.sql.BatchUpdateException;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
@RequiredArgsConstructor
|
||||
public class DataJdbcRepository {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private static final String INSERT_SQL = """
|
||||
insert into t_data (
|
||||
data_idx,
|
||||
sid,
|
||||
raw_data,
|
||||
reg_user,
|
||||
reg_dt
|
||||
) values (?, ?, ?, ?, ?)
|
||||
""";
|
||||
|
||||
public record TDataInsertRow(
|
||||
String dataIdx,
|
||||
String sid,
|
||||
String rawDataJson,
|
||||
String regUser,
|
||||
LocalDateTime regDt
|
||||
) {
|
||||
}
|
||||
|
||||
public record TDataInsertFailure(
|
||||
int index,
|
||||
String reason
|
||||
) {
|
||||
}
|
||||
|
||||
public record TDataBatchInsertResult(
|
||||
List<Integer> successIndexes,
|
||||
List<TDataInsertFailure> failures
|
||||
) {
|
||||
public int successCount() {
|
||||
return successIndexes == null ? 0 : successIndexes.size();
|
||||
}
|
||||
|
||||
public int failCount() {
|
||||
return failures == null ? 0 : failures.size();
|
||||
}
|
||||
}
|
||||
|
||||
public TDataBatchInsertResult bulkInsert(List<TDataInsertRow> rows) {
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return new TDataBatchInsertResult(List.of(), List.of());
|
||||
}
|
||||
|
||||
try {
|
||||
jdbcTemplate.batchUpdate(INSERT_SQL, new BatchPreparedStatementSetter() {
|
||||
@Override
|
||||
public void setValues(PreparedStatement ps, int i) throws SQLException {
|
||||
TDataInsertRow row = rows.get(i);
|
||||
ps.setString(1, row.dataIdx());
|
||||
ps.setString(2, row.sid());
|
||||
ps.setString(3, row.rawDataJson());
|
||||
ps.setString(4, row.regUser());
|
||||
ps.setTimestamp(5, Timestamp.valueOf(row.regDt()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getBatchSize() {
|
||||
return rows.size();
|
||||
}
|
||||
});
|
||||
|
||||
List<Integer> successIndexes = new ArrayList<>(rows.size());
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
successIndexes.add(i);
|
||||
}
|
||||
return new TDataBatchInsertResult(successIndexes, List.of());
|
||||
|
||||
} catch (DataAccessException e) {
|
||||
return handleBatchFailure(rows, e);
|
||||
}
|
||||
}
|
||||
|
||||
private TDataBatchInsertResult handleBatchFailure(List<TDataInsertRow> rows, DataAccessException e) {
|
||||
BatchUpdateException batchEx = findBatchUpdateException(e);
|
||||
|
||||
List<Integer> successIndexes = new ArrayList<>();
|
||||
List<TDataInsertFailure> failures = new ArrayList<>();
|
||||
|
||||
if (batchEx == null) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
retryOne(rows.get(i), i, successIndexes, failures);
|
||||
}
|
||||
return new TDataBatchInsertResult(successIndexes, failures);
|
||||
}
|
||||
|
||||
int[] counts = batchEx.getUpdateCounts();
|
||||
boolean[] needsRetry = new boolean[rows.size()];
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
needsRetry[i] = true;
|
||||
}
|
||||
|
||||
if (counts != null) {
|
||||
int limit = Math.min(counts.length, rows.size());
|
||||
for (int i = 0; i < limit; i++) {
|
||||
int c = counts[i];
|
||||
if (isBatchSuccess(c)) {
|
||||
successIndexes.add(i);
|
||||
needsRetry[i] = false;
|
||||
} else if (c == Statement.EXECUTE_FAILED) {
|
||||
needsRetry[i] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
if (needsRetry[i]) {
|
||||
retryOne(rows.get(i), i, successIndexes, failures);
|
||||
}
|
||||
}
|
||||
|
||||
return new TDataBatchInsertResult(successIndexes, failures);
|
||||
}
|
||||
|
||||
private void retryOne(
|
||||
TDataInsertRow row,
|
||||
int index,
|
||||
List<Integer> successIndexes,
|
||||
List<TDataInsertFailure> failures
|
||||
) {
|
||||
try {
|
||||
jdbcTemplate.update(
|
||||
INSERT_SQL,
|
||||
row.dataIdx(),
|
||||
row.sid(),
|
||||
row.rawDataJson(),
|
||||
row.regUser(),
|
||||
Timestamp.valueOf(row.regDt())
|
||||
);
|
||||
successIndexes.add(index);
|
||||
} catch (DataAccessException ex) {
|
||||
failures.add(new TDataInsertFailure(index, rootMessage(ex)));
|
||||
}
|
||||
}
|
||||
|
||||
private static BatchUpdateException findBatchUpdateException(Throwable t) {
|
||||
Throwable cur = t;
|
||||
while (cur != null) {
|
||||
if (cur instanceof BatchUpdateException bue) {
|
||||
return bue;
|
||||
}
|
||||
cur = cur.getCause();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean isBatchSuccess(int count) {
|
||||
return count >= 0 || count == Statement.SUCCESS_NO_INFO;
|
||||
}
|
||||
|
||||
private static String rootMessage(Throwable t) {
|
||||
Throwable cur = t;
|
||||
while (cur.getCause() != null) {
|
||||
cur = cur.getCause();
|
||||
}
|
||||
return cur.getMessage();
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.kri.lge.penetration.service;
|
||||
|
||||
import com.kri.common.auth.dto.JwtUserClaimsDto;
|
||||
import com.kri.common.auth.jwt.JwtService;
|
||||
import com.kri.common.web.BusinessException;
|
||||
import com.kri.common.feature.auth.dto.JwtUserClaimsDto;
|
||||
import com.kri.common.feature.auth.jwt.JwtService;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.lge.penetration.common.consts.LgePenetrationConst;
|
||||
import com.kri.lge.penetration.dto.LoginRequestDto;
|
||||
import com.kri.lge.penetration.dto.LoginResponseDto;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.kri.lge.penetration.service;
|
||||
|
||||
import com.kri.common.web.BusinessException;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.lge.penetration.common.cache.StatisticsDbCache;
|
||||
import com.kri.lge.penetration.common.cache.VariableCache;
|
||||
import com.kri.lge.penetration.common.cache.YearCache;
|
||||
@@ -21,6 +21,7 @@ import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.DoubleSummaryStatistics;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -200,7 +201,9 @@ public class StatisticsService {
|
||||
String qid = v.getTargetQid();
|
||||
List<BannerSpec> choices = new ArrayList<>();
|
||||
|
||||
for (VariableItemInfoDto scalar : v.getScalarInfo()) {
|
||||
VariableItemInfoDto[] bannerScalarList =filterVariableScalarInfo(v.getScalarInfo(), null);
|
||||
|
||||
for (VariableItemInfoDto scalar : bannerScalarList) {
|
||||
if (scalar == null) continue;
|
||||
List<Integer> vals = scalar.getValue();
|
||||
if (vals == null || vals.isEmpty()) continue;
|
||||
@@ -666,9 +669,18 @@ public class StatisticsService {
|
||||
return new VariableItemInfoDto[0];
|
||||
}
|
||||
|
||||
return filterVariableScalarInfo(scalarInfo, sValueSet);
|
||||
}
|
||||
|
||||
private VariableItemInfoDto[] filterVariableScalarInfo(VariableItemInfoDto[] scalarInfo, Set<Integer> sValueSet) {
|
||||
//sValueSet 이 없을 수도 있음 없으면 filter를 하지 말아야 함
|
||||
return Arrays.stream(scalarInfo)
|
||||
.filter(Objects::nonNull)
|
||||
.filter(info -> containsAny(info.getValue(), sValueSet))
|
||||
.filter(info -> sValueSet == null || sValueSet.isEmpty() || containsAny(info.getValue(), sValueSet))
|
||||
.sorted(Comparator.comparing(
|
||||
VariableItemInfoDto::getOrder,
|
||||
Comparator.nullsLast(Integer::compareTo)
|
||||
))
|
||||
.toArray(VariableItemInfoDto[]::new);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,33 +1,33 @@
|
||||
package com.kri.lge.penetration.service;
|
||||
|
||||
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.BATCH_SIZE;
|
||||
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.EXCEL_ID;
|
||||
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.INSERT_CHUNK_SIZE;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kri.common.datetime.DateTimeUtil;
|
||||
import com.kri.common.excel.ExcelMapper;
|
||||
import com.kri.common.web.BusinessException;
|
||||
import com.kri.common.feature.datetime.DateTimeUtil;
|
||||
import com.kri.common.feature.excel.ExcelMapper;
|
||||
import com.kri.lge.penetration.dto.DataItemDto;
|
||||
import com.kri.lge.penetration.dto.ExcelUploadResultDto;
|
||||
import com.kri.lge.penetration.persistence.entity.TDataEntity;
|
||||
import com.kri.lge.penetration.persistence.repository.DataRepository;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.persistence.PersistenceContext;
|
||||
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository;
|
||||
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataBatchInsertResult;
|
||||
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataInsertFailure;
|
||||
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataInsertRow;
|
||||
import com.kri.lge.penetration.service.file.DataFileService;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@Log4j2
|
||||
@@ -35,58 +35,53 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
@RequiredArgsConstructor
|
||||
public class UploadService {
|
||||
|
||||
@PersistenceContext
|
||||
private EntityManager em;
|
||||
private final DataFileService dataFileService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final DataRepository dataRepository;
|
||||
private final DataJdbcRepository tDataJdbcRepository;
|
||||
|
||||
public ExcelUploadResultDto uploadData(MultipartFile file){
|
||||
try (InputStream in = file.getInputStream()) {
|
||||
List<RowContext> buffer = new ArrayList<>(BATCH_SIZE);
|
||||
List<Map<String, String>> failRows = new ArrayList<>();
|
||||
@Transactional
|
||||
public ExcelUploadResultDto uploadData(MultipartFile file) {
|
||||
Path dataFilePath = dataFileService.saveListFile(file);
|
||||
|
||||
AtomicInteger total = new AtomicInteger();
|
||||
AtomicInteger success = new AtomicInteger();
|
||||
AtomicInteger fail = new AtomicInteger();
|
||||
List<RowContext> buffer = new ArrayList<>(INSERT_CHUNK_SIZE);
|
||||
List<Map<String, String>> failRows = new ArrayList<>();
|
||||
|
||||
final LocalDateTime now = DateTimeUtil.nowKstLocalDateTime();
|
||||
final String regUser = "system";
|
||||
AtomicInteger total = new AtomicInteger();
|
||||
AtomicInteger success = new AtomicInteger();
|
||||
AtomicInteger fail = new AtomicInteger();
|
||||
|
||||
ExcelMapper.readStreaming(in, (row, rowIndex) -> {
|
||||
total.incrementAndGet();
|
||||
final LocalDateTime now = DateTimeUtil.nowKstLocalDateTime();
|
||||
final String regUser = "system";
|
||||
|
||||
ParsedRow parsed = parseRowToEntity(row, regUser, now);
|
||||
if (parsed.reason != null) {
|
||||
fail.incrementAndGet();
|
||||
failRows.add(buildFailRow(row, rowIndex, parsed.reason));
|
||||
return;
|
||||
}
|
||||
ExcelMapper.readStreaming(dataFilePath, (row, rowIndex) -> {
|
||||
total.incrementAndGet();
|
||||
|
||||
buffer.add(new RowContext(rowIndex, row, parsed.entity));
|
||||
ParsedRow parsed = parseRow(row, regUser, now);
|
||||
if (parsed.reason != null) {
|
||||
fail.incrementAndGet();
|
||||
failRows.add(buildFailRow(row, rowIndex, parsed.reason));
|
||||
return;
|
||||
}
|
||||
|
||||
if (buffer.size() >= BATCH_SIZE) {
|
||||
flushBuffer(buffer, success, fail, failRows);
|
||||
}
|
||||
});
|
||||
buffer.add(new RowContext(rowIndex, row, parsed.insertRow));
|
||||
|
||||
if (!buffer.isEmpty()) {
|
||||
if (buffer.size() >= INSERT_CHUNK_SIZE) {
|
||||
flushBuffer(buffer, success, fail, failRows);
|
||||
}
|
||||
});
|
||||
|
||||
byte[] errorFile = null;
|
||||
boolean hasErrorFile = !failRows.isEmpty();
|
||||
|
||||
if (hasErrorFile) {
|
||||
errorFile = toXlsx(failRows);
|
||||
}
|
||||
|
||||
return ExcelUploadResultDto.of(total.get(), success.get(), fail.get(), errorFile);
|
||||
|
||||
} catch (IOException ex){
|
||||
throw BusinessException.badRequest("CAN_NOT_OPEN", "파일 스트림을 열 수 없습니다.")
|
||||
.with("fileName",file.getOriginalFilename())
|
||||
.with("errorMessage",ex.getMessage());
|
||||
if (!buffer.isEmpty()) {
|
||||
flushBuffer(buffer, success, fail, failRows);
|
||||
}
|
||||
|
||||
byte[] errorFile = null;
|
||||
if (!failRows.isEmpty()) {
|
||||
errorFile = toXlsx(failRows);
|
||||
}
|
||||
|
||||
dataFileService.deleteListFile(dataFilePath.getFileName().toString());
|
||||
|
||||
return ExcelUploadResultDto.of(total.get(), success.get(), fail.get(), errorFile);
|
||||
}
|
||||
|
||||
private void flushBuffer(
|
||||
@@ -95,29 +90,30 @@ public class UploadService {
|
||||
AtomicInteger fail,
|
||||
List<Map<String, String>> failRows
|
||||
) {
|
||||
try {
|
||||
List<TDataEntity> entities = buffer.stream().map(rc -> rc.entity).toList();
|
||||
if (buffer.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
dataRepository.saveAll(entities);
|
||||
success.addAndGet(buffer.size());
|
||||
} catch (DataAccessException e) {
|
||||
// 배치가 한 번에 터지면, 개별 저장으로 폴백해서 "어느 행이 문제인지" 잡는다
|
||||
for (RowContext rc : buffer) {
|
||||
try {
|
||||
dataRepository.saveAndFlush(rc.entity);
|
||||
success.incrementAndGet();
|
||||
} catch (DataAccessException ex) {
|
||||
fail.incrementAndGet();
|
||||
failRows.add(buildFailRow(rc.rawRow, rc.rowIndex, "DB 저장 실패: " + rootMessage(ex)));
|
||||
}
|
||||
try {
|
||||
List<TDataInsertRow> rows = buffer.stream()
|
||||
.map(RowContext::insertRow)
|
||||
.toList();
|
||||
|
||||
TDataBatchInsertResult result = tDataJdbcRepository.bulkInsert(rows);
|
||||
|
||||
success.addAndGet(result.successCount());
|
||||
|
||||
for (TDataInsertFailure failure : result.failures()) {
|
||||
RowContext rc = buffer.get(failure.index());
|
||||
fail.incrementAndGet();
|
||||
failRows.add(buildFailRow(rc.rawRow(), rc.rowIndex(), "DB 저장 실패: " + failure.reason()));
|
||||
}
|
||||
} finally {
|
||||
em.clear();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private ParsedRow parseRowToEntity(
|
||||
private ParsedRow parseRow(
|
||||
Map<String, String> row,
|
||||
String regUser,
|
||||
LocalDateTime regDt
|
||||
@@ -126,7 +122,6 @@ public class UploadService {
|
||||
return ParsedRow.fail("빈 행");
|
||||
}
|
||||
|
||||
// sid
|
||||
String sid = firstNonBlank(row.get(EXCEL_ID), row.get("sid"), row.get("SID"));
|
||||
if (sid == null) {
|
||||
return ParsedRow.fail("ID(sid) 누락");
|
||||
@@ -135,21 +130,16 @@ public class UploadService {
|
||||
return ParsedRow.fail("ID(sid) 길이 초과(>32): " + sid);
|
||||
}
|
||||
|
||||
// rawData 구성
|
||||
Map<String, List<DataItemDto>> rawData = new LinkedHashMap<>();
|
||||
|
||||
for (Map.Entry<String, String> e : row.entrySet()) {
|
||||
String key = e.getKey();
|
||||
if (key == null) continue;
|
||||
|
||||
// 제외
|
||||
if (EXCEL_ID.equals(key)) continue;
|
||||
//if (EXCEL_YEAR.equals(key)) continue;
|
||||
//if (EXCEL_PRODUCT.equals(key)) continue;
|
||||
|
||||
String v = e.getValue();
|
||||
if (v == null || v.isBlank()) {
|
||||
continue; // 빈 값은 rawData에 안 넣음
|
||||
if (v == null || v.isBlank() || "#NULL!".equals(v.trim())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Double s = parseDoubleLoose(v);
|
||||
@@ -157,23 +147,33 @@ public class UploadService {
|
||||
return ParsedRow.fail("숫자 파싱 실패: key=" + key + ", value=" + safe(v));
|
||||
}
|
||||
|
||||
if(key.contains("#")){
|
||||
key = key.substring(0,key.indexOf("#"));
|
||||
if (key.contains("#")) {
|
||||
key = key.substring(0, key.indexOf("#"));
|
||||
}
|
||||
rawData.computeIfAbsent(key, k -> new ArrayList<>()).add(DataItemDto.from(null, null, s));
|
||||
|
||||
rawData.computeIfAbsent(key, k -> new ArrayList<>())
|
||||
.add(DataItemDto.from(null, null, s));
|
||||
}
|
||||
|
||||
TDataEntity entity = new TDataEntity();
|
||||
entity.setSid(sid);
|
||||
entity.setRawData(rawData);
|
||||
entity.setRegUser(regUser);
|
||||
entity.setRegDt(regDt);
|
||||
String rawDataJson;
|
||||
try {
|
||||
rawDataJson = objectMapper.writeValueAsString(rawData);
|
||||
} catch (JsonProcessingException e) {
|
||||
return ParsedRow.fail("rawData JSON 직렬화 실패: " + rootMessage(e));
|
||||
}
|
||||
|
||||
return ParsedRow.ok(entity);
|
||||
TDataInsertRow insertRow = new TDataInsertRow(
|
||||
UUID.randomUUID().toString(),
|
||||
sid,
|
||||
rawDataJson,
|
||||
regUser,
|
||||
regDt
|
||||
);
|
||||
|
||||
return ParsedRow.ok(insertRow);
|
||||
}
|
||||
|
||||
private Map<String, String> buildFailRow(Map<String, String> row, int rowIndex, String reason) {
|
||||
// 에러 엑셀에 행번호/사유를 앞에 고정 컬럼으로 넣고, 원본 컬럼들도 같이 싣는다
|
||||
Map<String, String> m = new LinkedHashMap<>();
|
||||
m.put("rowIndex", String.valueOf(rowIndex));
|
||||
m.put("reason", reason);
|
||||
@@ -195,26 +195,16 @@ public class UploadService {
|
||||
|
||||
private static String rootMessage(Throwable t) {
|
||||
Throwable cur = t;
|
||||
while (cur.getCause() != null) cur = cur.getCause();
|
||||
return cur.getMessage();
|
||||
}
|
||||
|
||||
private void doDebugLog(TDataEntity e){
|
||||
try {
|
||||
String json = objectMapper
|
||||
.writerWithDefaultPrettyPrinter()
|
||||
.writeValueAsString(e.getRawData());
|
||||
log.info("rawDataJson={}", json);
|
||||
} catch (JsonProcessingException ex) {
|
||||
log.warn("rawData json serialize failed", ex);
|
||||
while (cur.getCause() != null) {
|
||||
cur = cur.getCause();
|
||||
}
|
||||
return cur.getMessage();
|
||||
}
|
||||
|
||||
private static Double parseDoubleLoose(String s) {
|
||||
if (s == null) return null;
|
||||
String t = s.trim();
|
||||
if (t.isEmpty()) return null;
|
||||
// 천단위 콤마 제거
|
||||
t = t.replace(",", "");
|
||||
try {
|
||||
return Double.parseDouble(t);
|
||||
@@ -238,15 +228,15 @@ public class UploadService {
|
||||
private record RowContext(
|
||||
int rowIndex,
|
||||
Map<String, String> rawRow,
|
||||
TDataEntity entity
|
||||
TDataInsertRow insertRow
|
||||
) {}
|
||||
|
||||
private record ParsedRow(
|
||||
TDataEntity entity,
|
||||
TDataInsertRow insertRow,
|
||||
String reason
|
||||
) {
|
||||
static ParsedRow ok(TDataEntity entity) {
|
||||
return new ParsedRow(entity, null);
|
||||
static ParsedRow ok(TDataInsertRow insertRow) {
|
||||
return new ParsedRow(insertRow, null);
|
||||
}
|
||||
|
||||
static ParsedRow fail(String reason) {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.kri.lge.penetration.service.file;
|
||||
|
||||
import com.kri.common.feature.datetime.DateTimeUtil;
|
||||
import com.kri.common.feature.util.CommonUtil;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Optional;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
public abstract class AbstractFileService {
|
||||
|
||||
protected abstract Path getBaseDir();
|
||||
|
||||
protected String extractExtension(String originalFileName) {
|
||||
if (originalFileName == null || originalFileName.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
int dot = originalFileName.lastIndexOf('.');
|
||||
if (dot > -1 && dot < originalFileName.length() - 1) {
|
||||
return originalFileName.substring(dot);
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
protected String extractBaseName(String originalFileName) {
|
||||
if (originalFileName == null || originalFileName.isBlank()) {
|
||||
return "file";
|
||||
}
|
||||
|
||||
String fileName = java.nio.file.Paths.get(originalFileName).getFileName().toString();
|
||||
int dot = fileName.lastIndexOf('.');
|
||||
|
||||
if (dot > 0) {
|
||||
return fileName.substring(0, dot);
|
||||
}
|
||||
|
||||
return fileName;
|
||||
}
|
||||
|
||||
protected String buildFileName(MultipartFile file) {
|
||||
String original = Optional.ofNullable(file.getOriginalFilename()).orElse("file");
|
||||
return CommonUtil.uuid32() + extractExtension(original);
|
||||
}
|
||||
|
||||
protected String buildFileNameWithTime(MultipartFile file, String prefix) {
|
||||
String original = Optional.ofNullable(file.getOriginalFilename()).orElse("file");
|
||||
String baseName = extractBaseName(original);
|
||||
String ext = extractExtension(original);
|
||||
|
||||
String timestamp = DateTimeUtil.nowKstLocalDateTime()
|
||||
.format(java.time.format.DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"));
|
||||
|
||||
return prefix + "_" + timestamp + "_" + baseName + ext;
|
||||
}
|
||||
|
||||
protected Path createDirectories(Path dir) throws IOException {
|
||||
return Files.createDirectories(dir.toAbsolutePath().normalize());
|
||||
}
|
||||
|
||||
protected String saveFile(Path dir, MultipartFile file) throws IOException {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new IllegalArgumentException("업로드할 파일이 없습니다.");
|
||||
}
|
||||
|
||||
Path normalizedDir = createDirectories(dir);
|
||||
String fileName = buildFileName(file);
|
||||
Path target = normalizedDir.resolve(fileName).normalize();
|
||||
|
||||
if (!target.startsWith(normalizedDir)) {
|
||||
throw new IllegalStateException("잘못된 파일 경로입니다.");
|
||||
}
|
||||
|
||||
file.transferTo(target.toFile());
|
||||
return fileName;
|
||||
}
|
||||
|
||||
protected void saveFile(Path dir, MultipartFile file, String fileName) throws IOException {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new IllegalArgumentException("업로드할 파일이 없습니다.");
|
||||
}
|
||||
|
||||
Path normalizedDir = createDirectories(dir);
|
||||
Path target = normalizedDir.resolve(fileName).normalize();
|
||||
|
||||
if (!target.startsWith(normalizedDir)) {
|
||||
throw new IllegalStateException("잘못된 파일 경로입니다.");
|
||||
}
|
||||
|
||||
file.transferTo(target.toFile());
|
||||
}
|
||||
|
||||
protected void moveFile(Path sourceDir, Path targetDir, String fileName) throws IOException {
|
||||
if (fileName == null || fileName.isBlank()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Path normalizedSourceDir = sourceDir.toAbsolutePath().normalize();
|
||||
Path normalizedTargetDir = createDirectories(targetDir);
|
||||
|
||||
Path source = normalizedSourceDir.resolve(fileName).normalize();
|
||||
Path target = normalizedTargetDir.resolve(fileName).normalize();
|
||||
|
||||
if (!source.startsWith(normalizedSourceDir) || !target.startsWith(normalizedTargetDir)) {
|
||||
throw new IllegalStateException("잘못된 파일 경로입니다.");
|
||||
}
|
||||
if (!Files.exists(source)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
protected void deleteFile(Path dir, String fileName) throws IOException {
|
||||
if (fileName == null || fileName.isBlank()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Path normalizedDir = dir.toAbsolutePath().normalize();
|
||||
Path target = normalizedDir.resolve(fileName).normalize();
|
||||
|
||||
if (!target.startsWith(normalizedDir)) {
|
||||
throw new IllegalStateException("잘못된 파일 경로입니다.");
|
||||
}
|
||||
|
||||
Files.deleteIfExists(target);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.kri.lge.penetration.service.file;
|
||||
|
||||
import com.kri.common.feature.util.CommonUtil;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.lge.penetration.config.FileStorageProperties;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DataFileService extends AbstractFileService {
|
||||
|
||||
private final FileStorageProperties properties;
|
||||
|
||||
@Override
|
||||
protected Path getBaseDir() {
|
||||
return Paths.get(properties.getDataBaseDir()).toAbsolutePath().normalize();
|
||||
}
|
||||
|
||||
public Path saveListFile(MultipartFile file) {
|
||||
String newFileName = buildFileNameWithTime(file, CommonUtil.uuid32());
|
||||
Path dir = getBaseDir().toAbsolutePath().normalize();
|
||||
Path savedPath = dir.resolve(newFileName).toAbsolutePath().normalize();
|
||||
|
||||
try {
|
||||
saveFile(dir, file, newFileName);
|
||||
return savedPath;
|
||||
} catch (IOException e) {
|
||||
throw BusinessException.internalError("FAIL_DATA_UPLOAD", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteListFile(String fileName) {
|
||||
Path dir = getBaseDir().toAbsolutePath().normalize();
|
||||
|
||||
try {
|
||||
deleteFile(dir, fileName);
|
||||
} catch (IOException e) {
|
||||
throw BusinessException.internalError("FAIL_DATA_DELETE", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,6 @@ spring:
|
||||
datasource:
|
||||
#private 도메인
|
||||
url: jdbc:mysql://db-md87m.vpc-cdb.ntruss.com:3306/test_lge_cx_dash?characterEncoding=UTF-8&rewriteBatchedStatements=true
|
||||
username: kriit
|
||||
password: it#51304
|
||||
username: prjct_lge_cx
|
||||
password: lge#260324
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
@@ -2,8 +2,8 @@ spring:
|
||||
datasource:
|
||||
#public 도메인
|
||||
url: jdbc:mysql://db-md87m-kr.vpc-pub-cdb.ntruss.com:3306/test_lge_cx_dash?characterEncoding=UTF-8&rewriteBatchedStatements=true
|
||||
username: kriit
|
||||
password: it#51304
|
||||
username: prjct_lge_cx
|
||||
password: lge#260324
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
|
||||
jpa:
|
||||
@@ -20,6 +20,6 @@ spring:
|
||||
logging:
|
||||
level:
|
||||
org.hibernate.SQL: DEBUG
|
||||
org.hibernate.orm.jdbc.bind: TRACE
|
||||
#org.hibernate.orm.jdbc.bind: TRACE
|
||||
#org.hibernate.orm.jdbc.extract: TRACE
|
||||
org.hibernate.engine.jdbc.batch.internal: DEBUG
|
||||
@@ -1,7 +1,7 @@
|
||||
spring:
|
||||
datasource:
|
||||
#private 도메인
|
||||
url: jdbc:mysql://db-md87m.vpc-cdb.ntruss.com:3306/test_lge_cx_dash?characterEncoding=UTF-8&rewriteBatchedStatements=true
|
||||
username: kriit
|
||||
password: it#51304
|
||||
url: jdbc:mysql://db-md87m.vpc-cdb.ntruss.com:3306/lge_cx_dash?characterEncoding=UTF-8&rewriteBatchedStatements=true
|
||||
username: prjct_lge_cx
|
||||
password: lge#260324
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
@@ -2,7 +2,7 @@ server:
|
||||
port: 6101
|
||||
forward-headers-strategy: framework
|
||||
tomcat:
|
||||
max-http-form-post-size: 50MB
|
||||
max-http-form-post-size: 100MB
|
||||
max-swallow-size: -1
|
||||
|
||||
spring:
|
||||
@@ -22,8 +22,8 @@ spring:
|
||||
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 50MB
|
||||
max-request-size: 50MB
|
||||
max-file-size: 100MB
|
||||
max-request-size: 100MB
|
||||
|
||||
logging:
|
||||
file:
|
||||
@@ -42,12 +42,16 @@ kri:
|
||||
|
||||
swagger:
|
||||
enabled: true
|
||||
allowed-ips:
|
||||
- 211.254.145.130
|
||||
title: "LGE CX Dash"
|
||||
description: "LGE 가전 보급률 Dashboard"
|
||||
version: "v1.0.0"
|
||||
base-package: "com.kri.lge.penetration"
|
||||
|
||||
common:
|
||||
file:
|
||||
data-base-dir: ./upload-file/data
|
||||
|
||||
web:
|
||||
enabled: true # 공통 Web 구성 사용 여부 (기본 true)
|
||||
webmvc:
|
||||
|
||||
Reference in New Issue
Block a user