260311_01_ENH: JDBC Template 기반 Bulk Insert로 교체 및 디스크 기반 엑셀 스트림 리더 적용

This commit is contained in:
2026-03-11 16:22:24 +09:00
parent ad1c445f5b
commit a30f7d09d4
10 changed files with 466 additions and 101 deletions

View File

@@ -4,11 +4,13 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
@Slf4j
@SpringBootApplication
@ConfigurationPropertiesScan
public class LgePenetrationBackApplication {
public static void main(String[] args) {

View File

@@ -0,0 +1,12 @@
package com.kri.lge.penetration.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Getter
@Setter
@ConfigurationProperties(prefix = "kri.file")
public class FileStorageProperties {
private String dataBaseDir;
}

View File

@@ -0,0 +1,15 @@
package com.kri.lge.penetration.config;
import javax.sql.DataSource;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
@Configuration
public class JdbcConfig {
@Bean
public JdbcTemplate jdbcTemplate(DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
}

View File

@@ -0,0 +1,178 @@
package com.kri.lge.penetration.persistence.jdbc;
import java.sql.BatchUpdateException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
@Repository
@RequiredArgsConstructor
public class DataJdbcRepository {
private final JdbcTemplate jdbcTemplate;
private static final String INSERT_SQL = """
insert into t_data (
data_idx,
sid,
raw_data,
reg_user,
reg_dt
) values (?, ?, ?, ?, ?)
""";
public record TDataInsertRow(
String dataIdx,
String sid,
String rawDataJson,
String regUser,
LocalDateTime regDt
) {
}
public record TDataInsertFailure(
int index,
String reason
) {
}
public record TDataBatchInsertResult(
List<Integer> successIndexes,
List<TDataInsertFailure> failures
) {
public int successCount() {
return successIndexes == null ? 0 : successIndexes.size();
}
public int failCount() {
return failures == null ? 0 : failures.size();
}
}
public TDataBatchInsertResult bulkInsert(List<TDataInsertRow> rows) {
if (rows == null || rows.isEmpty()) {
return new TDataBatchInsertResult(List.of(), List.of());
}
try {
jdbcTemplate.batchUpdate(INSERT_SQL, new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
TDataInsertRow row = rows.get(i);
ps.setString(1, row.dataIdx());
ps.setString(2, row.sid());
ps.setString(3, row.rawDataJson());
ps.setString(4, row.regUser());
ps.setTimestamp(5, Timestamp.valueOf(row.regDt()));
}
@Override
public int getBatchSize() {
return rows.size();
}
});
List<Integer> successIndexes = new ArrayList<>(rows.size());
for (int i = 0; i < rows.size(); i++) {
successIndexes.add(i);
}
return new TDataBatchInsertResult(successIndexes, List.of());
} catch (DataAccessException e) {
return handleBatchFailure(rows, e);
}
}
private TDataBatchInsertResult handleBatchFailure(List<TDataInsertRow> rows, DataAccessException e) {
BatchUpdateException batchEx = findBatchUpdateException(e);
List<Integer> successIndexes = new ArrayList<>();
List<TDataInsertFailure> failures = new ArrayList<>();
if (batchEx == null) {
for (int i = 0; i < rows.size(); i++) {
retryOne(rows.get(i), i, successIndexes, failures);
}
return new TDataBatchInsertResult(successIndexes, failures);
}
int[] counts = batchEx.getUpdateCounts();
boolean[] needsRetry = new boolean[rows.size()];
for (int i = 0; i < rows.size(); i++) {
needsRetry[i] = true;
}
if (counts != null) {
int limit = Math.min(counts.length, rows.size());
for (int i = 0; i < limit; i++) {
int c = counts[i];
if (isBatchSuccess(c)) {
successIndexes.add(i);
needsRetry[i] = false;
} else if (c == Statement.EXECUTE_FAILED) {
needsRetry[i] = true;
}
}
}
for (int i = 0; i < rows.size(); i++) {
if (needsRetry[i]) {
retryOne(rows.get(i), i, successIndexes, failures);
}
}
return new TDataBatchInsertResult(successIndexes, failures);
}
private void retryOne(
TDataInsertRow row,
int index,
List<Integer> successIndexes,
List<TDataInsertFailure> failures
) {
try {
jdbcTemplate.update(
INSERT_SQL,
row.dataIdx(),
row.sid(),
row.rawDataJson(),
row.regUser(),
Timestamp.valueOf(row.regDt())
);
successIndexes.add(index);
} catch (DataAccessException ex) {
failures.add(new TDataInsertFailure(index, rootMessage(ex)));
}
}
private static BatchUpdateException findBatchUpdateException(Throwable t) {
Throwable cur = t;
while (cur != null) {
if (cur instanceof BatchUpdateException bue) {
return bue;
}
cur = cur.getCause();
}
return null;
}
private static boolean isBatchSuccess(int count) {
return count >= 0 || count == Statement.SUCCESS_NO_INFO;
}
private static String rootMessage(Throwable t) {
Throwable cur = t;
while (cur.getCause() != null) {
cur = cur.getCause();
}
return cur.getMessage();
}
}

View File

@@ -1,33 +1,33 @@
package com.kri.lge.penetration.service;
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.INSERT_CHUNK_SIZE;
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.EXCEL_ID;
import static com.kri.lge.penetration.common.consts.LgePenetrationConst.INSERT_CHUNK_SIZE;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kri.common.datetime.DateTimeUtil;
import com.kri.common.excel.ExcelMapper;
import com.kri.common.web.BusinessException;
import com.kri.lge.penetration.dto.DataItemDto;
import com.kri.lge.penetration.dto.ExcelUploadResultDto;
import com.kri.lge.penetration.persistence.entity.TDataEntity;
import com.kri.lge.penetration.persistence.repository.DataRepository;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository;
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataBatchInsertResult;
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataInsertFailure;
import com.kri.lge.penetration.persistence.jdbc.DataJdbcRepository.TDataInsertRow;
import com.kri.lge.penetration.service.file.DataFileService;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.dao.DataAccessException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
@Log4j2
@@ -35,58 +35,51 @@ import org.springframework.web.multipart.MultipartFile;
@RequiredArgsConstructor
public class UploadService {
@PersistenceContext
private EntityManager em;
private final DataFileService dataFileService;
private final ObjectMapper objectMapper;
private final DataRepository dataRepository;
private final DataJdbcRepository tDataJdbcRepository;
public ExcelUploadResultDto uploadData(MultipartFile file){
try (InputStream in = file.getInputStream()) {
List<RowContext> buffer = new ArrayList<>(INSERT_CHUNK_SIZE);
List<Map<String, String>> failRows = new ArrayList<>();
@Transactional
public ExcelUploadResultDto uploadData(MultipartFile file) {
Path dataFilePath = dataFileService.saveListFile(file);
AtomicInteger total = new AtomicInteger();
AtomicInteger success = new AtomicInteger();
AtomicInteger fail = new AtomicInteger();
List<RowContext> buffer = new ArrayList<>(INSERT_CHUNK_SIZE);
List<Map<String, String>> failRows = new ArrayList<>();
final LocalDateTime now = DateTimeUtil.nowKstLocalDateTime();
final String regUser = "system";
AtomicInteger total = new AtomicInteger();
AtomicInteger success = new AtomicInteger();
AtomicInteger fail = new AtomicInteger();
ExcelMapper.readStreaming(in, (row, rowIndex) -> {
total.incrementAndGet();
final LocalDateTime now = DateTimeUtil.nowKstLocalDateTime();
final String regUser = "system";
ParsedRow parsed = parseRowToEntity(row, regUser, now);
if (parsed.reason != null) {
fail.incrementAndGet();
failRows.add(buildFailRow(row, rowIndex, parsed.reason));
return;
}
ExcelMapper.readStreaming(dataFilePath, (row, rowIndex) -> {
total.incrementAndGet();
buffer.add(new RowContext(rowIndex, row, parsed.entity));
ParsedRow parsed = parseRow(row, regUser, now);
if (parsed.reason != null) {
fail.incrementAndGet();
failRows.add(buildFailRow(row, rowIndex, parsed.reason));
return;
}
if (buffer.size() >= INSERT_CHUNK_SIZE) {
flushBuffer(buffer, success, fail, failRows);
}
});
buffer.add(new RowContext(rowIndex, row, parsed.insertRow));
if (!buffer.isEmpty()) {
if (buffer.size() >= INSERT_CHUNK_SIZE) {
flushBuffer(buffer, success, fail, failRows);
}
});
byte[] errorFile = null;
boolean hasErrorFile = !failRows.isEmpty();
if (hasErrorFile) {
errorFile = toXlsx(failRows);
}
return ExcelUploadResultDto.of(total.get(), success.get(), fail.get(), errorFile);
} catch (IOException ex){
throw BusinessException.badRequest("CAN_NOT_OPEN", "파일 스트림을 열 수 없습니다.")
.with("fileName",file.getOriginalFilename())
.with("errorMessage",ex.getMessage());
if (!buffer.isEmpty()) {
flushBuffer(buffer, success, fail, failRows);
}
byte[] errorFile = null;
if (!failRows.isEmpty()) {
errorFile = toXlsx(failRows);
}
return ExcelUploadResultDto.of(total.get(), success.get(), fail.get(), errorFile);
}
private void flushBuffer(
@@ -95,29 +88,30 @@ public class UploadService {
AtomicInteger fail,
List<Map<String, String>> failRows
) {
try {
List<TDataEntity> entities = buffer.stream().map(rc -> rc.entity).toList();
if (buffer.isEmpty()) {
return;
}
dataRepository.saveAll(entities);
success.addAndGet(buffer.size());
} catch (DataAccessException e) {
// 배치가 한 번에 터지면, 개별 저장으로 폴백해서 "어느 행이 문제인지" 잡는다
for (RowContext rc : buffer) {
try {
dataRepository.saveAndFlush(rc.entity);
success.incrementAndGet();
} catch (DataAccessException ex) {
fail.incrementAndGet();
failRows.add(buildFailRow(rc.rawRow, rc.rowIndex, "DB 저장 실패: " + rootMessage(ex)));
}
try {
List<TDataInsertRow> rows = buffer.stream()
.map(RowContext::insertRow)
.toList();
TDataBatchInsertResult result = tDataJdbcRepository.bulkInsert(rows);
success.addAndGet(result.successCount());
for (TDataInsertFailure failure : result.failures()) {
RowContext rc = buffer.get(failure.index());
fail.incrementAndGet();
failRows.add(buildFailRow(rc.rawRow(), rc.rowIndex(), "DB 저장 실패: " + failure.reason()));
}
} finally {
em.clear();
buffer.clear();
}
}
private ParsedRow parseRowToEntity(
private ParsedRow parseRow(
Map<String, String> row,
String regUser,
LocalDateTime regDt
@@ -126,7 +120,6 @@ public class UploadService {
return ParsedRow.fail("빈 행");
}
// sid
String sid = firstNonBlank(row.get(EXCEL_ID), row.get("sid"), row.get("SID"));
if (sid == null) {
return ParsedRow.fail("ID(sid) 누락");
@@ -135,21 +128,16 @@ public class UploadService {
return ParsedRow.fail("ID(sid) 길이 초과(>32): " + sid);
}
// rawData 구성
Map<String, List<DataItemDto>> rawData = new LinkedHashMap<>();
for (Map.Entry<String, String> e : row.entrySet()) {
String key = e.getKey();
if (key == null) continue;
// 제외
if (EXCEL_ID.equals(key)) continue;
//if (EXCEL_YEAR.equals(key)) continue;
//if (EXCEL_PRODUCT.equals(key)) continue;
String v = e.getValue();
if (v == null || v.isBlank()) {
continue; // 빈 값은 rawData에 안 넣음
continue;
}
Double s = parseDoubleLoose(v);
@@ -157,23 +145,33 @@ public class UploadService {
return ParsedRow.fail("숫자 파싱 실패: key=" + key + ", value=" + safe(v));
}
if(key.contains("#")){
key = key.substring(0,key.indexOf("#"));
if (key.contains("#")) {
key = key.substring(0, key.indexOf("#"));
}
rawData.computeIfAbsent(key, k -> new ArrayList<>()).add(DataItemDto.from(null, null, s));
rawData.computeIfAbsent(key, k -> new ArrayList<>())
.add(DataItemDto.from(null, null, s));
}
TDataEntity entity = new TDataEntity();
entity.setSid(sid);
entity.setRawData(rawData);
entity.setRegUser(regUser);
entity.setRegDt(regDt);
String rawDataJson;
try {
rawDataJson = objectMapper.writeValueAsString(rawData);
} catch (JsonProcessingException e) {
return ParsedRow.fail("rawData JSON 직렬화 실패: " + rootMessage(e));
}
return ParsedRow.ok(entity);
TDataInsertRow insertRow = new TDataInsertRow(
UUID.randomUUID().toString(),
sid,
rawDataJson,
regUser,
regDt
);
return ParsedRow.ok(insertRow);
}
private Map<String, String> buildFailRow(Map<String, String> row, int rowIndex, String reason) {
// 에러 엑셀에 행번호/사유를 앞에 고정 컬럼으로 넣고, 원본 컬럼들도 같이 싣는다
Map<String, String> m = new LinkedHashMap<>();
m.put("rowIndex", String.valueOf(rowIndex));
m.put("reason", reason);
@@ -195,26 +193,16 @@ public class UploadService {
private static String rootMessage(Throwable t) {
Throwable cur = t;
while (cur.getCause() != null) cur = cur.getCause();
return cur.getMessage();
}
private void doDebugLog(TDataEntity e){
try {
String json = objectMapper
.writerWithDefaultPrettyPrinter()
.writeValueAsString(e.getRawData());
log.info("rawDataJson={}", json);
} catch (JsonProcessingException ex) {
log.warn("rawData json serialize failed", ex);
while (cur.getCause() != null) {
cur = cur.getCause();
}
return cur.getMessage();
}
private static Double parseDoubleLoose(String s) {
if (s == null) return null;
String t = s.trim();
if (t.isEmpty()) return null;
// 천단위 콤마 제거
t = t.replace(",", "");
try {
return Double.parseDouble(t);
@@ -238,19 +226,19 @@ public class UploadService {
private record RowContext(
int rowIndex,
Map<String, String> rawRow,
TDataEntity entity
TDataInsertRow insertRow
) {}
private record ParsedRow(
TDataEntity entity,
TDataInsertRow insertRow,
String reason
) {
static ParsedRow ok(TDataEntity entity) {
return new ParsedRow(entity, null);
static ParsedRow ok(TDataInsertRow insertRow) {
return new ParsedRow(insertRow, null);
}
static ParsedRow fail(String reason) {
return new ParsedRow(null, reason);
}
}
}
}

View File

@@ -0,0 +1,131 @@
package com.kri.lge.penetration.service.file;
import com.kri.common.datetime.DateTimeUtil;
import com.kri.common.util.CommonUtil;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Optional;
import org.springframework.web.multipart.MultipartFile;
public abstract class AbstractFileService {
protected abstract Path getBaseDir();
protected String extractExtension(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "";
}
int dot = originalFileName.lastIndexOf('.');
if (dot > -1 && dot < originalFileName.length() - 1) {
return originalFileName.substring(dot);
}
return "";
}
protected String extractBaseName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "file";
}
String fileName = java.nio.file.Paths.get(originalFileName).getFileName().toString();
int dot = fileName.lastIndexOf('.');
if (dot > 0) {
return fileName.substring(0, dot);
}
return fileName;
}
protected String buildFileName(MultipartFile file) {
String original = Optional.ofNullable(file.getOriginalFilename()).orElse("file");
return CommonUtil.uuid32() + extractExtension(original);
}
protected String buildFileNameWithTime(MultipartFile file, String prefix) {
String original = Optional.ofNullable(file.getOriginalFilename()).orElse("file");
String baseName = extractBaseName(original);
String ext = extractExtension(original);
String timestamp = DateTimeUtil.nowKstLocalDateTime()
.format(java.time.format.DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"));
return prefix + "_" + timestamp + "_" + baseName + ext;
}
protected Path createDirectories(Path dir) throws IOException {
return Files.createDirectories(dir.toAbsolutePath().normalize());
}
protected String saveFile(Path dir, MultipartFile file) throws IOException {
if (file == null || file.isEmpty()) {
throw new IllegalArgumentException("업로드할 파일이 없습니다.");
}
Path normalizedDir = createDirectories(dir);
String fileName = buildFileName(file);
Path target = normalizedDir.resolve(fileName).normalize();
if (!target.startsWith(normalizedDir)) {
throw new IllegalStateException("잘못된 파일 경로입니다.");
}
file.transferTo(target.toFile());
return fileName;
}
protected void saveFile(Path dir, MultipartFile file, String fileName) throws IOException {
if (file == null || file.isEmpty()) {
throw new IllegalArgumentException("업로드할 파일이 없습니다.");
}
Path normalizedDir = createDirectories(dir);
Path target = normalizedDir.resolve(fileName).normalize();
if (!target.startsWith(normalizedDir)) {
throw new IllegalStateException("잘못된 파일 경로입니다.");
}
file.transferTo(target.toFile());
}
protected void moveFile(Path sourceDir, Path targetDir, String fileName) throws IOException {
if (fileName == null || fileName.isBlank()) {
return;
}
Path normalizedSourceDir = sourceDir.toAbsolutePath().normalize();
Path normalizedTargetDir = createDirectories(targetDir);
Path source = normalizedSourceDir.resolve(fileName).normalize();
Path target = normalizedTargetDir.resolve(fileName).normalize();
if (!source.startsWith(normalizedSourceDir) || !target.startsWith(normalizedTargetDir)) {
throw new IllegalStateException("잘못된 파일 경로입니다.");
}
if (!Files.exists(source)) {
return;
}
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
}
protected void deleteFile(Path dir, String fileName) throws IOException {
if (fileName == null || fileName.isBlank()) {
return;
}
Path normalizedDir = dir.toAbsolutePath().normalize();
Path target = normalizedDir.resolve(fileName).normalize();
if (!target.startsWith(normalizedDir)) {
throw new IllegalStateException("잘못된 파일 경로입니다.");
}
Files.deleteIfExists(target);
}
}

View File

@@ -0,0 +1,36 @@
package com.kri.lge.penetration.service.file;
import com.kri.common.util.CommonUtil;
import com.kri.common.web.BusinessException;
import com.kri.lge.penetration.config.FileStorageProperties;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
@Service
@RequiredArgsConstructor
public class DataFileService extends AbstractFileService {
private final FileStorageProperties properties;
@Override
protected Path getBaseDir() {
return Paths.get(properties.getDataBaseDir()).toAbsolutePath().normalize();
}
public Path saveListFile(MultipartFile file) {
String newFileName = buildFileNameWithTime(file, CommonUtil.uuid32());
Path dir = getBaseDir().toAbsolutePath().normalize();
Path savedPath = dir.resolve(newFileName).toAbsolutePath().normalize();
try {
saveFile(dir, file, newFileName);
return savedPath;
} catch (IOException e) {
throw BusinessException.internalError("FAIL_DATA_UPLOAD", e.getMessage());
}
}
}

View File

@@ -20,6 +20,6 @@ spring:
logging:
level:
org.hibernate.SQL: DEBUG
org.hibernate.orm.jdbc.bind: TRACE
#org.hibernate.orm.jdbc.bind: TRACE
#org.hibernate.orm.jdbc.extract: TRACE
org.hibernate.engine.jdbc.batch.internal: DEBUG

View File

@@ -47,6 +47,9 @@ kri:
version: "v1.0.0"
base-package: "com.kri.lge.penetration"
file:
data-base-dir: ./upload-file/data
common:
web:
enabled: true # 공통 Web 구성 사용 여부 (기본 true)