diff --git a/src/main/java/com/kri/kart/server/controller/ExportController.java b/src/main/java/com/kri/kart/server/controller/ExportController.java index f97c0b0..44bc782 100644 --- a/src/main/java/com/kri/kart/server/controller/ExportController.java +++ b/src/main/java/com/kri/kart/server/controller/ExportController.java @@ -14,6 +14,9 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import jakarta.validation.Valid; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.nio.file.Files; import java.nio.charset.StandardCharsets; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -93,13 +96,28 @@ public class ExportController { @PostMapping( value = "/project/{projectType}/{pkey}/export/data", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) - public ResponseEntity data( + public void data( @PathVariable ProjectType projectType, @PathVariable String pkey, @Valid @RequestBody RawDataExportRequestDto request, - @Parameter(hidden = true) RequestUserContext ctx) { - return download( - exportService.data(new ProjectRefDto(projectType, pkey), ctx.getUserRole(), request)); + @Parameter(hidden = true) RequestUserContext ctx, HttpServletResponse response) throws IOException { + ProjectRefDto project = new ProjectRefDto(projectType, pkey); + try (var file = exportService.prepareData(project, ctx.getUserRole(), request)) { + String fileName = FileUtil.extractFileNameFromPath(file.fileName()) + .replaceAll("[\\p{Cntrl}\\\\/:*?\"<>|]", "_"); + response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, + ContentDisposition.attachment().filename(fileName, StandardCharsets.UTF_8).build().toString()); + response.setHeader(HttpHeaders.CACHE_CONTROL, "no-store"); + response.setContentLengthLong(Files.size(file.path())); + Files.copy(file.path(), response.getOutputStream()); + } catch (IOException | RuntimeException error) { + // Before any file bytes are committed, preserve the normal JSON error response. + if (!response.isCommitted()) { + response.reset(); + } + throw error; + } } private ResponseEntity download(ExportFileDto file) { diff --git a/src/main/java/com/kri/kart/server/dto/export/ExportTempFileDto.java b/src/main/java/com/kri/kart/server/dto/export/ExportTempFileDto.java new file mode 100644 index 0000000..3b27d9c --- /dev/null +++ b/src/main/java/com/kri/kart/server/dto/export/ExportTempFileDto.java @@ -0,0 +1,13 @@ +package com.kri.kart.server.dto.export; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Completed download file owned by the caller until close. */ +public record ExportTempFileDto(String fileName, Path path) implements AutoCloseable { + @Override + public void close() throws IOException { + Files.deleteIfExists(path); + } +} diff --git a/src/main/java/com/kri/kart/server/persistence/mapper/DerivedMapper.java b/src/main/java/com/kri/kart/server/persistence/mapper/DerivedMapper.java index 6aeefa2..5a9c298 100644 --- a/src/main/java/com/kri/kart/server/persistence/mapper/DerivedMapper.java +++ b/src/main/java/com/kri/kart/server/persistence/mapper/DerivedMapper.java @@ -14,6 +14,8 @@ public interface DerivedMapper { String lockProject(@Param("pkey") String pkey); + boolean projectExists(@Param("pkey") String pkey); + List definitions(@Param("pkey") String pkey); DerivedDefinitionDto find(@Param("pkey") String pkey, @Param("qid") String qid); diff --git a/src/main/java/com/kri/kart/server/persistence/mapper/ExportMapper.java b/src/main/java/com/kri/kart/server/persistence/mapper/ExportMapper.java index 3816a4a..bd1f8dc 100644 --- a/src/main/java/com/kri/kart/server/persistence/mapper/ExportMapper.java +++ b/src/main/java/com/kri/kart/server/persistence/mapper/ExportMapper.java @@ -7,6 +7,7 @@ import com.kri.kart.server.dto.export.ExportTitleDto; import java.time.LocalDate; import java.util.List; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.session.ResultHandler; public interface ExportMapper { String additionalColumns(@Param("pkey") String pkey); @@ -19,10 +20,11 @@ public interface ExportMapper { String surveyType(@Param("pkey") String pkey); - List data( + void data( @Param("pkey") String pkey, @Param("type") String type, @Param("startDate") LocalDate startDate, @Param("endDate") LocalDate endDate, - @Param("includeInterviewer") Boolean includeInterviewer); + @Param("includeInterviewer") Boolean includeInterviewer, + ResultHandler handler); } diff --git a/src/main/java/com/kri/kart/server/persistence/repository/DerivedRepository.java b/src/main/java/com/kri/kart/server/persistence/repository/DerivedRepository.java index 4fb02af..444ab8f 100644 --- a/src/main/java/com/kri/kart/server/persistence/repository/DerivedRepository.java +++ b/src/main/java/com/kri/kart/server/persistence/repository/DerivedRepository.java @@ -27,6 +27,10 @@ public class DerivedRepository { .lockProject(project.pkey()); } + public boolean projectExists(ProjectRefDto project) { + return surveySessionProvider.mapper(project, DerivedMapper.class).projectExists(project.pkey()); + } + public List definitions(ProjectRefDto project) { return surveySessionProvider.mapper(project, DerivedMapper.class) .definitions(project.pkey()); diff --git a/src/main/java/com/kri/kart/server/persistence/repository/ExportRepository.java b/src/main/java/com/kri/kart/server/persistence/repository/ExportRepository.java index ad9a119..d291323 100644 --- a/src/main/java/com/kri/kart/server/persistence/repository/ExportRepository.java +++ b/src/main/java/com/kri/kart/server/persistence/repository/ExportRepository.java @@ -9,6 +9,7 @@ import com.kri.kart.server.dto.project.ProjectRefDto; import com.kri.kart.server.persistence.SurveySessionProvider; import com.kri.kart.server.persistence.mapper.ExportMapper; import java.util.List; +import java.util.function.Consumer; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Repository; @@ -42,14 +43,14 @@ public class ExportRepository { return surveySessionProvider.mapper(project, ExportMapper.class).surveyType(project.pkey()); } - public List data( - ProjectRefDto project, RawDataExportRequestDto request, boolean includeInterviewer) { - return surveySessionProvider.mapper(project, ExportMapper.class) + public void data( + ProjectRefDto project, RawDataExportRequestDto request, boolean includeInterviewer, Consumer consumer) { + surveySessionProvider.mapper(project, ExportMapper.class) .data( project.pkey(), request.getType(), request.getStartDate(), request.getEndDate() == null ? request.getStartDate() : request.getEndDate(), - includeInterviewer); + includeInterviewer, context -> consumer.accept(context.getResultObject())); } } diff --git a/src/main/java/com/kri/kart/server/service/DerivedManagementService.java b/src/main/java/com/kri/kart/server/service/DerivedManagementService.java index a1bffd3..919ef2a 100644 --- a/src/main/java/com/kri/kart/server/service/DerivedManagementService.java +++ b/src/main/java/com/kri/kart/server/service/DerivedManagementService.java @@ -6,12 +6,15 @@ import com.kri.common.feature.web.BusinessException; import com.kri.common.feature.web.RequestUserContext; import com.kri.kart.server.common.enums.ProjectType; import com.kri.kart.server.dto.derived.DerivedCalculationDto; +import com.kri.kart.server.dto.derived.DerivedDataDto; import com.kri.kart.server.dto.derived.DerivedDefinitionDto; import com.kri.kart.server.dto.derived.DerivedSaveRequestDto; import com.kri.kart.server.dto.project.ProjectRefDto; +import com.kri.kart.server.dto.question.QuestionDto; import com.kri.kart.server.persistence.repository.DerivedRepository; import com.kri.kart.server.service.derived.DerivedComputationService; import com.kri.kart.server.service.derived.DerivedRecalculationGuard; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -156,23 +159,62 @@ public class DerivedManagementService { if (idkey == null || idkey.isBlank() || idkey.length() > 512) { throw BusinessException.badRequest("응답자 ID를 확인해 주세요."); } - return derivedRecalculationGuard.execute(project, () -> transaction( - project, - () -> { - requireProject(project); - // Legacy creates an empty DERIVED_DATA row even if there is no eligible - // response. + return derivedRecalculationGuard.execute(project, idkey, () -> { + if (!derivedRepository.projectExists(project)) { + throw BusinessException.notFound("PROJECT_NOT_FOUND", "프로젝트를 찾을 수 없습니다."); + } + List definitions = derivedRepository.definitions(project); + List derivedIds = derivedIds(definitions); + List questions = definitions.isEmpty() ? List.of() : questionService.list(project, 1001); + List data = definitions.isEmpty() ? List.of() + : derivedRepository.respondentResponses(project, idkey); + if (data.size() > 1) { + // Legacy joins can return multiple rows for one ID. Preserve their database reread + // semantics rather than assuming uniqueness or collapsing results in memory. + return transaction(project, () -> { derivedRepository.ensureRespondent(project, idkey); - var definitions = derivedRepository.definitions(project); - var derivedIds = derivedIds(definitions); int matched = 0; - for (var definition : definitions) { - matched += calculate(project, definition, idkey, derivedIds); + for (DerivedDefinitionDto definition : definitions) { + matched += calculate(project, definition, idkey, derivedIds, questions); } return new DerivedCalculationDto(definitions.size(), matched); - })); + }); + } + definitions.forEach(definition -> validateId(definition.getQid())); + var results = derivedComputationService.calculateRespondent(definitions, questions, data, derivedIds); + List writes = new ArrayList<>(); + int matched = 0; + for (int variable = 0; variable < definitions.size(); variable++) { + List groups = new ArrayList<>(); + for (var group : results.get(variable)) { + groups.add(group.isEmpty() ? null : JsonUtil.toJson(objectMapper, group)); + if (!group.isEmpty()) { + matched++; + } + } + writes.add(new RespondentWrite(definitions.get(variable).getQid(), groups)); + } + int matchedRespondents = matched; + return transaction(project, () -> { + // Keep the original empty-row contract even when there are no definitions or responses. + derivedRepository.ensureRespondent(project, idkey); + for (RespondentWrite write : writes) { + if (write.groups().isEmpty()) { + derivedRepository.clearRespondentData(project, write.qid(), idkey); + } else { + for (String group : write.groups()) { + // JSON_MODIFY updates only this variable on the current row; never replace DATJSON. + derivedRepository.saveData(project, write.qid(), idkey, group); + } + } + } + return new DerivedCalculationDto(definitions.size(), matchedRespondents); + }); + }); } + private record RespondentWrite(String qid, List groups) {} + /** Stored derived variable IDs; uploaded response columns with these IDs are not imported. */ public List derivedIds(ProjectRefDto project) { return derivedIds(derivedRepository.definitions(project)); @@ -205,6 +247,12 @@ public class DerivedManagementService { DerivedDefinitionDto definition, String idkey, List derivedIds) { + return calculate(project, definition, idkey, derivedIds, null); + } + + private int calculate( + ProjectRefDto project, DerivedDefinitionDto definition, String idkey, + List derivedIds, List questions) { validateId(definition.getQid()); if (idkey == null) { derivedRepository.clearData(project, definition.getQid()); @@ -217,7 +265,8 @@ public class DerivedManagementService { : derivedRepository.respondentResponses(project, idkey); var results = derivedComputationService.calculate( - definition, questionService.list(project, idkey == null ? 0 : 1001), data, derivedIds); + definition, questions == null ? questionService.list(project, idkey == null ? 0 : 1001) : questions, + data, derivedIds); int matched = 0; for (int i = 0; i < data.size(); i++) { var group = results.get(i); diff --git a/src/main/java/com/kri/kart/server/service/ExportService.java b/src/main/java/com/kri/kart/server/service/ExportService.java index b7bd9f6..6e34863 100644 --- a/src/main/java/com/kri/kart/server/service/ExportService.java +++ b/src/main/java/com/kri/kart/server/service/ExportService.java @@ -10,6 +10,7 @@ import com.kri.kart.server.common.consts.KartConstant; import com.kri.kart.server.dto.export.ExportDataDto; import com.kri.kart.server.dto.export.ExportDerivedHeaderDto; import com.kri.kart.server.dto.export.ExportFileDto; +import com.kri.kart.server.dto.export.ExportTempFileDto; import com.kri.kart.server.dto.export.ExportGuideDto; import com.kri.kart.server.dto.export.ExportHeaderDto; import com.kri.kart.server.dto.export.ExportTitleDto; @@ -20,6 +21,7 @@ import com.kri.kart.server.persistence.repository.ExportRepository; import com.kri.kart.server.persistence.repository.QuestionRepository; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.format.DateTimeFormatter; @@ -128,8 +130,26 @@ public class ExportService { return rows; } - public ExportFileDto data( - ProjectRefDto project, Integer userRole, RawDataExportRequestDto request) { + public ExportTempFileDto prepareData( + ProjectRefDto project, Integer userRole, RawDataExportRequestDto request) throws IOException { + Path path = Files.createTempFile("kart-data-export-", ".xlsx"); + try { + try (OutputStream output = Files.newOutputStream(path)) { + data(project, userRole, request, output); + } + return new ExportTempFileDto(dataFileName(project), path); + } catch (IOException | RuntimeException | Error error) { + try { + Files.deleteIfExists(path); + } catch (IOException cleanupError) { + error.addSuppressed(cleanupError); + } + throw error; + } + } + + void data( + ProjectRefDto project, Integer userRole, RawDataExportRequestDto request, OutputStream output) throws IOException { if (!List.of("COMPLETE", "INCOMPLETE").contains(Objects.toString(request.getType(), "")) || (request.getStartDate() == null && request.getEndDate() != null) || (request.getStartDate() != null @@ -138,37 +158,49 @@ public class ExportService { throw BusinessException.badRequest("다운로드 유형과 날짜 범위를 확인해 주세요."); } Map headers = headers(project, userRole); - List> rows = new ArrayList<>(); - rows.add(headers); // Row 1 contains storage keys, row 2 contains variable aliases. boolean includeInterviewer = "2".equals(exportRepository.surveyType(project)); - for (ExportDataDto response : exportRepository.data(project, request, includeInterviewer)) { - String dat = response.getDat(); - if (dat == null || dat.isEmpty()) { - continue; // SQL STUFF returns NULL when all response groups are empty. + try (var workbook = ExcelMapper.createStreamingWorkbook(100)) { + try (var writer = ExcelMapper.openStreamingMapWriter(workbook, new ArrayList<>(headers.keySet()), "DATA", 1)) { + writer.append(headers); + exportRepository.data(project, request, includeInterviewer, response -> { + String dat = response.getDat(); + if (dat == null || dat.isEmpty()) { + return; + } + JsonNode values = jsonObject(dat.replaceAll("\\\\U\\d*", "")); + Map row = new LinkedHashMap<>(); + for (String key : headers.keySet()) { + String value = responseValue(response, values, key, includeInterviewer); + if (value != null && !value.isEmpty()) { + row.put(key, value); + } + } + writer.append(row); + }); } - // Preserve legacy ReplaceUnicodeEmpty after SQL UPPER/FOR XML serialization. - JsonNode values = jsonObject(dat.replaceAll("\\\\U\\d*", "")); - Map row = new LinkedHashMap<>(); - for (String key : headers.keySet()) { - row.put(key, responseValue(response, values, key, includeInterviewer)); + // Metadata is bounded by the questionnaire; respondent rows are never accumulated. + try (var writer = ExcelMapper.openStreamingMapWriter(workbook, + List.of("문항 타입", "문항 / 보기번호", "내용", "VALUE LABEL", "VARIABLE LABEL"), "GUIDE", 1)) { + for (ExportGuideDto guide : guideRows(project)) { + Map row = new LinkedHashMap<>(); + row.put("문항 타입", guide.questionType()); + row.put("문항 / 보기번호", guide.variable()); + row.put("내용", guide.content()); + row.put("VALUE LABEL", guide.valueLabel()); + row.put("VARIABLE LABEL", guide.variableLabel()); + row.values().removeIf(value -> value == null || value.isEmpty()); + writer.append(row); + } } - rows.add(row); - } - Workbook workbook = ExcelMapper.writeFromMapList(rows, "DATA"); - try { - ExcelMapper.appendSheetToWorkbook( - workbook, guideRows(project), ExportGuideDto.class, "GUIDE", 1); - return file("DATA", project, workbook); - } catch (RuntimeException exception) { - try { - workbook.close(); - } catch (IOException closeException) { - exception.addSuppressed(closeException); - } - throw exception; + workbook.write(output); } } + public String dataFileName(ProjectRefDto project) { + return "[DATA]" + project.pkey() + "_" + + DateTimeUtil.nowKstLocalDateTime().format(FILE_TIMESTAMP) + ".xlsx"; + } + private String responseValue( ExportDataDto response, JsonNode values, String key, boolean includeInterviewer) { // Fixed SQL columns take precedence; IDKEY remains the value embedded in DAT. diff --git a/src/main/java/com/kri/kart/server/service/derived/DerivedComputationService.java b/src/main/java/com/kri/kart/server/service/derived/DerivedComputationService.java index 8944398..2f18c97 100644 --- a/src/main/java/com/kri/kart/server/service/derived/DerivedComputationService.java +++ b/src/main/java/com/kri/kart/server/service/derived/DerivedComputationService.java @@ -134,6 +134,37 @@ public class DerivedComputationService { return results; } + /** Reuse one respondent snapshot, feeding preceding variable results into later conditions. */ + public List> calculateRespondent( + List definitions, List questions, + List data, Collection derivedIds) { + List working = new ArrayList<>(); + for (DerivedDataDto row : data) { + DerivedDataDto copy = new DerivedDataDto(); + copy.setIdkey(row.getIdkey()); + copy.setDatjson(row.getDatjson()); + copy.setDerivedJson(row.getDerivedJson()); + working.add(copy); + } + List> prepared = new ArrayList<>(); + for (DerivedDefinitionDto definition : definitions) { + List groups = calculate(definition, questions, working, derivedIds); + prepared.add(groups); + for (int row = 0; row < working.size(); row++) { + DerivedDataDto response = working.get(row); + ObjectNode derived = response.getDerivedJson() == null + ? objectMapper.createObjectNode() : object(response.getDerivedJson()); + if (groups.get(row).isEmpty()) { + derived.remove(definition.getQid()); + } else { + derived.set(definition.getQid(), groups.get(row)); + } + response.setDerivedJson(JsonUtil.toJson(objectMapper, derived)); + } + } + return prepared; + } + private void flatten(ObjectNode grouped, Map flat) { grouped .fields() diff --git a/src/main/java/com/kri/kart/server/service/derived/DerivedRecalculationGuard.java b/src/main/java/com/kri/kart/server/service/derived/DerivedRecalculationGuard.java index d34c6f8..59bdd7f 100644 --- a/src/main/java/com/kri/kart/server/service/derived/DerivedRecalculationGuard.java +++ b/src/main/java/com/kri/kart/server/service/derived/DerivedRecalculationGuard.java @@ -1,56 +1,41 @@ package com.kri.kart.server.service.derived; import com.kri.common.feature.web.BusinessException; -import com.kri.kart.server.common.enums.ProjectType; -import com.kri.kart.server.config.KartDatabaseProperties; import com.kri.kart.server.dto.project.ProjectRefDto; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; -/** Wait before borrowing JDBC connections; database locks still enforce cross-instance safety. */ +/** Serialize duplicate respondent recalculations before reading their response snapshots. */ @Component @ConditionalOnProperty(prefix = "kri.kart.database", name = "enabled", havingValue = "true") public class DerivedRecalculationGuard { - private final ConcurrentHashMap projects = new ConcurrentHashMap<>(); - private final Semaphore kon; - private final Semaphore konkiwi; + private final ConcurrentHashMap respondents = new ConcurrentHashMap<>(); private final long waitNanos; - @Autowired - public DerivedRecalculationGuard(KartDatabaseProperties properties) { - this(properties, TimeUnit.SECONDS.toNanos(10)); + public DerivedRecalculationGuard() { + this(TimeUnit.SECONDS.toNanos(10)); } - DerivedRecalculationGuard(KartDatabaseProperties properties, long waitNanos) { - // Reserve one connection for access checks and other operations when pool size permits. - this.kon = new Semaphore(Math.max(1, properties.getKon().getMaximumPoolSize() - 1), true); - this.konkiwi = new Semaphore(Math.max(1, properties.getKonkiwi().getMaximumPoolSize() - 1), true); + DerivedRecalculationGuard(long waitNanos) { this.waitNanos = waitNanos; } - public T execute(ProjectRefDto project, Supplier action) { - long deadline = System.nanoTime() + waitNanos; - ProjectGate gate = projects.compute(project, (key, current) -> { - ProjectGate selected = current == null ? new ProjectGate() : current; + public T execute(ProjectRefDto project, String idkey, Supplier action) { + RespondentKey respondent = new RespondentKey(project, idkey); + RespondentGate gate = respondents.compute(respondent, (key, current) -> { + RespondentGate selected = current == null ? new RespondentGate() : current; selected.users++; return selected; }); - Semaphore source = project.projectType() == ProjectType.KON ? kon : konkiwi; - boolean projectAcquired = false; - boolean sourceAcquired = false; + boolean acquired = false; try { - projectAcquired = gate.permit.tryAcquire(remaining(deadline), TimeUnit.NANOSECONDS); - if (!projectAcquired) { - throw busy(); - } - sourceAcquired = source.tryAcquire(remaining(deadline), TimeUnit.NANOSECONDS); - if (!sourceAcquired) { + acquired = gate.permit.tryAcquire(waitNanos, TimeUnit.NANOSECONDS); + if (!acquired) { throw busy(); } return action.get(); @@ -58,29 +43,24 @@ public class DerivedRecalculationGuard { Thread.currentThread().interrupt(); throw busy(); } finally { - if (sourceAcquired) { - source.release(); - } - if (projectAcquired) { + if (acquired) { gate.permit.release(); } - projects.computeIfPresent(project, (key, current) -> --current.users == 0 ? null : current); + respondents.computeIfPresent(respondent, (key, current) -> --current.users == 0 ? null : current); } } - private long remaining(long deadline) { - return Math.max(0, deadline - System.nanoTime()); - } - private BusinessException busy() { return BusinessException.of( HttpStatus.SERVICE_UNAVAILABLE, "DERIVED_RECALCULATION_BUSY", "파생변수 재계산 요청이 많습니다. 잠시 후 다시 요청해 주세요."); } - private static final class ProjectGate { + private record RespondentKey(ProjectRefDto project, String idkey) {} + + private static final class RespondentGate { private final Semaphore permit = new Semaphore(1, true); - // Accessed only inside ConcurrentHashMap.compute/computeIfPresent for the project. + // Accessed only inside ConcurrentHashMap.compute/computeIfPresent for the respondent. private int users; } } diff --git a/src/main/resources/mybatis/mapper/DerivedMapper.xml b/src/main/resources/mybatis/mapper/DerivedMapper.xml index 84f3858..74f7327 100644 --- a/src/main/resources/mybatis/mapper/DerivedMapper.xml +++ b/src/main/resources/mybatis/mapper/DerivedMapper.xml @@ -36,6 +36,9 @@ + SELECT JSON_VALUE(PROJECTJSON, '$.must.surveytype') FROM PROJECT WHERE PKEY = #{pkey} -