261023_01_ENH: 파생변수 동일 ID 수정 시 중복 검사 제외 및 데이터 Import 파생변수 열 제외·업로드 후 재계산
This commit is contained in:
@@ -145,6 +145,7 @@
|
||||
|
||||
- Derived create/update/delete and single/all recalculate routes are under /project/{projectType}/{pkey}/derived. Preserve saved QUESTIONJSON/DERIVEDJSON and original single/multi/ELSE/open-num response keys; parse legacy conditions through shared service/statistics/LegacyFilterService and compute with StatisticsFilterService. Unsupported SQL grammar is explicit failure, not SQL execution. Quota retains QUOTA_FILTER_INVALID at its boundary.
|
||||
- Use KON/KONKIWI-specific TransactionTemplate with project locking for definition/data writes and order changes. Preserve other JSON groups with bound JSON_MODIFY paths and original GETDATE timestamp behavior. actor/team come from RequestUserContext, logs through LogService. Return IDs/boolean/counts; frontend owns messages. Shared DB execution and ordinary role authorization are separate.
|
||||
- Per user decision (2026-10-23), derived values take precedence over raw DATA keys with the same derived ID: derived calculation ignores raw DATA keys of every stored derived ID. Statistics/Export SQL is unchanged (approach C2: keep such keys out of DATA instead of rewriting read queries). An update whose variable ID equals originId (case-insensitive) skips the duplicate check; renames and creates keep the full check.
|
||||
- Aggregate Excel POST routes are /project/{projectType}/{pkey}/export/excel/table and /quota/{projectType}/{pkey}/export/excel. Accept typed ordered tables/turn and legacy cell display/span strings, retaining frontend N/% presentation choice; do not requery raw data. Keep original workbook layout, multiple quota sheets and complete/target output. Link scopes must allow only their matching aggregate paths.
|
||||
- Use KriCommon ExcelMapper then format with current POI, without new dependencies. Wordcloud input is validated PNG Base64 from frontend, preserving original 1000x512 position; no SVG or external URL loading. Use static measured Malgun Gothic glyph advances to avoid platform autoSize changes. Compare original EPPlus workbooks: cells/styles/merged ranges/row sizes exact, column rounding within one 1/256-character unit and image anchor rounding within one pixel. Actual production files, shaping/emoji, frontend PNG/render/download and live SQL checks remain separate.
|
||||
|
||||
@@ -158,7 +159,7 @@
|
||||
|
||||
- Implement POST /project/{projectType}/{pkey}/import/title, /question and /data as multipart file uploads. Return processedRows/affectedRows/removedRows/archivedRows; no presentation messages. Common JWT is required; the existing External allowlist denies all imports. Ordinary role/team restrictions stay deferred.
|
||||
- Title upload resets eligible explain values, then updates only explain by pkey/QID; variable/title are ignored, unknown IDs no-op and duplicate IDs are sequential. Question upload resets QUESTION/QOPTION/QUOTASET/QUOTA(original QSORDER condition) and STRUCTURE before sequential Q1/Qorder creation; retain existing dataset/option quirks and JSON strings in QINFO/LINFO, never introduce JSON_QUERY around those string parameters. Per user decision (2026-10-22), O/N questions with no input rows (none, attribute-only or blank titles) get one open input {label:"", option all/num, must 1} instead of failing; other types still require choices.
|
||||
- Data upload deletes UPLOAD responses, moves actual completed responses to DEL_+PKEY, and inserts UPLOAD rows; do not mutate TEST/incomplete/other projects/DERIVED_DATA. Preserve two header rows, six distinct fixed-group skip, multi pull-up keys including duplicates, INDEX and midpoint/final timestamps, empty defaults and original text normalization.
|
||||
- Data upload deletes UPLOAD responses, moves actual completed responses to DEL_+PKEY, and inserts UPLOAD rows; do not mutate TEST/incomplete/other projects. Per user decision (2026-10-23), columns equal to a stored derived ID or starting with "{ID}_" (case-insensitive) are not uploaded, and after all rows are inserted every derived variable is recalculated in the same KON/KONKIWI transaction (no role check or extra SiteLog; failure rolls back the upload). Preserve two header rows, six distinct fixed-group skip, multi pull-up keys including duplicates, INDEX and midpoint/final timestamps, empty defaults and original text normalization.
|
||||
- Parse/convert the entire file before destructive SQL; all changes use the selected KON/KONKIWI transaction with a project lock and rollback. SQL values/JSON paths are bound. No shared DB execution or schema changes are authorized.
|
||||
- Preserve xlsx first-sheet/positional columns/blank rows/cached formulas and csv/txt simple delimiter plus EUC-KR/BOM behavior. Use Common ExcelUtil; its map reader drops rows and changes positional/formula rules, so a domain-specific POI reader is necessary. Existing 20MB limit applies.
|
||||
- Keep original file archives under kri.kart.upload.directory (./upload_data, Git ignored) and Upload File logging through LogService with authenticated context. Archives and best-effort audit attempts may remain after database rollback; do not expose stored paths or raw content on parse errors.
|
||||
|
||||
@@ -55,7 +55,7 @@ configurations.configureEach {
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation 'com.kri.common:starter:beta.1.6.9'
|
||||
implementation 'com.kri.common:starter:beta.1.7.0'
|
||||
implementation 'org.apache.poi:poi-ooxml:5.5.1'
|
||||
implementation "kr.pe.bab2min:kiwi-java:${kiwiVersion}:${kiwiPlatform}@jar"
|
||||
|
||||
|
||||
@@ -68,7 +68,9 @@ public class DerivedManagementService {
|
||||
if (original != null) {
|
||||
requireTeam(original, ctx);
|
||||
}
|
||||
if (derivedRepository.duplicate(project, definition.getQid(), originId)) {
|
||||
// An update that keeps its ID cannot collide; raw DATA keys with that ID are ignored.
|
||||
boolean sameId = originId != null && originId.equalsIgnoreCase(definition.getQid());
|
||||
if (!sameId && derivedRepository.duplicate(project, definition.getQid(), originId)) {
|
||||
throw BusinessException.conflict("DERIVED_ID_DUPLICATE", "이미 사용 중인 변수 ID입니다.");
|
||||
}
|
||||
if (derivedRepository.save(
|
||||
@@ -79,7 +81,7 @@ public class DerivedManagementService {
|
||||
if (original != null && !original.getQid().equals(definition.getQid())) {
|
||||
derivedRepository.clearData(project, original.getQid());
|
||||
}
|
||||
calculate(project, definition);
|
||||
calculate(project, definition, null, derivedIds(derivedRepository.definitions(project)));
|
||||
var content = objectMapper.createObjectNode();
|
||||
content.set("NEW", objectMapper.valueToTree(definition));
|
||||
if (original != null) {
|
||||
@@ -129,9 +131,11 @@ public class DerivedManagementService {
|
||||
qid == null
|
||||
? derivedRepository.definitions(project)
|
||||
: List.of(requireDefinition(project, qid));
|
||||
var derivedIds =
|
||||
derivedIds(qid == null ? definitions : derivedRepository.definitions(project));
|
||||
int matched = 0;
|
||||
for (var definition : definitions) {
|
||||
matched += calculate(project, definition);
|
||||
matched += calculate(project, definition, null, derivedIds);
|
||||
}
|
||||
if (qid == null) {
|
||||
logService.writeSiteLog(
|
||||
@@ -156,19 +160,47 @@ public class DerivedManagementService {
|
||||
// response.
|
||||
derivedRepository.ensureRespondent(project, idkey);
|
||||
var definitions = derivedRepository.definitions(project);
|
||||
var derivedIds = derivedIds(definitions);
|
||||
int matched = 0;
|
||||
for (var definition : definitions) {
|
||||
matched += calculate(project, definition, idkey);
|
||||
matched += calculate(project, definition, idkey, derivedIds);
|
||||
}
|
||||
return new DerivedCalculationDto(definitions.size(), matched);
|
||||
});
|
||||
}
|
||||
|
||||
private int calculate(ProjectRefDto project, DerivedDefinitionDto definition) {
|
||||
return calculate(project, definition, null);
|
||||
/** Stored derived variable IDs; uploaded response columns with these IDs are not imported. */
|
||||
public List<String> derivedIds(ProjectRefDto project) {
|
||||
return derivedIds(derivedRepository.definitions(project));
|
||||
}
|
||||
|
||||
private int calculate(ProjectRefDto project, DerivedDefinitionDto definition, String idkey) {
|
||||
/**
|
||||
* Recalculates every definition after a data upload. It joins the caller's KON/KONKIWI
|
||||
* transaction, so a failure rolls back the upload; no role check or extra audit log applies.
|
||||
*/
|
||||
public DerivedCalculationDto recalculateAfterImport(ProjectRefDto project) {
|
||||
return transaction(
|
||||
project,
|
||||
() -> {
|
||||
var definitions = derivedRepository.definitions(project);
|
||||
var derivedIds = derivedIds(definitions);
|
||||
int matched = 0;
|
||||
for (var definition : definitions) {
|
||||
matched += calculate(project, definition, null, derivedIds);
|
||||
}
|
||||
return new DerivedCalculationDto(definitions.size(), matched);
|
||||
});
|
||||
}
|
||||
|
||||
private List<String> derivedIds(List<DerivedDefinitionDto> definitions) {
|
||||
return definitions.stream().map(DerivedDefinitionDto::getQid).toList();
|
||||
}
|
||||
|
||||
private int calculate(
|
||||
ProjectRefDto project,
|
||||
DerivedDefinitionDto definition,
|
||||
String idkey,
|
||||
List<String> derivedIds) {
|
||||
validateId(definition.getQid());
|
||||
if (idkey == null) {
|
||||
derivedRepository.clearData(project, definition.getQid());
|
||||
@@ -181,7 +213,7 @@ public class DerivedManagementService {
|
||||
: derivedRepository.respondentResponses(project, idkey);
|
||||
var results =
|
||||
derivedComputationService.calculate(
|
||||
definition, questionService.list(project, idkey == null ? 0 : 1001), data);
|
||||
definition, questionService.list(project, idkey == null ? 0 : 1001), data, derivedIds);
|
||||
int matched = 0;
|
||||
for (int i = 0; i < data.size(); i++) {
|
||||
var group = results.get(i);
|
||||
|
||||
@@ -33,6 +33,7 @@ public class ImportService {
|
||||
private final QuestionImportService questionImportService;
|
||||
private final ResponseImportService responseImportService;
|
||||
private final UploadArchiveService uploadArchiveService;
|
||||
private final DerivedManagementService derivedManagementService;
|
||||
private final LogService logService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final PlatformTransactionManager konTransactionManager;
|
||||
@@ -45,6 +46,7 @@ public class ImportService {
|
||||
QuestionImportService questionImportService,
|
||||
ResponseImportService responseImportService,
|
||||
UploadArchiveService uploadArchiveService,
|
||||
DerivedManagementService derivedManagementService,
|
||||
LogService logService,
|
||||
ObjectMapper objectMapper,
|
||||
@Qualifier("konTransactionManager") PlatformTransactionManager konTransactionManager,
|
||||
@@ -56,6 +58,7 @@ public class ImportService {
|
||||
this.questionImportService = questionImportService;
|
||||
this.responseImportService = responseImportService;
|
||||
this.uploadArchiveService = uploadArchiveService;
|
||||
this.derivedManagementService = derivedManagementService;
|
||||
this.logService = logService;
|
||||
this.objectMapper = objectMapper;
|
||||
this.konTransactionManager = konTransactionManager;
|
||||
@@ -118,12 +121,16 @@ public class ImportService {
|
||||
.filter(question -> "multi".equalsIgnoreCase(question.getQtype()))
|
||||
.map(question -> question.getQid().split("_", -1)[0])
|
||||
.collect(Collectors.toSet());
|
||||
var responses = responseImportService.prepare(table, multi);
|
||||
// Derived columns are not uploaded; derived values are regenerated from the new data.
|
||||
var responses =
|
||||
responseImportService.prepare(
|
||||
table, multi, derivedManagementService.derivedIds(project));
|
||||
int removed = importRepository.deleteUploaded(project);
|
||||
int archived = importRepository.archiveCompleted(project);
|
||||
for (var response : responses) {
|
||||
requireOne(importRepository.response(project, response));
|
||||
}
|
||||
derivedManagementService.recalculateAfterImport(project);
|
||||
return new ImportResultDto(responses.size(), responses.size(), removed, archived);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import com.kri.kart.server.dto.statistics.SurveyResponseDto;
|
||||
import com.kri.kart.server.service.statistics.LegacyFilterService;
|
||||
import com.kri.kart.server.service.statistics.StatisticsFilterService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -62,8 +63,32 @@ public class DerivedComputationService {
|
||||
return definition;
|
||||
}
|
||||
|
||||
/**
|
||||
* A response key belongs to a derived variable when it equals the variable ID or starts with
|
||||
* "{ID}_" (case-insensitive), matching the exported single/num/multi column names.
|
||||
*/
|
||||
public static boolean isDerivedKey(String key, Collection<String> derivedIds) {
|
||||
String upperKey = key.toUpperCase(Locale.ROOT);
|
||||
for (String derivedId : derivedIds) {
|
||||
String upperId = derivedId.toUpperCase(Locale.ROOT);
|
||||
if (upperKey.equals(upperId) || upperKey.startsWith(upperId + "_")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public List<ObjectNode> calculate(
|
||||
DerivedDefinitionDto definition, List<QuestionDto> questions, List<DerivedDataDto> data) {
|
||||
return calculate(definition, questions, data, List.of(definition.getQid()));
|
||||
}
|
||||
|
||||
/** Raw DATA keys of any derived variable are ignored so derived values take precedence. */
|
||||
public List<ObjectNode> calculate(
|
||||
DerivedDefinitionDto definition,
|
||||
List<QuestionDto> questions,
|
||||
List<DerivedDataDto> data,
|
||||
Collection<String> derivedIds) {
|
||||
ObjectNode info = object(definition.getDerivedJson());
|
||||
String type = object(definition.getQuestionJson()).path("qtype").asText();
|
||||
String qid = definition.getQid();
|
||||
@@ -84,6 +109,7 @@ public class DerivedComputationService {
|
||||
for (DerivedDataDto row : data) {
|
||||
Map<String, String> flat = new LinkedHashMap<>();
|
||||
flatten(object(row.getDatjson()), flat);
|
||||
flat.keySet().removeIf(key -> isDerivedKey(key, derivedIds));
|
||||
if (row.getDerivedJson() != null) {
|
||||
ObjectNode derived = object(row.getDerivedJson());
|
||||
derived.remove(qid);
|
||||
|
||||
@@ -6,10 +6,12 @@ import com.kri.common.feature.json.JsonUtil;
|
||||
import com.kri.common.feature.web.BusinessException;
|
||||
import com.kri.kart.server.dto.upload.ImportResponseDto;
|
||||
import com.kri.kart.server.dto.upload.UploadTableDto;
|
||||
import com.kri.kart.server.service.derived.DerivedComputationService;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
@@ -24,6 +26,14 @@ public class ResponseImportService {
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public List<ImportResponseDto> prepare(UploadTableDto table, Set<String> multiQuestions) {
|
||||
return prepare(table, multiQuestions, List.of());
|
||||
}
|
||||
|
||||
/**
|
||||
* Columns of stored derived variables are skipped; their values are regenerated after the upload.
|
||||
*/
|
||||
public List<ImportResponseDto> prepare(
|
||||
UploadTableDto table, Set<String> multiQuestions, Collection<String> derivedIds) {
|
||||
List<String> headers = table.headers();
|
||||
for (String required : List.of("IDKEY", "Status", "start_date", "end_date")) {
|
||||
if (headers.stream().noneMatch(header -> header.equalsIgnoreCase(required))) {
|
||||
@@ -74,6 +84,9 @@ public class ResponseImportService {
|
||||
|| key.startsWith("T" + root + "_"))) {
|
||||
continue;
|
||||
}
|
||||
if (DerivedComputationService.isDerivedKey(key, derivedIds)) {
|
||||
continue;
|
||||
}
|
||||
String cell = row.get(col).trim().replace(".00", "");
|
||||
if (cell.isEmpty()) {
|
||||
continue;
|
||||
|
||||
Reference in New Issue
Block a user