diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java index f66bbb1..5c77371 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java @@ -2,14 +2,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.CandidateReviewResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; import com.jobdri.jobdri_api.domain.analysis.type.QuestionAnalysisStatus; -import com.jobdri.jobdri_api.domain.analysis.service.core.AnalysisResultConstants; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateReviewDecision; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateReviewSnapshot; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationLlmSnapshot; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationQuestionAnalysis; +import com.jobdri.jobdri_api.domain.evaluation.analysis.policy.EvaluationAnalysisPolicyConstants; import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; import lombok.extern.slf4j.Slf4j; @@ -26,7 +27,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; @Service @@ -136,17 +136,17 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa evaluationCase.question(), evaluationCase.answer() )); - AnalysisLlmResponse llmResponse = generatedResult.response(); + EvaluationLlmSnapshot llmResponse = generatedResult.responseSnapshot(); int jobFit = validateScore("jobFit", llmResponse == null ? null : llmResponse.jobFit()); int impact = validateScore("impact", llmResponse == null ? null : llmResponse.impact()); int completeness = validateScore("completeness", llmResponse == null ? null : llmResponse.completeness()); - List missingKeywords = buildMissingKeywords(evaluationCase, llmResponse); + List missingKeywords = buildMissingKeywords(evaluationCase, llmResponse); List questionAnalyses = buildQuestionAnalyses(evaluationCase, llmResponse); log.debug( "Evaluation serialized missing keyword flow. caseId={}, candidateMissingKeywordCount={}, finalMissingKeywordCount={}, evaluationSerializedMissingKeywordCount={}", evaluationCase.caseId(), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().missingKeywordCandidates()), size(llmResponse == null ? null : llmResponse.missingKeywords()), missingKeywords.size() ); @@ -167,21 +167,21 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa normalizeFeedback(llmResponse.feedback()), writeJson(missingKeywords), writeJson(questionAnalyses), - writeJson(llmResponse), - writeJson(generatedResult.rawCandidateResponse()), - writeJson(generatedResult.sanitizedCandidateResponse()), - writeJson(generatedResult.candidateReviewResponse()), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), - acceptedDecisionCount(generatedResult.candidateReviewResponse()), - rejectedDecisionCount(generatedResult.candidateReviewResponse()), - rejectionCodeCounts(generatedResult.candidateReviewResponse()), + generatedResult.rawLlmResponseJson(), + generatedResult.rawCandidateResponseJson(), + generatedResult.sanitizedCandidateResponseJson(), + generatedResult.candidateReviewResponseJson(), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().analysisCandidates()), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().analysisCandidates()), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().strengthCandidates()), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().missingKeywordCandidates()), + acceptedDecisionCount(generatedResult.candidateReviewSnapshot()), + rejectedDecisionCount(generatedResult.candidateReviewSnapshot()), + rejectionCodeCounts(generatedResult.candidateReviewSnapshot()), questionAnalyses.size(), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()), - llmResponse.keyStrengths() == null ? 0 : llmResponse.keyStrengths().size(), - size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().strengthCandidates()), + llmResponse.keyStrengthQuotes() == null ? 0 : llmResponse.keyStrengthQuotes().size(), + size(generatedResult.sanitizedCandidateSnapshot() == null ? null : generatedResult.sanitizedCandidateSnapshot().missingKeywordCandidates()), missingKeywords.size(), generatedResult.candidateCallLatencyMs(), generatedResult.finalCallLatencyMs(), @@ -199,34 +199,30 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa ); } - private List buildMissingKeywords( + private List buildMissingKeywords( EvaluationAnalysisCase evaluationCase, - AnalysisLlmResponse llmResponse + EvaluationLlmSnapshot llmResponse ) { if (llmResponse == null || llmResponse.missingKeywords() == null) { return List.of(); } - List result = new ArrayList<>(); + List result = new ArrayList<>(); Set seenKeywords = new HashSet<>(); - for (AnalysisLlmResponse.MissingKeywordItem item : llmResponse.missingKeywords()) { + for (EvaluationMissingKeyword item : llmResponse.missingKeywords()) { if (item == null || !StringUtils.hasText(item.keyword())) { continue; } String keyword = item.keyword().trim(); - if (keyword.length() > AnalysisResultConstants.MAX_MISSING_KEYWORD_LENGTH) { + if (keyword.length() > EvaluationAnalysisPolicyConstants.MAX_MISSING_KEYWORD_LENGTH) { continue; } - Optional source = MissingKeywordSource.from(item.source()); - if (source.isEmpty()) { - continue; - } if (!evaluationSanitizationService.isValidMissingKeyword( keyword, - source.get(), + item.source(), evaluationCase.mainTasks(), evaluationCase.qualifications() )) { @@ -244,8 +240,8 @@ private List buildMissingKeywords( continue; } - result.add(new MissingKeywordResponse(keyword, source.get())); - if (result.size() >= AnalysisResultConstants.MAX_MISSING_KEYWORDS) { + result.add(new EvaluationMissingKeyword(keyword, item.source())); + if (result.size() >= EvaluationAnalysisPolicyConstants.MAX_MISSING_KEYWORDS) { break; } } @@ -254,7 +250,7 @@ private List buildMissingKeywords( private List buildQuestionAnalyses( EvaluationAnalysisCase evaluationCase, - AnalysisLlmResponse llmResponse + EvaluationLlmSnapshot llmResponse ) { if (llmResponse == null || llmResponse.questionAnalyses() == null) { return List.of(); @@ -272,7 +268,7 @@ private List buildQuestionAnalyses( Set fabricatedQuestionIds = new HashSet<>(); Set keyStrengthQuotes = normalizedKeyStrengthQuotes(llmResponse); - for (AnalysisLlmResponse.QuestionAnalysisItem item : llmResponse.questionAnalyses()) { + for (EvaluationQuestionAnalysis item : llmResponse.questionAnalyses()) { if (item == null || item.questionId() == null || !StringUtils.hasText(item.sentence())) { continue; } @@ -297,7 +293,7 @@ private List buildQuestionAnalyses( } int currentCount = analysisCountByQuestionId.getOrDefault(item.questionId(), 0); - if (currentCount >= AnalysisResultConstants.MAX_ANALYSES_PER_QUESTION) { + if (currentCount >= EvaluationAnalysisPolicyConstants.MAX_ANALYSES_PER_QUESTION) { continue; } @@ -339,20 +335,20 @@ private List buildQuestionAnalyses( return result; } - private Set normalizedKeyStrengthQuotes(AnalysisLlmResponse llmResponse) { - if (llmResponse == null || llmResponse.keyStrengths() == null) { + private Set normalizedKeyStrengthQuotes(EvaluationLlmSnapshot llmResponse) { + if (llmResponse == null || llmResponse.keyStrengthQuotes() == null) { return Set.of(); } - return llmResponse.keyStrengths().stream() - .filter(item -> item != null && StringUtils.hasText(item.quote())) - .map(item -> normalizeKeyword(item.quote())) + return llmResponse.keyStrengthQuotes().stream() + .filter(StringUtils::hasText) + .map(this::normalizeKeyword) .collect(java.util.stream.Collectors.toSet()); } private int validateScore(String fieldName, Integer score) { if (score == null - || score < AnalysisResultConstants.MIN_SCORE - || score > AnalysisResultConstants.MAX_SCORE) { + || score < EvaluationAnalysisPolicyConstants.MIN_SCORE + || score > EvaluationAnalysisPolicyConstants.MAX_SCORE) { throw new IllegalArgumentException("자소서 분석 AI 응답의 " + fieldName + " 점수 범위가 올바르지 않습니다."); } return score; @@ -360,9 +356,9 @@ private int validateScore(String fieldName, Integer score) { private int calculateScore(int jobFit, int impact, int completeness) { return (int) Math.round( - jobFit * AnalysisResultConstants.JOB_FIT_WEIGHT - + impact * AnalysisResultConstants.IMPACT_WEIGHT - + completeness * AnalysisResultConstants.COMPLETENESS_WEIGHT + jobFit * EvaluationAnalysisPolicyConstants.JOB_FIT_WEIGHT + + impact * EvaluationAnalysisPolicyConstants.IMPACT_WEIGHT + + completeness * EvaluationAnalysisPolicyConstants.COMPLETENESS_WEIGHT ); } @@ -399,7 +395,7 @@ private Integer size(List values) { return values == null ? null : values.size(); } - private Integer acceptedDecisionCount(CandidateReviewResponse reviewResponse) { + private Integer acceptedDecisionCount(EvaluationCandidateReviewSnapshot reviewResponse) { if (reviewResponse == null || reviewResponse.decisions() == null) { return null; } @@ -408,7 +404,7 @@ private Integer acceptedDecisionCount(CandidateReviewResponse reviewResponse) { .count(); } - private Integer rejectedDecisionCount(CandidateReviewResponse reviewResponse) { + private Integer rejectedDecisionCount(EvaluationCandidateReviewSnapshot reviewResponse) { if (reviewResponse == null || reviewResponse.decisions() == null) { return null; } @@ -417,16 +413,16 @@ private Integer rejectedDecisionCount(CandidateReviewResponse reviewResponse) { .count(); } - private String rejectionCodeCounts(CandidateReviewResponse reviewResponse) { + private String rejectionCodeCounts(EvaluationCandidateReviewSnapshot reviewResponse) { if (reviewResponse == null || reviewResponse.decisions() == null) { return ""; } Map counts = reviewResponse.decisions().stream() .filter(decision -> decision != null - && decision.rejectionCode() != null - && decision.rejectionCode() != CandidateReviewResponse.RejectionCode.NONE) + && StringUtils.hasText(decision.rejectionCode()) + && !"NONE".equals(decision.rejectionCode())) .collect(java.util.stream.Collectors.groupingBy( - decision -> decision.rejectionCode().name(), + EvaluationCandidateReviewDecision::rejectionCode, java.util.stream.Collectors.counting() )); return writeJson(counts); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java index 06d89c8..a2e9a91 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java @@ -1,10 +1,14 @@ package com.jobdri.jobdri_api.domain.evaluation.analysis; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationDecision; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationCandidateSnapshotParser; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateSnapshot; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordCandidate; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSanitizationDecision; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSanitizationResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -77,6 +81,7 @@ class MissingKeywordSanitizerReplayService { private final ObjectMapper objectMapper; private final EvaluationSanitizationService evaluationSanitizationService; + private final EvaluationCandidateSnapshotParser candidateSnapshotParser; @Autowired MissingKeywordSanitizerReplayService( @@ -85,6 +90,7 @@ class MissingKeywordSanitizerReplayService { ) { this.objectMapper = objectMapper; this.evaluationSanitizationService = evaluationSanitizationService; + this.candidateSnapshotParser = new EvaluationCandidateSnapshotParser(objectMapper); } MissingKeywordSanitizerReplayService(ObjectMapper objectMapper) { @@ -114,19 +120,18 @@ ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOExcept if (!caseIds.add(caseId)) { throw new IllegalArgumentException("Missing keyword replay input CSV has duplicate caseId: " + caseId); } - AnalysisCandidateResponse rawResponse = readCandidateResponse( + EvaluationCandidateSnapshot rawResponse = readCandidateResponse( value(row, "rawCandidateResponseJson"), "rawCandidateResponseJson", caseId ); - AnalysisCandidateResponse existingSanitized = readCandidateResponse( + EvaluationCandidateSnapshot existingSanitized = readCandidateResponse( value(row, "sanitizedCandidateResponseJson"), "sanitizedCandidateResponseJson", caseId ); - List rawCandidates = - safeMissingKeywordCandidates(rawResponse); - MissingKeywordSanitizationResult replayResult = evaluationSanitizationService.sanitizeMissingKeywordCandidates( + List rawCandidates = safeMissingKeywordCandidates(rawResponse); + EvaluationMissingKeywordSanitizationResult replayResult = evaluationSanitizationService.sanitizeMissingKeywordCandidates( value(row, "mainTasks"), value(row, "qualifications"), value(row, "answer"), @@ -144,7 +149,7 @@ ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOExcept acceptedTotal += acceptedCount; rejectedTotal += rejectedCount; - for (MissingKeywordSanitizationDecision decision : replayResult.decisions()) { + for (EvaluationMissingKeywordSanitizationDecision decision : replayResult.decisions()) { reasonCounts.merge(decision.rejectionReason().name(), 1L, Long::sum); replayRows.add(toReplayRow(caseId, rawCandidateCount, acceptedCount, rejectedCount, decision)); if (rawCandidateCount > 0) { @@ -192,23 +197,14 @@ private void validateHeaders(List headers) { } } - private AnalysisCandidateResponse readCandidateResponse(String json, String fieldName, String caseId) { - if (!StringUtils.hasText(json)) { - return new AnalysisCandidateResponse(List.of(), List.of(), List.of()); - } - try { - AnalysisCandidateResponse response = objectMapper.readValue(json, AnalysisCandidateResponse.class); - if (response == null) { - return new AnalysisCandidateResponse(List.of(), List.of(), List.of()); - } - return response; - } catch (JsonProcessingException e) { - throw new IllegalArgumentException(fieldName + " is not valid candidate JSON. caseId=" + caseId, e); - } + private EvaluationCandidateSnapshot readCandidateResponse(String json, String fieldName, String caseId) { + EvaluationCandidateSnapshot snapshot = candidateSnapshotParser.parse(json, fieldName, caseId); + validateMissingKeywordCandidates(json, fieldName, caseId); + return snapshot; } - private List safeMissingKeywordCandidates( - AnalysisCandidateResponse response + private List safeMissingKeywordCandidates( + EvaluationCandidateSnapshot response ) { if (response == null || response.missingKeywordCandidates() == null) { return List.of(); @@ -218,24 +214,91 @@ private List safeMissingKeywo private void validateExistingSanitized( String caseId, - List replayAccepted, - List existingAccepted + List replayAccepted, + List existingAccepted ) { - if (!Objects.equals(replayAccepted, existingAccepted)) { + if (!acceptedCandidatesMatch(replayAccepted, existingAccepted)) { throw new IllegalStateException( "Missing keyword replay accepted candidates mismatch. caseId=" + caseId ); } } + private void validateMissingKeywordCandidates(String json, String fieldName, String caseId) { + if (!StringUtils.hasText(json)) { + return; + } + try { + JsonNode missingKeywordCandidates = objectMapper.readTree(json).path("missingKeywordCandidates"); + if (!missingKeywordCandidates.isArray()) { + return; + } + for (int i = 0; i < missingKeywordCandidates.size(); i++) { + JsonNode item = missingKeywordCandidates.get(i); + String keyword = item.path("keyword").asText(null); + String source = item.path("source").asText(null); + if (!StringUtils.hasText(keyword)) { + throw new IllegalArgumentException( + fieldName + " has blank missingKeywordCandidates[" + i + "].keyword. caseId=" + caseId + ); + } + if (EvaluationMissingKeywordSource.from(source).isEmpty()) { + throw new IllegalArgumentException( + fieldName + " has invalid missingKeywordCandidates[" + i + "].source. caseId=" + caseId + ); + } + } + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(fieldName + " is not valid candidate JSON. caseId=" + caseId, e); + } + } + + private boolean acceptedCandidatesMatch( + List replayAccepted, + List existingAccepted + ) { + if (replayAccepted == null || existingAccepted == null) { + return Objects.equals(replayAccepted, existingAccepted); + } + if (replayAccepted.size() != existingAccepted.size()) { + return false; + } + for (int i = 0; i < replayAccepted.size(); i++) { + if (!acceptedCandidateMatches(replayAccepted.get(i), existingAccepted.get(i))) { + return false; + } + } + return true; + } + + private boolean acceptedCandidateMatches( + EvaluationMissingKeywordCandidate replayCandidate, + EvaluationMissingKeywordCandidate existingCandidate + ) { + if (replayCandidate == null || existingCandidate == null) { + return Objects.equals(replayCandidate, existingCandidate); + } + return Objects.equals(replayCandidate.keyword(), existingCandidate.keyword()) + && replayCandidate.source() == existingCandidate.source() + && relatedRequirementMatches(replayCandidate.relatedRequirement(), existingCandidate.relatedRequirement()); + } + + private boolean relatedRequirementMatches(String replayRelatedRequirement, String existingRelatedRequirement) { + String normalizedReplay = value(replayRelatedRequirement); + String normalizedExisting = value(existingRelatedRequirement); + return Objects.equals(normalizedReplay, normalizedExisting) + || !StringUtils.hasText(normalizedReplay) + || !StringUtils.hasText(normalizedExisting); + } + private Map toReplayRow( String caseId, int rawCandidateCount, int acceptedCandidateCount, int rejectedCandidateCount, - MissingKeywordSanitizationDecision decision + EvaluationMissingKeywordSanitizationDecision decision ) { - AnalysisCandidateResponse.MissingKeywordCandidate candidate = decision.originalCandidate(); + EvaluationMissingKeywordCandidate candidate = decision.originalCandidate(); Map row = new LinkedHashMap<>(); row.put("caseId", caseId); row.put("candidateIndex", String.valueOf(decision.candidateIndex())); @@ -259,9 +322,9 @@ private Map toReplayRow( private Map toReviewRow( String caseId, Map inputRow, - MissingKeywordSanitizationDecision decision + EvaluationMissingKeywordSanitizationDecision decision ) { - AnalysisCandidateResponse.MissingKeywordCandidate candidate = decision.originalCandidate(); + EvaluationMissingKeywordCandidate candidate = decision.originalCandidate(); Map row = new LinkedHashMap<>(); row.put("caseId", caseId); row.put("candidateIndex", String.valueOf(decision.candidateIndex())); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java index d286398..6020530 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationLlmSnapshotParser; -import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationMissingKeywordSourceMapper; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; @@ -465,7 +464,7 @@ private Optional missingKeywordMissInvalidReaso } if (!evaluationSanitizationService.isValidMissingKeyword( evaluation.keyword(), - EvaluationMissingKeywordSourceMapper.toAnalysisSource(source.get()), + source.get(), input.mainTasks(), input.qualifications() )) { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParser.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParser.java new file mode 100644 index 0000000..ba93771 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParser.java @@ -0,0 +1,49 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateReviewDecision; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateReviewSnapshot; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public class EvaluationCandidateReviewSnapshotParser { + private final ObjectMapper objectMapper; + + public EvaluationCandidateReviewSnapshotParser(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public EvaluationCandidateReviewSnapshot parse(String json) { + if (!StringUtils.hasText(json)) { + return emptySnapshot(); + } + try { + JsonNode root = objectMapper.readTree(json); + return new EvaluationCandidateReviewSnapshot(readDecisions(root.path("decisions"))); + } catch (JsonProcessingException e) { + return emptySnapshot(); + } + } + + private EvaluationCandidateReviewSnapshot emptySnapshot() { + return new EvaluationCandidateReviewSnapshot(List.of()); + } + + private List readDecisions(JsonNode decisionsNode) { + if (!decisionsNode.isArray()) { + return List.of(); + } + List decisions = new ArrayList<>(); + for (JsonNode item : decisionsNode) { + decisions.add(new EvaluationCandidateReviewDecision( + item.path("accepted").isBoolean() ? item.path("accepted").booleanValue() : null, + item.path("rejectionCode").asText(null) + )); + } + return decisions; + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapper.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapper.java deleted file mode 100644 index d8b7d0d..0000000 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapper.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateSnapshot; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordCandidate; - -import java.util.List; - -public final class EvaluationCandidateSnapshotMapper { - private EvaluationCandidateSnapshotMapper() { - } - - public static EvaluationCandidateSnapshot from(AnalysisCandidateResponse response) { - if (response == null || response.missingKeywordCandidates() == null) { - return new EvaluationCandidateSnapshot(List.of()); - } - List missingKeywordCandidates = response.missingKeywordCandidates().stream() - .map(candidate -> EvaluationMissingKeywordSourceMapper.fromAnalysisSource(candidate.source()) - .map(source -> new EvaluationMissingKeywordCandidate( - candidate.keyword(), - source, - candidate.relatedRequirement() - )) - .orElse(null)) - .filter(candidate -> candidate != null) - .toList(); - return new EvaluationCandidateSnapshot(missingKeywordCandidates); - } -} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParser.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParser.java new file mode 100644 index 0000000..5d084d6 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParser.java @@ -0,0 +1,71 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateSnapshot; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordCandidate; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public class EvaluationCandidateSnapshotParser { + private final ObjectMapper objectMapper; + + public EvaluationCandidateSnapshotParser(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public EvaluationCandidateSnapshot parse(String json, String fieldName, String caseId) { + if (!StringUtils.hasText(json)) { + return emptySnapshot(); + } + try { + JsonNode root = objectMapper.readTree(json); + return new EvaluationCandidateSnapshot( + readOpaqueItems(root.path("strengthCandidates")), + readOpaqueItems(root.path("analysisCandidates")), + readMissingKeywordCandidates(root.path("missingKeywordCandidates")) + ); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException(fieldName + " is not valid candidate JSON. caseId=" + caseId, e); + } + } + + private EvaluationCandidateSnapshot emptySnapshot() { + return new EvaluationCandidateSnapshot(List.of(), List.of(), List.of()); + } + + private List readOpaqueItems(JsonNode node) { + if (!node.isArray()) { + return List.of(); + } + List items = new ArrayList<>(); + node.forEach(item -> items.add(item)); + return items; + } + + private List readMissingKeywordCandidates(JsonNode node) { + if (!node.isArray()) { + return List.of(); + } + List candidates = new ArrayList<>(); + for (JsonNode item : node) { + String keyword = item.path("keyword").asText(null); + String source = item.path("source").asText(null); + String relatedRequirement = item.path("relatedRequirement").asText(null); + if (!StringUtils.hasText(keyword)) { + continue; + } + EvaluationMissingKeywordSource.from(source) + .ifPresent(value -> candidates.add(new EvaluationMissingKeywordCandidate( + keyword, + value, + relatedRequirement + ))); + } + return candidates; + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapper.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapper.java deleted file mode 100644 index 4c11126..0000000 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationLlmSnapshot; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationQuestionAnalysis; - -import java.util.List; -import java.util.Optional; - -public final class EvaluationLlmSnapshotMapper { - private EvaluationLlmSnapshotMapper() { - } - - public static EvaluationLlmSnapshot from(AnalysisLlmResponse response) { - if (response == null) { - return new EvaluationLlmSnapshot(List.of(), List.of(), List.of()); - } - List keyStrengthQuotes = response.keyStrengths() == null - ? List.of() - : response.keyStrengths().stream() - .map(AnalysisLlmResponse.HighlightItem::quote) - .toList(); - List missingKeywords = response.missingKeywords() == null - ? List.of() - : response.missingKeywords().stream() - .map(EvaluationMissingKeywordMapper::from) - .flatMap(Optional::stream) - .toList(); - List questionAnalyses = response.questionAnalyses() == null - ? List.of() - : response.questionAnalyses().stream() - .map(EvaluationQuestionAnalysisMapper::from) - .filter(questionAnalysis -> questionAnalysis != null) - .toList(); - return new EvaluationLlmSnapshot(keyStrengthQuotes, missingKeywords, questionAnalyses); - } -} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParser.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParser.java index 3850d80..76694c3 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParser.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParser.java @@ -26,6 +26,10 @@ public EvaluationLlmSnapshot parseRawLlmResponse(String rawLlmResponseJson) { try { JsonNode root = objectMapper.readTree(rawLlmResponseJson); return new EvaluationLlmSnapshot( + root.path("jobFit").isNumber() ? root.path("jobFit").intValue() : null, + root.path("impact").isNumber() ? root.path("impact").intValue() : null, + root.path("completeness").isNumber() ? root.path("completeness").intValue() : null, + root.path("feedback").asText(""), readKeyStrengthQuotes(root.path("keyStrengths")), readMissingKeywords(root.path("missingKeywords")), readQuestionAnalyses(root.path("questionAnalyses")) @@ -51,7 +55,7 @@ public List parseMissingKeywords(String json, String c } private EvaluationLlmSnapshot emptySnapshot() { - return new EvaluationLlmSnapshot(List.of(), List.of(), List.of()); + return new EvaluationLlmSnapshot(null, null, null, "", List.of(), List.of(), List.of()); } private List readKeyStrengthQuotes(JsonNode keyStrengthsNode) { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordMapper.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordMapper.java deleted file mode 100644 index 2ce2534..0000000 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordMapper.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword; - -import java.util.Optional; - -public final class EvaluationMissingKeywordMapper { - private EvaluationMissingKeywordMapper() { - } - - public static Optional from(AnalysisLlmResponse.MissingKeywordItem item) { - if (item == null) { - return Optional.empty(); - } - return EvaluationMissingKeywordSourceMapper.fromAnalysisSource(item.source()) - .map(source -> new EvaluationMissingKeyword(item.keyword(), source)); - } -} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordSourceMapper.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordSourceMapper.java deleted file mode 100644 index 383c5b9..0000000 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationMissingKeywordSourceMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; - -import java.util.Optional; - -public final class EvaluationMissingKeywordSourceMapper { - private EvaluationMissingKeywordSourceMapper() { - } - - public static Optional fromAnalysisSource(String value) { - return EvaluationMissingKeywordSource.from(value); - } - - public static Optional fromAnalysisSource(MissingKeywordSource source) { - if (source == null) { - return Optional.empty(); - } - return switch (source) { - case MAIN_TASK -> Optional.of(EvaluationMissingKeywordSource.MAIN_TASK); - case QUALIFICATION -> Optional.of(EvaluationMissingKeywordSource.QUALIFICATION); - case PREFERENCE -> Optional.of(EvaluationMissingKeywordSource.PREFERENCE); - }; - } - - public static MissingKeywordSource toAnalysisSource(EvaluationMissingKeywordSource source) { - if (source == null) { - return null; - } - return switch (source) { - case MAIN_TASK -> MissingKeywordSource.MAIN_TASK; - case QUALIFICATION -> MissingKeywordSource.QUALIFICATION; - case PREFERENCE -> MissingKeywordSource.PREFERENCE; - }; - } -} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationQuestionAnalysisMapper.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationQuestionAnalysisMapper.java deleted file mode 100644 index f1d3777..0000000 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationQuestionAnalysisMapper.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationQuestionAnalysis; - -public final class EvaluationQuestionAnalysisMapper { - private EvaluationQuestionAnalysisMapper() { - } - - public static EvaluationQuestionAnalysis from(AnalysisLlmResponse.QuestionAnalysisItem item) { - if (item == null) { - return null; - } - return new EvaluationQuestionAnalysis( - item.questionId(), - item.sentence(), - item.status(), - item.reason(), - item.improvement() - ); - } -} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewDecision.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewDecision.java new file mode 100644 index 0000000..ba6b657 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewDecision.java @@ -0,0 +1,7 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +public record EvaluationCandidateReviewDecision( + Boolean accepted, + String rejectionCode +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewSnapshot.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewSnapshot.java new file mode 100644 index 0000000..359e885 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateReviewSnapshot.java @@ -0,0 +1,8 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +import java.util.List; + +public record EvaluationCandidateReviewSnapshot( + List decisions +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateSnapshot.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateSnapshot.java index cd03d0a..24c140c 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateSnapshot.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationCandidateSnapshot.java @@ -1,8 +1,12 @@ package com.jobdri.jobdri_api.domain.evaluation.analysis.model; +import com.fasterxml.jackson.databind.JsonNode; + import java.util.List; public record EvaluationCandidateSnapshot( + List strengthCandidates, + List analysisCandidates, List missingKeywordCandidates ) { } diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java index 314183f..4e76113 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java @@ -1,14 +1,13 @@ package com.jobdri.jobdri_api.domain.evaluation.analysis.model; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.CandidateReviewResponse; - public record EvaluationGeneratedResult( - AnalysisLlmResponse response, - AnalysisCandidateResponse rawCandidateResponse, - AnalysisCandidateResponse sanitizedCandidateResponse, - CandidateReviewResponse candidateReviewResponse, + EvaluationLlmSnapshot responseSnapshot, + String rawLlmResponseJson, + String rawCandidateResponseJson, + String sanitizedCandidateResponseJson, + EvaluationCandidateSnapshot sanitizedCandidateSnapshot, + String candidateReviewResponseJson, + EvaluationCandidateReviewSnapshot candidateReviewSnapshot, long candidateCallLatencyMs, long finalCallLatencyMs ) { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationLlmSnapshot.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationLlmSnapshot.java index 4e01a38..e082b2a 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationLlmSnapshot.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationLlmSnapshot.java @@ -3,6 +3,10 @@ import java.util.List; public record EvaluationLlmSnapshot( + Integer jobFit, + Integer impact, + Integer completeness, + String feedback, List keyStrengthQuotes, List missingKeywords, List questionAnalyses diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordRejectionReason.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordRejectionReason.java new file mode 100644 index 0000000..e60c485 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordRejectionReason.java @@ -0,0 +1,15 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +public enum EvaluationMissingKeywordRejectionReason { + ACCEPTED, + NULL_CANDIDATE, + BLANK_KEYWORD, + MAX_ACCEPTED_LIMIT, + INVALID_FORMAT, + UNSUPPORTED_KEYWORD, + CERTIFICATE_OR_QUANTITATIVE_NOISE, + TOO_GENERIC, + NOT_RELATED_TO_JD, + DUPLICATE_KEYWORD, + NORMALIZATION_COLLISION +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationDecision.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationDecision.java new file mode 100644 index 0000000..1034d6a --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationDecision.java @@ -0,0 +1,14 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +public record EvaluationMissingKeywordSanitizationDecision( + int candidateIndex, + EvaluationMissingKeywordCandidate originalCandidate, + String normalizedKeyword, + boolean accepted, + EvaluationMissingKeywordRejectionReason rejectionReason, + boolean answerExactMatch, + boolean answerNormalizedMatch, + boolean jdRequirementMatched, + Integer duplicateOfCandidateIndex +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationResult.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationResult.java new file mode 100644 index 0000000..9a659aa --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationMissingKeywordSanitizationResult.java @@ -0,0 +1,9 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +import java.util.List; + +public record EvaluationMissingKeywordSanitizationResult( + List acceptedCandidates, + List decisions +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/policy/EvaluationAnalysisPolicyConstants.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/policy/EvaluationAnalysisPolicyConstants.java new file mode 100644 index 0000000..b61041d --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/policy/EvaluationAnalysisPolicyConstants.java @@ -0,0 +1,16 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.policy; + +public final class EvaluationAnalysisPolicyConstants { + + public static final int MIN_SCORE = 0; + public static final int MAX_SCORE = 100; + public static final int MAX_ANALYSES_PER_QUESTION = 3; + public static final int MAX_MISSING_KEYWORDS = 3; + public static final int MAX_MISSING_KEYWORD_LENGTH = 60; + public static final double JOB_FIT_WEIGHT = 0.50; + public static final double IMPACT_WEIGHT = 0.30; + public static final double COMPLETENESS_WEIGHT = 0.20; + + private EvaluationAnalysisPolicyConstants() { + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java index 19b940f..23078d5 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java @@ -1,18 +1,91 @@ package com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization; -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; -import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationResult; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizer; -import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationMissingKeywordSourceMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordCandidate; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordRejectionReason; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSanitizationDecision; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSanitizationResult; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @Service public class EvaluationSanitizationService { + private static final int MAX_ACCEPTED_COUNT = 3; + private static final double MIN_KEYWORD_TOKEN_MATCH_RATIO = 0.5; + private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); + private static final Pattern TOKEN_PATTERN = Pattern.compile("[a-zA-Z0-9+#.]+|[가-힣]+"); + private static final Pattern KOREAN_ONLY_PATTERN = Pattern.compile("[가-힣]+"); + private static final Pattern CAREER_YEAR_PATTERN = Pattern.compile("경력\\s*\\d+\\s*년|\\d+\\s*년\\s*(이상|이하|미만|초과)"); + private static final Set STRUCTURED_QUALIFICATION_TERMS = normalizedSet( + "자격증", "면허", "면허증", "공인성적", "어학성적", "토익", "toeic", "토플", "toefl", + "opic", "ielts", "학위", "전공", "졸업", "학력", "신입", "근무 가능", "국적", "나이", "연령", + "사회복지사", "청소년지도사", "청소년상담사", "직업상담사", "임상심리사", + "대졸", "초대졸", "전문대졸", "고졸" + ); + private static final Set META_IMPROVEMENT_TERMS = normalizedSet( + "구체적으로 작성했습니다", "명확히 설명했습니다", "성과를 강조했습니다", "수치로 명시했습니다", + "더 설득력 있게 작성했습니다", "경험을 구체적으로 작성했습니다", "구체적으로 설명했습니다", + "구체적으로 서술했습니다", "명확히 서술했습니다", "구체적으로 설명합니다", + "구체적으로 서술합니다", "명확히 설명합니다", "명확히 서술합니다", + "추가해 보", "추가하면 좋", "설명해 보", "구체적으로 작성", "구체적으로 설명", + "강조하겠", "추가하겠", "명확히 작성", "작성할 수 있", "설명할 수 있", + "보완하겠", "드러내겠", "제시하겠", "추가할 수 있", "수정할 수 있", + "수정하는 방향", "강조하는 방향" + ); + private static final String[] CONTRADICTORY_PROVEN_REASON_TERMS = { + "근거가 부족", "성과가 부족", "수치가 부족", "구체성이 부족", "보완이 필요", + "드러나지 않음", "확인하기 어려움" + }; + private static final Set FABRICATED_DIRECT_CONFLICT_TERMS = normalizedSet( + "직접 충돌", "명시적 사실과 충돌", "사실과 충돌", "조건과 충돌", "요건과 충돌", + "서로 충돌", "상충", "하지 않았다고", "수행하지 않았다고", "경험이 없다고", + "없다고 밝혔", "실제로 하지 않았", "모순", "불일치", "일치하지 않", "앞뒤가 맞지", + "다르게 서술" + ); + private static final String TEAM_PROJECT_TERM = normalizeStatic("팀 프로젝트"); + private static final String TEAM_PROGRESS_TERM = normalizeStatic("팀으로 진행"); + private static final String INDIVIDUAL_PROJECT_TERM = normalizeStatic("개인 프로젝트"); + private static final String SOLO_EXECUTION_TERM = normalizeStatic("혼자 수행"); + private static final String SOLO_PROGRESS_TERM = normalizeStatic("혼자 진행"); + private static final Set STOP_WORDS = Set.of( + "경험", "역량", "업무", "관련", "가능", "보유", "필수", "우대", "자격", "요건", + "사항", "직무", "수행", "활용", "사용", "기반", "중심", "대한", "통한", "등", + "및", "또는", "위한", "있는", "없는" + ); + private static final List BANNED_IMPROVEMENT_PHRASES = compactList( + "추가하세요", + "보완하세요", + "수정해주세요", + "수정하세요", + "작성해주세요", + "작성하세요", + "필요합니다", + "해야 합니다", + "해주세요", + "명확히 해야", + "명확히 하세요" + ); + private static final List IMPERATIVE_ENDINGS = compactList( + "하세요", + "하십시오", + "해주십시오", + "해 주십시오" + ); + private static final List KOREAN_SUFFIXES = List.of( + "했습니다", "았습니다", "었습니다", "으로", "에서", "하며", "하고", "하는", "까지", "부터", "에게", "보다", + "은", "는", "이", "가", "을", "를", "와", "과", "의", "에", "로", "한" + ); public boolean isValidMissingKeyword( String keyword, @@ -20,59 +93,415 @@ public boolean isValidMissingKeyword( String mainTasks, String qualifications ) { - MissingKeywordSource analysisSource = EvaluationMissingKeywordSourceMapper.toAnalysisSource(source); - return analysisSource != null && AnalysisSanitizationRules.isValidMissingKeyword( - keyword, - analysisSource, - mainTasks, - qualifications - ); - } - - public boolean isValidMissingKeyword( - String keyword, - MissingKeywordSource source, - String mainTasks, - String qualifications - ) { - return source != null && AnalysisSanitizationRules.isValidMissingKeyword( - keyword, - source, - mainTasks, - qualifications - ); + if (!StringUtils.hasText(keyword) || source == null) { + return false; + } + if (source == EvaluationMissingKeywordSource.PREFERENCE) { + return false; + } + if (isStructuredQualificationKeyword(keyword)) { + return false; + } + if (source == EvaluationMissingKeywordSource.MAIN_TASK) { + return isGroundedInSource(keyword, mainTasks); + } + if (source == EvaluationMissingKeywordSource.QUALIFICATION) { + return isGroundedInSource(keyword, qualifications); + } + return false; } public boolean isMissingKeywordMentionedInAnswers(String keyword, String answer) { - return AnalysisSanitizationRules.isMissingKeywordMentionedInAnswers(keyword, answer); + if (!StringUtils.hasText(keyword) || !StringUtils.hasText(answer)) { + return false; + } + if (containsNormalized(answer, keyword)) { + return true; + } + + Set keywordTokens = coreTokens(keyword); + Set answerTokens = coreTokens(answer); + if (keywordTokens.isEmpty() || answerTokens.isEmpty()) { + return false; + } + + long matchCount = keywordTokens.stream() + .filter(answerTokens::contains) + .count(); + if (keywordTokens.size() == 1) { + return matchCount == 1; + } + return (double) matchCount / keywordTokens.size() >= MIN_KEYWORD_TOKEN_MATCH_RATIO; } public boolean hasValidProvenReason(String reason) { - return AnalysisSanitizationRules.hasValidProvenReason(reason); + if (!StringUtils.hasText(reason)) { + return false; + } + String normalized = normalize(reason); + for (String term : CONTRADICTORY_PROVEN_REASON_TERMS) { + if (normalized.contains(normalize(term))) { + return false; + } + } + return true; } public boolean hasFabricatedDirectConflictEvidence(String sentence, String reason) { - return AnalysisSanitizationRules.hasFabricatedDirectConflictEvidence(sentence, reason); + if (!StringUtils.hasText(reason)) { + return false; + } + if (hasFabricatedDirectConflictReason(reason)) { + return true; + } + return StringUtils.hasText(sentence) + && hasFabricatedDirectConflictReason(sentence + " " + reason); } public String normalizeImprovement(String sentence, String answer, String improvement, boolean proven) { - return AnalysisSanitizationRules.normalizeImprovement(sentence, answer, improvement, proven); + if (proven || !StringUtils.hasText(improvement) || isNullLikeImprovement(improvement)) { + return ""; + } + + String normalized = improvement.trim(); + if (isInstructionLike(normalized) + || equalsNormalized(sentence, normalized) + || isCopiedFromAnotherAnswerSentence(sentence, answer, normalized) + || isMetaImprovement(normalized) + || changesSentenceTense(sentence, normalized)) { + return ""; + } + return normalized; } public boolean isStructuredQualificationKeyword(String value) { - return AnalysisSanitizationRules.isStructuredQualificationKeyword(value); + if (value == null) { + return false; + } + String normalized = normalize(value); + if (CAREER_YEAR_PATTERN.matcher(value).find()) { + return true; + } + for (String term : STRUCTURED_QUALIFICATION_TERMS) { + if (normalized.contains(term)) { + return true; + } + } + return false; } public String normalizeText(String value) { - return AnalysisSanitizationRules.normalizeText(value); + return normalize(value); + } + + public EvaluationMissingKeywordSanitizationResult sanitizeMissingKeywordCandidates( + String mainTasks, + String qualifications, + String answer, + List candidates + ) { + if (candidates == null) { + return new EvaluationMissingKeywordSanitizationResult(List.of(), List.of()); + } + + List acceptedCandidates = new ArrayList<>(); + List decisions = new ArrayList<>(); + Map seen = new HashMap<>(); + + for (int i = 0; i < candidates.size(); i++) { + EvaluationMissingKeywordCandidate candidate = candidates.get(i); + EvaluationMissingKeywordSanitizationDecision decision = decide( + i, + candidate, + mainTasks, + qualifications, + answer, + seen, + acceptedCandidates.size() + ); + decisions.add(decision); + if (decision.accepted()) { + acceptedCandidates.add(candidate); + seen.put(decision.normalizedKeyword(), new IndexedKeyword(i, candidate.keyword())); + } + } + + return new EvaluationMissingKeywordSanitizationResult(List.copyOf(acceptedCandidates), List.copyOf(decisions)); } - public MissingKeywordSanitizationResult sanitizeMissingKeywordCandidates( + private EvaluationMissingKeywordSanitizationDecision decide( + int index, + EvaluationMissingKeywordCandidate candidate, String mainTasks, String qualifications, String answer, - List candidates + Map seen, + int acceptedCount + ) { + if (candidate == null) { + return rejected(index, null, "", answer, false, null, EvaluationMissingKeywordRejectionReason.NULL_CANDIDATE); + } + String keyword = candidate.keyword(); + String normalizedKeyword = normalize(keyword); + if (!StringUtils.hasText(keyword)) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.BLANK_KEYWORD); + } + if (acceptedCount >= MAX_ACCEPTED_COUNT) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.MAX_ACCEPTED_LIMIT); + } + if (candidate.source() == null) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.INVALID_FORMAT); + } + if (candidate.source() == EvaluationMissingKeywordSource.PREFERENCE) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.UNSUPPORTED_KEYWORD); + } + if (isStructuredQualificationKeyword(keyword)) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.CERTIFICATE_OR_QUANTITATIVE_NOISE); + } + if (coreTokens(keyword).isEmpty()) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.TOO_GENERIC); + } + + boolean grounded = isValidMissingKeyword(keyword, candidate.source(), mainTasks, qualifications); + if (!grounded) { + return rejected(index, candidate, normalizedKeyword, answer, false, null, EvaluationMissingKeywordRejectionReason.NOT_RELATED_TO_JD); + } + + IndexedKeyword duplicate = seen.get(normalizedKeyword); + if (duplicate != null) { + EvaluationMissingKeywordRejectionReason reason = duplicate.keyword().trim().equals(keyword.trim()) + ? EvaluationMissingKeywordRejectionReason.DUPLICATE_KEYWORD + : EvaluationMissingKeywordRejectionReason.NORMALIZATION_COLLISION; + return rejected(index, candidate, normalizedKeyword, answer, grounded, duplicate.index(), reason); + } + + return new EvaluationMissingKeywordSanitizationDecision( + index, + candidate, + normalizedKeyword, + true, + EvaluationMissingKeywordRejectionReason.ACCEPTED, + containsExact(answer, keyword), + containsNormalized(answer, keyword), + true, + null + ); + } + + private EvaluationMissingKeywordSanitizationDecision rejected( + int index, + EvaluationMissingKeywordCandidate candidate, + String normalizedKeyword, + String answer, + boolean jdRequirementMatched, + Integer duplicateOfCandidateIndex, + EvaluationMissingKeywordRejectionReason reason + ) { + String keyword = candidate == null ? "" : candidate.keyword(); + return new EvaluationMissingKeywordSanitizationDecision( + index, + candidate, + normalizedKeyword, + false, + reason, + containsExact(answer, keyword), + containsNormalized(answer, keyword), + jdRequirementMatched, + duplicateOfCandidateIndex + ); + } + + private boolean hasFabricatedDirectConflictReason(String reason) { + if (!StringUtils.hasText(reason)) { + return false; + } + String normalized = normalize(reason); + for (String term : FABRICATED_DIRECT_CONFLICT_TERMS) { + if (normalized.contains(term)) { + return true; + } + } + boolean teamClaim = normalized.contains(TEAM_PROJECT_TERM) + || normalized.contains(TEAM_PROGRESS_TERM); + boolean individualClaim = normalized.contains(INDIVIDUAL_PROJECT_TERM) + || normalized.contains(SOLO_EXECUTION_TERM) + || normalized.contains(SOLO_PROGRESS_TERM); + return teamClaim && individualClaim; + } + + private boolean isInstructionLike(String improvement) { + if (improvement == null) { + return false; + } + String compact = compactWhitespace(improvement); + return BANNED_IMPROVEMENT_PHRASES.stream().anyMatch(compact::contains) + || IMPERATIVE_ENDINGS.stream() + .anyMatch(ending -> compact.endsWith(ending) || compact.endsWith(ending + ".")); + } + + private boolean isGroundedInSource(String keyword, String sourceText) { + Set keywordTokens = coreTokens(keyword); + if (keywordTokens.isEmpty() || !StringUtils.hasText(sourceText)) { + return false; + } + + Set sourceTokens = coreTokens(sourceText); + long matchCount = keywordTokens.stream() + .filter(sourceTokens::contains) + .count(); + if (matchCount < 1) { + return false; + } + if (keywordTokens.size() == 1) { + String token = keywordTokens.iterator().next(); + return token.length() >= 3 && sourceTokens.contains(token); + } + return (double) matchCount / keywordTokens.size() >= MIN_KEYWORD_TOKEN_MATCH_RATIO; + } + + private Set coreTokens(String value) { + Set tokens = new LinkedHashSet<>(); + if (!StringUtils.hasText(value)) { + return tokens; + } + + Matcher matcher = TOKEN_PATTERN.matcher(value.toLowerCase(Locale.ROOT)); + while (matcher.find()) { + String token = stripKoreanSuffix(matcher.group()); + if (isCoreToken(token)) { + tokens.add(token); + } + } + return tokens; + } + + private boolean isCoreToken(String token) { + return StringUtils.hasText(token) + && token.length() >= 2 + && !STOP_WORDS.contains(token); + } + + private String stripKoreanSuffix(String token) { + if (token == null || !KOREAN_ONLY_PATTERN.matcher(token).matches()) { + return token == null ? "" : token; + } + String result = token; + boolean changed; + do { + changed = false; + for (String suffix : KOREAN_SUFFIXES) { + if (result.length() > suffix.length() + 1 && result.endsWith(suffix)) { + result = result.substring(0, result.length() - suffix.length()); + changed = true; + break; + } + } + } while (changed); + return result; + } + + private boolean containsExact(String sourceText, String value) { + return StringUtils.hasText(sourceText) + && StringUtils.hasText(value) + && sourceText.contains(value); + } + + private boolean containsNormalized(String sourceText, String keyword) { + return StringUtils.hasText(sourceText) + && StringUtils.hasText(keyword) + && normalize(sourceText).contains(normalize(keyword)); + } + + private boolean equalsNormalized(String left, String right) { + return StringUtils.hasText(left) + && StringUtils.hasText(right) + && normalize(left).equals(normalize(right)); + } + + private boolean isCopiedFromAnotherAnswerSentence(String sentence, String answer, String improvement) { + if (!StringUtils.hasText(answer) || !StringUtils.hasText(improvement)) { + return false; + } + return containsNormalized(answer, improvement) && !equalsNormalized(sentence, improvement); + } + + private boolean isMetaImprovement(String improvement) { + String normalized = normalize(improvement); + for (String term : META_IMPROVEMENT_TERMS) { + if (normalized.contains(term)) { + return true; + } + } + return false; + } + + private boolean isNullLikeImprovement(String improvement) { + String normalized = normalize(improvement); + return "null".equals(normalized) + || "n/a".equals(normalized) + || "na".equals(normalized) + || "없음".equals(normalized); + } + + private boolean changesSentenceTense(String sentence, String improvement) { + return isPastSentence(sentence) && isFutureSentence(improvement) + || isFutureSentence(sentence) && isPastSentence(improvement); + } + + private boolean isPastSentence(String value) { + if (!StringUtils.hasText(value)) { + return false; + } + String normalized = normalize(value); + return normalized.contains("했습니다") + || normalized.contains("였습니다") + || normalized.contains("수행했습니다") + || normalized.contains("개선했습니다") + || normalized.contains("달성했습니다") + || normalized.contains("근무했습니다") + || normalized.contains("담당했습니다"); + } + + private boolean isFutureSentence(String value) { + if (!StringUtils.hasText(value)) { + return false; + } + String normalized = normalize(value); + return normalized.contains("하겠습니다") + || normalized.contains("되겠습니다") + || normalized.contains("기여하겠습니다") + || normalized.contains("노력하겠습니다") + || normalized.contains("성장하겠습니다") + || normalized.contains("싶습니다"); + } + + private String normalize(String value) { + return normalizeStatic(value); + } + + private static Set normalizedSet(String... values) { + return Arrays.stream(values) + .map(EvaluationSanitizationService::normalizeStatic) + .collect(LinkedHashSet::new, Set::add, Set::addAll); + } + + private static List compactList(String... values) { + return Arrays.stream(values) + .map(EvaluationSanitizationService::compactWhitespace) + .toList(); + } + + private static String compactWhitespace(String value) { + return value == null ? "" : WHITESPACE_PATTERN.matcher(value).replaceAll(""); + } + + private static String normalizeStatic(String value) { + return compactWhitespace(value).toLowerCase(Locale.ROOT); + } + + private record IndexedKeyword( + int index, + String keyword ) { - return MissingKeywordSanitizer.sanitize(mainTasks, qualifications, answer, candidates); } } diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/infrastructure/analysis/AnalysisAiEvaluationAnalysisGenerator.java similarity index 53% rename from src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java rename to src/main/java/com/jobdri/jobdri_api/domain/evaluation/infrastructure/analysis/AnalysisAiEvaluationAnalysisGenerator.java index 9ebbc37..0bcf9ed 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/infrastructure/analysis/AnalysisAiEvaluationAnalysisGenerator.java @@ -1,13 +1,17 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.adapter; +package com.jobdri.jobdri_api.domain.evaluation.infrastructure.analysis; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient; import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult; import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput; import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationCandidateReviewSnapshotParser; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationCandidateSnapshotParser; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationLlmSnapshotParser; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; -import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -16,12 +20,28 @@ import java.util.List; @Component -@RequiredArgsConstructor public class AnalysisAiEvaluationAnalysisGenerator implements EvaluationAnalysisGenerator { private static final Long EVALUATION_QUESTION_ID = 1L; private final AnalysisAiClient analysisAiClient; private final JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider; + private final ObjectMapper objectMapper; + private final EvaluationLlmSnapshotParser llmSnapshotParser; + private final EvaluationCandidateSnapshotParser candidateSnapshotParser; + private final EvaluationCandidateReviewSnapshotParser reviewSnapshotParser; + + public AnalysisAiEvaluationAnalysisGenerator( + AnalysisAiClient analysisAiClient, + JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider, + ObjectMapper objectMapper + ) { + this.analysisAiClient = analysisAiClient; + this.jobCategoryEvaluationCriteriaProvider = jobCategoryEvaluationCriteriaProvider; + this.objectMapper = objectMapper; + this.llmSnapshotParser = new EvaluationLlmSnapshotParser(objectMapper); + this.candidateSnapshotParser = new EvaluationCandidateSnapshotParser(objectMapper); + this.reviewSnapshotParser = new EvaluationCandidateReviewSnapshotParser(objectMapper); + } @Value("${evaluation.analysis.case-timeout.single-pass-seconds:70}") private long singlePassCaseTimeoutSeconds; @@ -56,11 +76,22 @@ public EvaluationGeneratedResult generate(EvaluationAnalysisCommand command) { .orElse(null), deadline ); + String rawLlmResponseJson = writeJson(aiCallResult.response()); + String rawCandidateResponseJson = writeJson(aiCallResult.rawCandidateResponse()); + String sanitizedCandidateResponseJson = writeJson(aiCallResult.sanitizedCandidateResponse()); + String candidateReviewResponseJson = writeJson(aiCallResult.candidateReviewResponse()); return new EvaluationGeneratedResult( - aiCallResult.response(), - aiCallResult.rawCandidateResponse(), - aiCallResult.sanitizedCandidateResponse(), - aiCallResult.candidateReviewResponse(), + llmSnapshotParser.parseRawLlmResponse(rawLlmResponseJson), + rawLlmResponseJson, + rawCandidateResponseJson, + sanitizedCandidateResponseJson, + candidateSnapshotParser.parse( + sanitizedCandidateResponseJson, + "sanitizedCandidateResponseJson", + command.caseId() + ), + candidateReviewResponseJson, + reviewSnapshotParser.parse(candidateReviewResponseJson), aiCallResult.candidateCallLatencyMs(), aiCallResult.finalCallLatencyMs() ); @@ -74,4 +105,12 @@ private Duration resolveCaseBudget() { }; return Duration.ofSeconds(Math.max(1L, seconds)); } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize evaluation analysis result.", e); + } + } } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java index 0c3e030..a7e9860 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java @@ -4,6 +4,9 @@ import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.CandidateReviewResponse; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationCandidateReviewSnapshotParser; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationCandidateSnapshotParser; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationLlmSnapshotParser; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; @@ -34,6 +37,8 @@ class EvaluationAnalysisBatchServiceTest { @TempDir Path tempDir; + private final ObjectMapper objectMapper = new ObjectMapper(); + @Test @DisplayName("LLM 응답을 검증해 평가 결과 CSV로 저장한다") void runWritesSanitizedEvaluationResults() throws Exception { @@ -450,7 +455,7 @@ void runWritesValidatedCandidateDecisionCounts() throws Exception { ObjectMapper objectMapper = new ObjectMapper(); EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); when(generator.generate(any())) - .thenReturn(new EvaluationGeneratedResult( + .thenReturn(result( new AnalysisLlmResponse( 80, 70, @@ -464,7 +469,6 @@ void runWritesValidatedCandidateDecisionCounts() throws Exception { ), List.of() ), - null, new AnalysisCandidateResponse( List.of(), List.of( @@ -564,6 +568,42 @@ void runWritesValidatedCandidateDecisionCounts() throws Exception { } private EvaluationGeneratedResult result(AnalysisLlmResponse response) { - return new EvaluationGeneratedResult(response, null, null, null, 0, 1); + try { + return result(response, null, null, 0, 1); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private EvaluationGeneratedResult result( + AnalysisLlmResponse response, + AnalysisCandidateResponse sanitizedCandidateResponse, + CandidateReviewResponse candidateReviewResponse, + long candidateCallLatencyMs, + long finalCallLatencyMs + ) throws Exception { + String rawLlmResponseJson = objectMapper.writeValueAsString(response == null ? List.of() : response); + String rawCandidateResponseJson = objectMapper.writeValueAsString(List.of()); + String sanitizedCandidateResponseJson = objectMapper.writeValueAsString( + sanitizedCandidateResponse == null ? List.of() : sanitizedCandidateResponse + ); + String candidateReviewResponseJson = objectMapper.writeValueAsString( + candidateReviewResponse == null ? List.of() : candidateReviewResponse + ); + return new EvaluationGeneratedResult( + new EvaluationLlmSnapshotParser(objectMapper).parseRawLlmResponse(rawLlmResponseJson), + rawLlmResponseJson, + rawCandidateResponseJson, + sanitizedCandidateResponseJson, + new EvaluationCandidateSnapshotParser(objectMapper).parse( + sanitizedCandidateResponseJson, + "sanitizedCandidateResponseJson", + "TEST" + ), + candidateReviewResponseJson, + new EvaluationCandidateReviewSnapshotParser(objectMapper).parse(candidateReviewResponseJson), + candidateCallLatencyMs, + finalCallLatencyMs + ); } } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisPackageDependencyTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisPackageDependencyTest.java new file mode 100644 index 0000000..af5bb3a --- /dev/null +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisPackageDependencyTest.java @@ -0,0 +1,53 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class EvaluationAnalysisPackageDependencyTest { + + private static final Path EVALUATION_ANALYSIS_ROOT = Path.of( + "src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis" + ); + private static final List FORBIDDEN_IMPORT_PREFIXES = List.of( + "com.jobdri.jobdri_api.domain.analysis.dto.", + "com.jobdri.jobdri_api.domain.analysis.service." + ); + + @Test + @DisplayName("evaluation analysis 패키지는 runtime analysis dto/service에 직접 의존하지 않는다") + void evaluationAnalysisPackageDoesNotDependOnRuntimeAnalysisDtoOrService() throws IOException { + List violations = new ArrayList<>(); + + try (var paths = Files.walk(EVALUATION_ANALYSIS_ROOT)) { + paths.filter(path -> path.toString().endsWith(".java")) + .forEach(path -> collectViolations(path, violations)); + } + + assertThat(violations).isEmpty(); + } + + private void collectViolations(Path file, List violations) { + try { + List lines = Files.readAllLines(file); + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i).stripLeading(); + for (String forbiddenPrefix : FORBIDDEN_IMPORT_PREFIXES) { + if (line.startsWith("import " + forbiddenPrefix) + || line.startsWith("import static " + forbiddenPrefix)) { + violations.add(file + ":" + (i + 1) + " -> " + forbiddenPrefix); + } + } + } + } catch (IOException e) { + throw new IllegalStateException("Failed to inspect file: " + file, e); + } + } +} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayServiceTest.java index b5862f7..72fba44 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayServiceTest.java @@ -85,6 +85,84 @@ void replayFailsWhenExistingSanitizedDoesNotMatch() throws Exception { .hasMessageContaining("EV-01"); } + @Test + @DisplayName("blank missing keyword 후보는 parser 단계에서 fail-fast 한다") + void blankMissingKeywordCandidateFailsFast() throws Exception { + Path input = tempDir.resolve("input.csv"); + Map row = row("EV-01", response(List.of()), response(List.of())); + row.put("rawCandidateResponseJson", """ + {"missingKeywordCandidates":[{"keyword":" ","source":"MAIN_TASK","relatedRequirement":"재고 관리"}]} + """); + EvaluationCsvSupport.writeRows(input, headers(), List.of(row)); + + assertThatThrownBy(() -> service.replay(input, tempDir.resolve("replay.csv"), tempDir.resolve("review.csv"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("rawCandidateResponseJson") + .hasMessageContaining("missingKeywordCandidates[0].keyword") + .hasMessageContaining("EV-01"); + } + + @Test + @DisplayName("unknown source 후보는 parser 단계에서 fail-fast 한다") + void unknownSourceCandidateFailsFast() throws Exception { + Path input = tempDir.resolve("input.csv"); + Map row = row("EV-01", response(List.of()), response(List.of())); + row.put("rawCandidateResponseJson", """ + {"missingKeywordCandidates":[{"keyword":"재고 분석 경험","source":"UNKNOWN","relatedRequirement":"재고 분석 경험"}]} + """); + EvaluationCsvSupport.writeRows(input, headers(), List.of(row)); + + assertThatThrownBy(() -> service.replay(input, tempDir.resolve("replay.csv"), tempDir.resolve("review.csv"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("rawCandidateResponseJson") + .hasMessageContaining("missingKeywordCandidates[0].source") + .hasMessageContaining("EV-01"); + } + + @Test + @DisplayName("relatedRequirement가 다르면 기존 sanitized JSON과 mismatch로 fail-fast 한다") + void replayFailsWhenRelatedRequirementDoesNotMatch() throws Exception { + Path input = tempDir.resolve("input.csv"); + AnalysisCandidateResponse raw = response(List.of(candidate("재고 관리 및 분석 경험", "MAIN_TASK"))); + AnalysisCandidateResponse sanitized = new AnalysisCandidateResponse( + List.of(), + List.of(), + List.of(new AnalysisCandidateResponse.MissingKeywordCandidate( + "재고 관리 및 분석 경험", + "MAIN_TASK", + "다른 요구사항" + )) + ); + writeInput(input, List.of(row("EV-01", raw, sanitized))); + + assertThatThrownBy(() -> service.replay(input, tempDir.resolve("replay.csv"), tempDir.resolve("review.csv"))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("accepted candidates mismatch") + .hasMessageContaining("EV-01"); + } + + @Test + @DisplayName("legacy sanitized JSON에 relatedRequirement가 비어 있으면 mismatch로 보지 않는다") + void replayAllowsLegacyMissingRelatedRequirement() throws Exception { + Path input = tempDir.resolve("input.csv"); + AnalysisCandidateResponse raw = response(List.of(candidate("재고 관리 및 분석 경험", "MAIN_TASK"))); + AnalysisCandidateResponse sanitized = new AnalysisCandidateResponse( + List.of(), + List.of(), + List.of(new AnalysisCandidateResponse.MissingKeywordCandidate( + "재고 관리 및 분석 경험", + "MAIN_TASK", + null + )) + ); + writeInput(input, List.of(row("EV-01", raw, sanitized))); + + MissingKeywordSanitizerReplayService.ReplaySummary summary = + service.replay(input, tempDir.resolve("replay.csv"), tempDir.resolve("review.csv")); + + assertThat(summary.acceptedCandidateCount()).isEqualTo(1); + } + @Test @DisplayName("malformed rawCandidateResponseJson은 fail-fast 한다") void malformedRawCandidateResponseFailsFast() throws Exception { diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParserTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParserTest.java new file mode 100644 index 0000000..4ddc1c8 --- /dev/null +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateReviewSnapshotParserTest.java @@ -0,0 +1,64 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateReviewSnapshot; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class EvaluationCandidateReviewSnapshotParserTest { + + private final EvaluationCandidateReviewSnapshotParser parser = + new EvaluationCandidateReviewSnapshotParser(new ObjectMapper()); + + @Test + @DisplayName("candidate review json을 evaluation snapshot으로 파싱한다") + void parsesReviewJson() { + String json = """ + { + "decisions": [ + {"accepted": true, "rejectionCode": "NONE"}, + {"accepted": false, "rejectionCode": "NOT_ACTIONABLE"} + ] + } + """; + + EvaluationCandidateReviewSnapshot snapshot = parser.parse(json); + + assertThat(snapshot.decisions()) + .extracting(decision -> decision.accepted() + ":" + decision.rejectionCode()) + .containsExactly("true:NONE", "false:NOT_ACTIONABLE"); + } + + @Test + @DisplayName("candidate review json이 비어 있거나 malformed면 empty snapshot을 반환한다") + void returnsEmptySnapshotForEmptyOrMalformedJson() { + assertThat(parser.parse(null).decisions()).isEmpty(); + assertThat(parser.parse("").decisions()).isEmpty(); + assertThat(parser.parse("{bad-json").decisions()).isEmpty(); + } + + @Test + @DisplayName("decisions가 없거나 배열이 아니면 empty snapshot을 반환한다") + void returnsEmptySnapshotWhenDecisionsMissingOrNotArray() { + assertThat(parser.parse("{\"other\":[]}").decisions()).isEmpty(); + assertThat(parser.parse("{\"decisions\":{}}").decisions()).isEmpty(); + } + + @Test + @DisplayName("accepted가 boolean이 아니면 null로 파싱한다") + void parsesNonBooleanAcceptedAsNull() { + EvaluationCandidateReviewSnapshot snapshot = parser.parse(""" + { + "decisions": [ + {"accepted": "true", "rejectionCode": "NONE"} + ] + } + """); + + assertThat(snapshot.decisions()).singleElement() + .extracting(decision -> decision.accepted() + ":" + decision.rejectionCode()) + .isEqualTo("null:NONE"); + } +} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapperTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapperTest.java deleted file mode 100644 index 42c0f62..0000000 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotMapperTest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateSnapshot; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -class EvaluationCandidateSnapshotMapperTest { - - @Test - @DisplayName("candidate response를 evaluation snapshot으로 변환한다") - void mapsCandidateResponseToEvaluationSnapshot() { - AnalysisCandidateResponse response = new AnalysisCandidateResponse( - List.of(), - List.of(), - List.of( - new AnalysisCandidateResponse.MissingKeywordCandidate("Spring Boot", "qualification", "Spring Boot 실무 경험"), - new AnalysisCandidateResponse.MissingKeywordCandidate("ignored", "unknown", "N/A") - ) - ); - - EvaluationCandidateSnapshot snapshot = EvaluationCandidateSnapshotMapper.from(response); - - assertThat(snapshot.missingKeywordCandidates()) - .extracting(candidate -> candidate.keyword() + ":" + candidate.source().name()) - .containsExactly("Spring Boot:QUALIFICATION"); - } -} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParserTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParserTest.java new file mode 100644 index 0000000..cf6cbdb --- /dev/null +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationCandidateSnapshotParserTest.java @@ -0,0 +1,73 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationCandidateSnapshot; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class EvaluationCandidateSnapshotParserTest { + + private final EvaluationCandidateSnapshotParser parser = new EvaluationCandidateSnapshotParser(new ObjectMapper()); + + @Test + @DisplayName("candidate response json을 evaluation snapshot으로 파싱한다") + void parsesCandidateResponseJson() { + String json = """ + { + "strengthCandidates": [{"quote": "강점"}], + "analysisCandidates": [{"candidateId": "candidate-1"}], + "missingKeywordCandidates": [ + {"keyword": "Spring Boot", "source": "qualification", "relatedRequirement": "Spring Boot 실무 경험"}, + {"keyword": "ignored", "source": "unknown", "relatedRequirement": "N/A"} + ] + } + """; + + EvaluationCandidateSnapshot snapshot = parser.parse(json, "rawCandidateResponseJson", "EV-01"); + + assertThat(snapshot.strengthCandidates()).hasSize(1); + assertThat(snapshot.strengthCandidates()) + .extracting(node -> ((JsonNode) node).path("quote").asText()) + .containsExactly("강점"); + assertThat(snapshot.analysisCandidates()).hasSize(1); + assertThat(snapshot.analysisCandidates()) + .extracting(node -> ((JsonNode) node).path("candidateId").asText()) + .containsExactly("candidate-1"); + assertThat(snapshot.missingKeywordCandidates()) + .extracting(candidate -> candidate.keyword() + ":" + candidate.source().name()) + .containsExactly("Spring Boot:QUALIFICATION"); + } + + @Test + @DisplayName("candidate response json이 malformed면 caseId를 포함해 fail-fast 한다") + void rejectsMalformedCandidateResponseJson() { + assertThatThrownBy(() -> parser.parse("{bad-json", "rawCandidateResponseJson", "EV-01")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("rawCandidateResponseJson") + .hasMessageContaining("EV-01"); + } + + @Test + @DisplayName("candidate 배열 필드가 없거나 배열이 아니면 빈 목록으로 파싱한다") + void returnsEmptyListsWhenCandidateArraysMissingOrNotArray() { + EvaluationCandidateSnapshot missingArrays = parser.parse("{}", "rawCandidateResponseJson", "EV-01"); + EvaluationCandidateSnapshot nonArrayFields = parser.parse(""" + { + "strengthCandidates": {}, + "analysisCandidates": "", + "missingKeywordCandidates": 1 + } + """, "rawCandidateResponseJson", "EV-01"); + + assertThat(missingArrays.strengthCandidates()).isEmpty(); + assertThat(missingArrays.analysisCandidates()).isEmpty(); + assertThat(missingArrays.missingKeywordCandidates()).isEmpty(); + assertThat(nonArrayFields.strengthCandidates()).isEmpty(); + assertThat(nonArrayFields.analysisCandidates()).isEmpty(); + assertThat(nonArrayFields.missingKeywordCandidates()).isEmpty(); + } +} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapperTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapperTest.java deleted file mode 100644 index f43dcc9..0000000 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotMapperTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.jobdri.jobdri_api.domain.evaluation.analysis.mapper; - -import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; -import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationLlmSnapshot; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -class EvaluationLlmSnapshotMapperTest { - - @Test - @DisplayName("runtime llm response를 evaluation snapshot으로 변환한다") - void mapsAnalysisLlmResponseToEvaluationSnapshot() { - AnalysisLlmResponse response = new AnalysisLlmResponse( - 4, - 4, - 4, - "feedback", - List.of( - new AnalysisLlmResponse.HighlightItem("강점", "문장 A") - ), - List.of(), - List.of( - new AnalysisLlmResponse.MissingKeywordItem("Spring Boot", "mainTask"), - new AnalysisLlmResponse.MissingKeywordItem("무시됨", "unknown") - ), - List.of( - new AnalysisLlmResponse.QuestionAnalysisItem( - 1L, - "문장", - "mentioned", - "근거", - "개선" - ) - ) - ); - - EvaluationLlmSnapshot snapshot = EvaluationLlmSnapshotMapper.from(response); - - assertThat(snapshot.keyStrengthQuotes()).containsExactly("문장 A"); - assertThat(snapshot.missingKeywords()) - .extracting(keyword -> keyword.keyword() + ":" + keyword.source().name()) - .containsExactly("Spring Boot:MAIN_TASK"); - assertThat(snapshot.questionAnalyses()) - .extracting(questionAnalysis -> questionAnalysis.sentence() + ":" + questionAnalysis.status()) - .containsExactly("문장:mentioned"); - } -} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParserTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParserTest.java index a9a7eec..d0f12d4 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParserTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/mapper/EvaluationLlmSnapshotParserTest.java @@ -27,6 +27,10 @@ void parsesRawLlmResponseJson() { EvaluationLlmSnapshot snapshot = parser.parseRawLlmResponse(rawLlmResponseJson); + assertThat(snapshot.jobFit()).isNull(); + assertThat(snapshot.impact()).isNull(); + assertThat(snapshot.completeness()).isNull(); + assertThat(snapshot.feedback()).isEmpty(); assertThat(snapshot.keyStrengthQuotes()).containsExactly("강점 문장"); assertThat(snapshot.missingKeywords()) .extracting(keyword -> keyword.keyword() + ":" + keyword.source().name()) @@ -36,6 +40,22 @@ void parsesRawLlmResponseJson() { .containsExactly("문장:mentioned"); } + @Test + @DisplayName("숫자 점수 필드를 모두 파싱한다") + void parsesNumericScores() { + EvaluationLlmSnapshot snapshot = parser.parseRawLlmResponse(""" + { + "jobFit": 80, + "impact": 70, + "completeness": 60 + } + """); + + assertThat(snapshot.jobFit()).isEqualTo(80); + assertThat(snapshot.impact()).isEqualTo(70); + assertThat(snapshot.completeness()).isEqualTo(60); + } + @Test @DisplayName("missing keyword json을 evaluation 모델로 파싱한다") void parsesMissingKeywordJson() {