diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/infrastructure/ai/OpenAiAnalysisAdapter.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/infrastructure/ai/OpenAiAnalysisAdapter.java index 08daf24..49ba7b6 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/infrastructure/ai/OpenAiAnalysisAdapter.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/infrastructure/ai/OpenAiAnalysisAdapter.java @@ -4,12 +4,15 @@ import com.jobdri.jobdri_api.global.config.LlmConcurrencyLimiter; import com.jobdri.jobdri_api.global.metrics.AsyncMetricsRecorder; import com.openai.client.OpenAIClient; +import com.openai.core.RequestOptions; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.StructuredResponse; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.time.Duration; + @Component @RequiredArgsConstructor public class OpenAiAnalysisAdapter { @@ -22,18 +25,32 @@ public class OpenAiAnalysisAdapter { private String analysisModel; public T createStructuredResponse(String operationName, String prompt, Class responseType) { + return createStructuredResponse(operationName, prompt, responseType, null); + } + + public T createStructuredResponse( + String operationName, + String prompt, + Class responseType, + Duration timeout + ) { var params = ResponseCreateParams.builder() .model(analysisModel) .input(prompt) .temperature(0.2) .text(responseType) .build(); + RequestOptions requestOptions = timeout == null + ? null + : RequestOptions.builder().timeout(timeout).build(); long startedAt = System.nanoTime(); boolean success = false; try { StructuredResponse response = llmConcurrencyLimiter.execute( operationName, - () -> openAIClient.responses().create(params) + () -> requestOptions == null + ? openAIClient.responses().create(params) + : openAIClient.responses().create(params, requestOptions) ); T structuredContent = analysisResponseParser.extractStructuredContent(response); success = true; diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/AnalysisAiClient.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/AnalysisAiClient.java index 88d93be..56e72fb 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/AnalysisAiClient.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/AnalysisAiClient.java @@ -26,6 +26,8 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import java.time.Duration; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.HashMap; @@ -100,19 +102,22 @@ public AnalysisLlmResponse analyze( promptInput, referenceContext, jobCategoryEvaluationCriteria, - "cover-letter-analysis" + "cover-letter-analysis", + null ).response(); case HYBRID_EXACT -> analyzeHybridExact( promptInput, referenceContext, jobCategoryEvaluationCriteria, - "cover-letter-analysis" + "cover-letter-analysis", + null ).response(); case SINGLE_PASS -> analyzeSinglePass( promptInput, referenceContext, jobCategoryEvaluationCriteria, - "cover-letter-analysis" + "cover-letter-analysis", + null ).response(); }; } catch (GeneralException e) { @@ -155,6 +160,14 @@ public AnalysisLlmResponse analyzeForEvaluation( public AnalysisAiCallResult analyzeForEvaluationResult( AnalysisPromptInput promptInput, JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria + ) { + return analyzeForEvaluationResult(promptInput, jobCategoryEvaluationCriteria, null); + } + + public AnalysisAiCallResult analyzeForEvaluationResult( + AnalysisPromptInput promptInput, + JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, + Instant deadline ) { try { return switch (resolveAnalysisMode()) { @@ -162,19 +175,22 @@ public AnalysisAiCallResult analyzeForEvaluationResult( promptInput, emptyContext(), jobCategoryEvaluationCriteria, - "cover-letter-analysis-evaluation" + "cover-letter-analysis-evaluation", + deadline ); case HYBRID_EXACT -> analyzeHybridExact( promptInput, emptyContext(), jobCategoryEvaluationCriteria, - "cover-letter-analysis-evaluation" + "cover-letter-analysis-evaluation", + deadline ); case SINGLE_PASS -> analyzeSinglePass( promptInput, emptyContext(), jobCategoryEvaluationCriteria, - "cover-letter-analysis-evaluation" + "cover-letter-analysis-evaluation", + deadline ); }; } catch (GeneralException e) { @@ -192,13 +208,15 @@ private AnalysisAiCallResult analyzeSinglePass( AnalysisPromptInput promptInput, RetrievalContext referenceContext, JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, - String operationName + String operationName, + Instant deadline ) { long startedAt = System.nanoTime(); AnalysisLlmResponse response = createStructuredResponse( operationName, buildPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria), - AnalysisLlmResponse.class + AnalysisLlmResponse.class, + deadline ); response = analysisResponseParser.sanitizeSinglePassSubheadings(promptInput, response); return AnalysisAiCallResult.singlePass(response, elapsedMillis(startedAt)); @@ -208,13 +226,15 @@ private AnalysisAiCallResult analyzeTwoPass( AnalysisPromptInput promptInput, RetrievalContext referenceContext, JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, - String operationName + String operationName, + Instant deadline ) { long candidateStartedAt = System.nanoTime(); AnalysisCandidateResponse rawCandidates = createStructuredResponse( operationName + "-candidates", buildCandidatePrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria), - AnalysisCandidateResponse.class + AnalysisCandidateResponse.class, + deadline ); long candidateLatencyMs = elapsedMillis(candidateStartedAt); AnalysisCandidateResponse sanitizedCandidates = sanitizeCandidates(promptInput, rawCandidates); @@ -235,7 +255,8 @@ private AnalysisAiCallResult analyzeTwoPass( CandidateReviewResponse reviewResponse = createStructuredResponse( operationName + "-final", buildFinalPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria, sanitizedCandidates), - CandidateReviewResponse.class + CandidateReviewResponse.class, + deadline ); CandidateReviewResponse validatedReviewResponse = validateCandidateReview( promptInput, @@ -257,7 +278,8 @@ private AnalysisAiCallResult analyzeTwoPass( jobCategoryEvaluationCriteria, sanitizedCandidates, validatedReviewResponse, - operationName + operationName, + deadline ); AnalysisLlmResponse response = buildFinalResponse(promptInput, sanitizedCandidates, recheckedReviewResponse); long finalLatencyMs = elapsedMillis(finalStartedAt); @@ -294,19 +316,22 @@ private AnalysisAiCallResult analyzeHybridExact( AnalysisPromptInput promptInput, RetrievalContext referenceContext, JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, - String operationName + String operationName, + Instant deadline ) { AnalysisAiCallResult singlePassResult = analyzeSinglePass( promptInput, referenceContext, jobCategoryEvaluationCriteria, - operationName + "-single-pass" + operationName + "-single-pass", + deadline ); AnalysisAiCallResult twoPassResult = analyzeTwoPass( promptInput, referenceContext, jobCategoryEvaluationCriteria, - operationName + "-two-pass" + operationName + "-two-pass", + deadline ); AnalysisLlmResponse merged = mergeHybridExact( singlePassResult.response(), @@ -335,6 +360,29 @@ private T createStructuredResponse(String operationName, String prompt, Clas return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType); } + private T createStructuredResponse( + String operationName, + String prompt, + Class responseType, + Instant deadline + ) { + if (deadline == null) { + return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType); + } + Duration remaining = Duration.between(Instant.now(), deadline); + if (remaining.isZero() || remaining.isNegative()) { + throw new GeneralException( + GeneralErrorCode.EXTERNAL_SERVICE_TIMEOUT, + "평가 사례 처리 시간이 제한을 초과했습니다." + ); + } + return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType, remaining); + } + + private T createStructuredResponse(String operationName, String prompt, Class responseType, Duration timeout) { + return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType); + } + String buildPrompt( JobPosting jobPosting, List questions, @@ -553,7 +601,8 @@ CandidateReviewResponse recheckWhenAllCandidatesRejected( JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria, AnalysisCandidateResponse sanitizedCandidates, CandidateReviewResponse reviewResponse, - String operationName + String operationName, + Instant deadline ) { int firstPassCandidates = sanitizedCandidates == null || sanitizedCandidates.analysisCandidates() == null ? 0 @@ -573,7 +622,8 @@ CandidateReviewResponse recheckWhenAllCandidatesRejected( CandidateRecheckResponse recheckResponse = createStructuredResponse( operationName + "-recheck", buildRecheckPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria, sanitizedCandidates, reviewResponse), - CandidateRecheckResponse.class + CandidateRecheckResponse.class, + deadline ); CandidateReviewResponse rechecked = applyRecheckResponse(promptInput, sanitizedCandidates, reviewResponse, recheckResponse); int recoveredMentionedCount = recoveredDecisionCount(rechecked, QuestionAnalysisStatus.MENTIONED); @@ -1635,7 +1685,7 @@ private RetrievalContext emptyContext() { return new RetrievalContext(List.of(), List.of()); } - AnalysisMode resolveAnalysisMode() { + public AnalysisMode resolveAnalysisMode() { if (StringUtils.hasText(analysisMode)) { String normalized = analysisMode.trim().replace('-', '_').toUpperCase(java.util.Locale.ROOT); try { @@ -1647,7 +1697,7 @@ AnalysisMode resolveAnalysisMode() { return twoPassEnabled ? AnalysisMode.TWO_PASS : AnalysisMode.SINGLE_PASS; } - enum AnalysisMode { + public enum AnalysisMode { SINGLE_PASS, TWO_PASS, HYBRID_EXACT diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java index 0808cc7..f66bbb1 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchService.java @@ -7,14 +7,13 @@ import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordResponse; import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; import com.jobdri.jobdri_api.domain.analysis.type.QuestionAnalysisStatus; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput; -import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider; import com.jobdri.jobdri_api.domain.analysis.service.core.AnalysisResultConstants; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules; -import lombok.RequiredArgsConstructor; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; +import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -31,7 +30,6 @@ import java.util.Set; @Service -@RequiredArgsConstructor @Slf4j public class EvaluationAnalysisBatchService { private static final Long EVALUATION_QUESTION_ID = 1L; @@ -46,9 +44,31 @@ public class EvaluationAnalysisBatchService { "answer" ); - private final AnalysisAiClient analysisAiClient; - private final JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider; + private final EvaluationAnalysisGenerator evaluationAnalysisGenerator; private final ObjectMapper objectMapper; + private final EvaluationSanitizationService evaluationSanitizationService; + + @Autowired + public EvaluationAnalysisBatchService( + EvaluationAnalysisGenerator evaluationAnalysisGenerator, + ObjectMapper objectMapper, + EvaluationSanitizationService evaluationSanitizationService + ) { + this.evaluationAnalysisGenerator = evaluationAnalysisGenerator; + this.objectMapper = objectMapper; + this.evaluationSanitizationService = evaluationSanitizationService; + } + + EvaluationAnalysisBatchService( + EvaluationAnalysisGenerator evaluationAnalysisGenerator, + ObjectMapper objectMapper + ) { + this( + evaluationAnalysisGenerator, + objectMapper, + new EvaluationSanitizationService() + ); + } public EvaluationBatchSummary run(Path inputPath, Path outputPath) throws IOException { List cases = readCases(inputPath); @@ -106,27 +126,17 @@ private void validateHeaders(List headers) { } private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCase) { - AnalysisPromptInput promptInput = new AnalysisPromptInput( + EvaluationGeneratedResult generatedResult = evaluationAnalysisGenerator.generate(new EvaluationAnalysisCommand( evaluationCase.caseId(), - "평가용 회사", + evaluationCase.jobCategoryMiddle(), evaluationCase.jobCategorySmall(), evaluationCase.mainTasks(), evaluationCase.qualifications(), evaluationCase.preferences(), - List.of(new AnalysisPromptInput.QuestionAnswer( - EVALUATION_QUESTION_ID, - evaluationCase.question(), - evaluationCase.answer() - )) - ); - - AnalysisAiCallResult aiCallResult = analysisAiClient.analyzeForEvaluationResult( - promptInput, - jobCategoryEvaluationCriteriaProvider - .findByMiddleName(evaluationCase.jobCategoryMiddle()) - .orElse(null) - ); - AnalysisLlmResponse llmResponse = aiCallResult.response(); + evaluationCase.question(), + evaluationCase.answer() + )); + AnalysisLlmResponse llmResponse = generatedResult.response(); int jobFit = validateScore("jobFit", llmResponse == null ? null : llmResponse.jobFit()); int impact = validateScore("impact", llmResponse == null ? null : llmResponse.impact()); @@ -136,7 +146,7 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa log.debug( "Evaluation serialized missing keyword flow. caseId={}, candidateMissingKeywordCount={}, finalMissingKeywordCount={}, evaluationSerializedMissingKeywordCount={}", evaluationCase.caseId(), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), size(llmResponse == null ? null : llmResponse.missingKeywords()), missingKeywords.size() ); @@ -158,25 +168,25 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa writeJson(missingKeywords), writeJson(questionAnalyses), writeJson(llmResponse), - writeJson(aiCallResult.rawCandidateResponse()), - writeJson(aiCallResult.sanitizedCandidateResponse()), - writeJson(aiCallResult.candidateReviewResponse()), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().analysisCandidates()), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().analysisCandidates()), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().strengthCandidates()), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()), - acceptedDecisionCount(aiCallResult.candidateReviewResponse()), - rejectedDecisionCount(aiCallResult.candidateReviewResponse()), - rejectionCodeCounts(aiCallResult.candidateReviewResponse()), + writeJson(generatedResult.rawCandidateResponse()), + writeJson(generatedResult.sanitizedCandidateResponse()), + writeJson(generatedResult.candidateReviewResponse()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), + acceptedDecisionCount(generatedResult.candidateReviewResponse()), + rejectedDecisionCount(generatedResult.candidateReviewResponse()), + rejectionCodeCounts(generatedResult.candidateReviewResponse()), questionAnalyses.size(), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().strengthCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()), llmResponse.keyStrengths() == null ? 0 : llmResponse.keyStrengths().size(), - size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()), + size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()), missingKeywords.size(), - aiCallResult.candidateCallLatencyMs(), - aiCallResult.finalCallLatencyMs(), - aiCallResult.candidateCallLatencyMs(), - aiCallResult.finalCallLatencyMs(), + generatedResult.candidateCallLatencyMs(), + generatedResult.finalCallLatencyMs(), + generatedResult.candidateCallLatencyMs(), + generatedResult.finalCallLatencyMs(), null, null, null, @@ -214,7 +224,7 @@ private List buildMissingKeywords( if (source.isEmpty()) { continue; } - if (!AnalysisSanitizationRules.isValidMissingKeyword( + if (!evaluationSanitizationService.isValidMissingKeyword( keyword, source.get(), evaluationCase.mainTasks(), @@ -222,7 +232,7 @@ private List buildMissingKeywords( )) { continue; } - if (AnalysisSanitizationRules.isMissingKeywordMentionedInAnswers( + if (evaluationSanitizationService.isMissingKeywordMentionedInAnswers( keyword, evaluationCase.answer() )) { @@ -275,11 +285,11 @@ private List buildQuestionAnalyses( continue; } if (status == QuestionAnalysisStatus.PROVEN - && !AnalysisSanitizationRules.hasValidProvenReason(item.reason())) { + && !evaluationSanitizationService.hasValidProvenReason(item.reason())) { continue; } if (status == QuestionAnalysisStatus.FABRICATED - && !AnalysisSanitizationRules.hasFabricatedDirectConflictEvidence( + && !evaluationSanitizationService.hasFabricatedDirectConflictEvidence( item.sentence(), item.reason() )) { @@ -435,7 +445,7 @@ private String normalizeImprovement( String improvement, QuestionAnalysisStatus status ) { - return AnalysisSanitizationRules.normalizeImprovement( + return evaluationSanitizationService.normalizeImprovement( sentence, answer, improvement, diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java index 751aab7..06d89c8 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/MissingKeywordSanitizerReplayService.java @@ -5,9 +5,10 @@ import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationDecision; import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationResult; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizer; +import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -23,7 +24,6 @@ import java.util.stream.Collectors; @Service -@RequiredArgsConstructor @Slf4j class MissingKeywordSanitizerReplayService { private static final List REQUIRED_HEADERS = List.of( @@ -76,6 +76,20 @@ class MissingKeywordSanitizerReplayService { ); private final ObjectMapper objectMapper; + private final EvaluationSanitizationService evaluationSanitizationService; + + @Autowired + MissingKeywordSanitizerReplayService( + ObjectMapper objectMapper, + EvaluationSanitizationService evaluationSanitizationService + ) { + this.objectMapper = objectMapper; + this.evaluationSanitizationService = evaluationSanitizationService; + } + + MissingKeywordSanitizerReplayService(ObjectMapper objectMapper) { + this(objectMapper, new EvaluationSanitizationService()); + } ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOException { List headers = EvaluationCsvSupport.readHeaders(input); @@ -112,7 +126,7 @@ ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOExcept ); List rawCandidates = safeMissingKeywordCandidates(rawResponse); - MissingKeywordSanitizationResult replayResult = MissingKeywordSanitizer.sanitize( + MissingKeywordSanitizationResult replayResult = evaluationSanitizationService.sanitizeMissingKeywordCandidates( value(row, "mainTasks"), value(row, "qualifications"), value(row, "answer"), diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java index 5183acf..d286398 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/NlgEvaluationBatchService.java @@ -9,7 +9,7 @@ import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationMissingKeywordSourceMapper; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword; import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; -import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules; +import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.NestedExceptionUtils; @@ -66,15 +66,22 @@ class NlgEvaluationBatchService { private final NlgEvaluationAiClient nlgEvaluationAiClient; private final ObjectMapper objectMapper; private final EvaluationLlmSnapshotParser evaluationLlmSnapshotParser; + private final EvaluationSanitizationService evaluationSanitizationService; @Autowired NlgEvaluationBatchService( NlgEvaluationAiClient nlgEvaluationAiClient, - ObjectMapper objectMapper + ObjectMapper objectMapper, + EvaluationSanitizationService evaluationSanitizationService ) { this.nlgEvaluationAiClient = nlgEvaluationAiClient; this.objectMapper = objectMapper; this.evaluationLlmSnapshotParser = new EvaluationLlmSnapshotParser(objectMapper); + this.evaluationSanitizationService = evaluationSanitizationService; + } + + NlgEvaluationBatchService(NlgEvaluationAiClient nlgEvaluationAiClient, ObjectMapper objectMapper) { + this(nlgEvaluationAiClient, objectMapper, new EvaluationSanitizationService()); } NlgEvaluationSummary run(Path inputPath, Path outputPath) throws IOException { @@ -446,8 +453,8 @@ private Optional missingKeywordMissInvalidReaso if (source.isEmpty() || source.get() == EvaluationMissingKeywordSource.PREFERENCE) { return Optional.of(MissingKeywordMissInvalidReason.INVALID_SOURCE); } - if (AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.keyword()) - || AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.relatedRequirement())) { + if (evaluationSanitizationService.isStructuredQualificationKeyword(evaluation.keyword()) + || evaluationSanitizationService.isStructuredQualificationKeyword(evaluation.relatedRequirement())) { return Optional.of(MissingKeywordMissInvalidReason.STRUCTURED_QUALIFICATION); } String sourceText = source.get() == EvaluationMissingKeywordSource.MAIN_TASK @@ -456,7 +463,7 @@ private Optional missingKeywordMissInvalidReaso if (!containsNormalized(sourceText, evaluation.relatedRequirement())) { return Optional.of(MissingKeywordMissInvalidReason.RELATED_REQUIREMENT_NOT_IN_JD); } - if (!AnalysisSanitizationRules.isValidMissingKeyword( + if (!evaluationSanitizationService.isValidMissingKeyword( evaluation.keyword(), EvaluationMissingKeywordSourceMapper.toAnalysisSource(source.get()), input.mainTasks(), @@ -862,7 +869,7 @@ private boolean containsNormalized(String sourceText, String fragment) { } private String normalize(String value) { - return AnalysisSanitizationRules.normalizeText(value); + return evaluationSanitizationService.normalizeText(value); } private String truncateShortRationale(String rationale) { diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java new file mode 100644 index 0000000..9ebbc37 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/adapter/AnalysisAiEvaluationAnalysisGenerator.java @@ -0,0 +1,77 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.adapter; + +import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient; +import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult; +import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput; +import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +@Component +@RequiredArgsConstructor +public class AnalysisAiEvaluationAnalysisGenerator implements EvaluationAnalysisGenerator { + private static final Long EVALUATION_QUESTION_ID = 1L; + + private final AnalysisAiClient analysisAiClient; + private final JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider; + + @Value("${evaluation.analysis.case-timeout.single-pass-seconds:70}") + private long singlePassCaseTimeoutSeconds; + + @Value("${evaluation.analysis.case-timeout.two-pass-seconds:130}") + private long twoPassCaseTimeoutSeconds; + + @Value("${evaluation.analysis.case-timeout.hybrid-exact-seconds:190}") + private long hybridExactCaseTimeoutSeconds; + + @Override + public EvaluationGeneratedResult generate(EvaluationAnalysisCommand command) { + AnalysisPromptInput promptInput = new AnalysisPromptInput( + command.caseId(), + "평가용 회사", + command.jobCategorySmall(), + command.mainTasks(), + command.qualifications(), + command.preferences(), + List.of(new AnalysisPromptInput.QuestionAnswer( + EVALUATION_QUESTION_ID, + command.question(), + command.answer() + )) + ); + Instant deadline = Instant.now().plus(resolveCaseBudget()); + + AnalysisAiCallResult aiCallResult = analysisAiClient.analyzeForEvaluationResult( + promptInput, + jobCategoryEvaluationCriteriaProvider + .findByMiddleName(command.jobCategoryMiddle()) + .orElse(null), + deadline + ); + return new EvaluationGeneratedResult( + aiCallResult.response(), + aiCallResult.rawCandidateResponse(), + aiCallResult.sanitizedCandidateResponse(), + aiCallResult.candidateReviewResponse(), + aiCallResult.candidateCallLatencyMs(), + aiCallResult.finalCallLatencyMs() + ); + } + + private Duration resolveCaseBudget() { + long seconds = switch (analysisAiClient.resolveAnalysisMode()) { + case SINGLE_PASS -> singlePassCaseTimeoutSeconds; + case TWO_PASS -> twoPassCaseTimeoutSeconds; + case HYBRID_EXACT -> hybridExactCaseTimeoutSeconds; + }; + return Duration.ofSeconds(Math.max(1L, seconds)); + } +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationAnalysisCommand.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationAnalysisCommand.java new file mode 100644 index 0000000..847f40d --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationAnalysisCommand.java @@ -0,0 +1,13 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +public record EvaluationAnalysisCommand( + String caseId, + String jobCategoryMiddle, + String jobCategorySmall, + String mainTasks, + String qualifications, + String preferences, + String question, + String answer +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java new file mode 100644 index 0000000..314183f --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/model/EvaluationGeneratedResult.java @@ -0,0 +1,15 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.model; + +import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; +import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; +import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.CandidateReviewResponse; + +public record EvaluationGeneratedResult( + AnalysisLlmResponse response, + AnalysisCandidateResponse rawCandidateResponse, + AnalysisCandidateResponse sanitizedCandidateResponse, + CandidateReviewResponse candidateReviewResponse, + long candidateCallLatencyMs, + long finalCallLatencyMs +) { +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/port/EvaluationAnalysisGenerator.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/port/EvaluationAnalysisGenerator.java new file mode 100644 index 0000000..482a6f3 --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/port/EvaluationAnalysisGenerator.java @@ -0,0 +1,8 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.port; + +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; + +public interface EvaluationAnalysisGenerator { + EvaluationGeneratedResult generate(EvaluationAnalysisCommand command); +} diff --git a/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java new file mode 100644 index 0000000..19b940f --- /dev/null +++ b/src/main/java/com/jobdri/jobdri_api/domain/evaluation/analysis/sanitization/EvaluationSanitizationService.java @@ -0,0 +1,78 @@ +package com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization; + +import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; +import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; +import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules; +import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationResult; +import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizer; +import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationMissingKeywordSourceMapper; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class EvaluationSanitizationService { + + public boolean isValidMissingKeyword( + String keyword, + EvaluationMissingKeywordSource source, + String mainTasks, + String qualifications + ) { + MissingKeywordSource analysisSource = EvaluationMissingKeywordSourceMapper.toAnalysisSource(source); + return analysisSource != null && AnalysisSanitizationRules.isValidMissingKeyword( + keyword, + analysisSource, + mainTasks, + qualifications + ); + } + + public boolean isValidMissingKeyword( + String keyword, + MissingKeywordSource source, + String mainTasks, + String qualifications + ) { + return source != null && AnalysisSanitizationRules.isValidMissingKeyword( + keyword, + source, + mainTasks, + qualifications + ); + } + + public boolean isMissingKeywordMentionedInAnswers(String keyword, String answer) { + return AnalysisSanitizationRules.isMissingKeywordMentionedInAnswers(keyword, answer); + } + + public boolean hasValidProvenReason(String reason) { + return AnalysisSanitizationRules.hasValidProvenReason(reason); + } + + public boolean hasFabricatedDirectConflictEvidence(String sentence, String reason) { + return AnalysisSanitizationRules.hasFabricatedDirectConflictEvidence(sentence, reason); + } + + public String normalizeImprovement(String sentence, String answer, String improvement, boolean proven) { + return AnalysisSanitizationRules.normalizeImprovement(sentence, answer, improvement, proven); + } + + public boolean isStructuredQualificationKeyword(String value) { + return AnalysisSanitizationRules.isStructuredQualificationKeyword(value); + } + + public String normalizeText(String value) { + return AnalysisSanitizationRules.normalizeText(value); + } + + public MissingKeywordSanitizationResult sanitizeMissingKeywordCandidates( + String mainTasks, + String qualifications, + String answer, + List candidates + ) { + return MissingKeywordSanitizer.sanitize(mainTasks, qualifications, answer, candidates); + } +} diff --git a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java index 62c11de..0c3e030 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/evaluation/analysis/EvaluationAnalysisBatchServiceTest.java @@ -4,13 +4,15 @@ import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse; import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisLlmResponse; import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.CandidateReviewResponse; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult; -import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput; -import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand; +import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult; +import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator; +import com.jobdri.jobdri_api.global.apiPayload.code.GeneralErrorCode; +import com.jobdri.jobdri_api.global.apiPayload.exception.GeneralException; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -21,7 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -35,14 +37,10 @@ class EvaluationAnalysisBatchServiceTest { @Test @DisplayName("LLM 응답을 검증해 평가 결과 CSV로 저장한다") void runWritesSanitizedEvaluationResults() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenReturn(result(new AnalysisLlmResponse( 80, 70, @@ -103,20 +101,58 @@ void runWritesSanitizedEvaluationResults() throws Exception { assertThat(row.get("aiMissingKeywordsJson")).doesNotContain("잘못된 출처"); assertThat(row.get("aiQuestionAnalysesJson")).doesNotContain("답변에 없는 문장"); assertThat(row.get("aiQuestionAnalysesJson")).doesNotContain("missing은 저장하지 않습니다."); - verify(analysisAiClient).analyzeForEvaluationResult(any(AnalysisPromptInput.class), any()); + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(EvaluationAnalysisCommand.class); + verify(generator).generate(commandCaptor.capture()); + assertThat(commandCaptor.getValue().caseId()).isEqualTo("EV-01"); + assertThat(commandCaptor.getValue().jobCategoryMiddle()).isEqualTo("AI·개발·데이터"); + assertThat(commandCaptor.getValue().jobCategorySmall()).isEqualTo("백엔드"); + assertThat(commandCaptor.getValue().question()).isEqualTo("경험을 쓰세요"); + assertThat(commandCaptor.getValue().answer()).isEqualTo("데이터 처리 경험이 있습니다."); + } + + @Test + @DisplayName("CSV 입력 순서대로 evaluation command를 generator에 전달한다") + void runPassesEvaluationCommandsInCsvOrder() throws Exception { + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); + ObjectMapper objectMapper = new ObjectMapper(); + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())).thenReturn(result(new AnalysisLlmResponse(70, 70, 70, "피드백", List.of(), List.of()))); + + Path input = tempDir.resolve("evaluation_cases_order.csv"); + Path output = tempDir.resolve("evaluation_cases_order_results.csv"); + Files.writeString( + input, + "caseId,jobCategoryMiddle,jobCategorySmall,mainTasks,qualifications,preferences,question,answer\n" + + "EV-01,AI·개발·데이터,백엔드,API 개발,Spring,,첫 질문,첫 답변\n" + + "EV-02,디자인,프로덕트 디자이너,UX 설계,Figma,커뮤니케이션,둘째 질문,둘째 답변\n", + StandardCharsets.UTF_8 + ); + + service.run(input, output); + + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(EvaluationAnalysisCommand.class); + verify(generator, org.mockito.Mockito.times(2)).generate(commandCaptor.capture()); + assertThat(commandCaptor.getAllValues()) + .extracting( + EvaluationAnalysisCommand::caseId, + EvaluationAnalysisCommand::jobCategoryMiddle, + EvaluationAnalysisCommand::jobCategorySmall, + EvaluationAnalysisCommand::question, + EvaluationAnalysisCommand::answer + ) + .containsExactly( + org.assertj.core.groups.Tuple.tuple("EV-01", "AI·개발·데이터", "백엔드", "첫 질문", "첫 답변"), + org.assertj.core.groups.Tuple.tuple("EV-02", "디자인", "프로덕트 디자이너", "둘째 질문", "둘째 답변") + ); } @Test @DisplayName("평가 결과도 운영과 동일하게 missingKeywords와 improvement를 후처리한다") void runAppliesProductionSanitizationRules() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenReturn(result(new AnalysisLlmResponse( 80, 70, @@ -177,14 +213,10 @@ void runAppliesProductionSanitizationRules() throws Exception { @Test @DisplayName("평가 결과도 운영과 동일하게 유효한 PROVEN/FABRICATED를 보존하고 raw/final 비교 정보를 남긴다") void runKeepsRawAndAppliesFinalStatusFilter() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenReturn(result(new AnalysisLlmResponse( 80, 70, @@ -253,14 +285,10 @@ void runKeepsRawAndAppliesFinalStatusFilter() throws Exception { @Test @DisplayName("평가 저장 경로도 reason이 null 또는 빈 PROVEN을 제외한다") void runSkipsProvenWithMissingReason() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenReturn(result(new AnalysisLlmResponse( 80, 70, @@ -301,16 +329,12 @@ void runSkipsProvenWithMissingReason() throws Exception { } @Test - @DisplayName("없는 직무 중분류는 보조 기준 없이 분석한다") - void runOmitsCriteriaWhenMiddleNameNotFound() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + @DisplayName("중분류 기준 조회 여부와 무관하게 evaluation generator를 통해 분석한다") + void runDelegatesAnalysisThroughEvaluationGenerator() throws Exception { + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), isNull())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenReturn(result(new AnalysisLlmResponse(70, 70, 70, "피드백", List.of(), List.of()))); Path input = tempDir.resolve("evaluation_cases.csv"); @@ -324,19 +348,15 @@ void runOmitsCriteriaWhenMiddleNameNotFound() throws Exception { service.run(input, output); - verify(analysisAiClient).analyzeForEvaluationResult(any(AnalysisPromptInput.class), isNull()); + verify(generator).generate(any()); } @Test @DisplayName("필수 CSV header가 없으면 AI 호출 전에 실패한다") void runFailsFastWhenRequiredHeaderIsMissing() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); Path input = tempDir.resolve("evaluation_cases.csv"); Path output = tempDir.resolve("evaluation_ai_results.csv"); Files.writeString( @@ -350,20 +370,16 @@ void runFailsFastWhenRequiredHeaderIsMissing() throws Exception { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("missing required headers") .hasMessageContaining("answer"); - verifyNoInteractions(analysisAiClient); + verifyNoInteractions(generator); } @Test @DisplayName("LLM 호출 실패 케이스도 실패 row로 CSV에 기록한다") void runWritesFailureRowWhenAnalyzeFails() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) .thenThrow(new RuntimeException("rate limit exceeded")); Path input = tempDir.resolve("evaluation_cases.csv"); @@ -387,18 +403,54 @@ void runWritesFailureRowWhenAnalyzeFails() throws Exception { assertThat(row.get("errorMessage")).contains("rate limit exceeded"); } + @Test + @DisplayName("한 사례가 timeout 나도 다음 사례와 CSV 출력은 계속된다") + void runContinuesAfterCaseDeadlineTimeout() throws Exception { + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); + ObjectMapper objectMapper = new ObjectMapper(); + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + doAnswer(invocation -> { + EvaluationAnalysisCommand command = invocation.getArgument(0); + if ("EV-TIMEOUT".equals(command.caseId())) { + throw new GeneralException(GeneralErrorCode.EXTERNAL_SERVICE_TIMEOUT, "평가 사례 처리 시간이 제한을 초과했습니다."); + } + return result(new AnalysisLlmResponse(90, 80, 70, "정상", List.of(), List.of())); + }).when(generator).generate(any()); + + Path input = tempDir.resolve("evaluation_timeout_cases.csv"); + Path output = tempDir.resolve("evaluation_timeout_results.csv"); + Files.writeString( + input, + "caseId,jobCategoryMiddle,jobCategorySmall,mainTasks,qualifications,preferences,question,answer\n" + + "EV-TIMEOUT,AI·개발·데이터,백엔드,API 개발,Spring,,첫 질문,첫 답변\n" + + "EV-OK,AI·개발·데이터,백엔드,API 개발,Spring,,둘째 질문,둘째 답변\n", + StandardCharsets.UTF_8 + ); + + EvaluationAnalysisBatchService.EvaluationBatchSummary summary = service.run(input, output); + + assertThat(summary.totalCount()).isEqualTo(2); + assertThat(summary.successCount()).isEqualTo(1); + assertThat(summary.failureCount()).isEqualTo(1); + List> rows = EvaluationCsvSupport.read(output); + assertThat(rows).hasSize(2); + assertThat(rows.get(0).get("caseId")).isEqualTo("EV-TIMEOUT"); + assertThat(rows.get(0).get("errorMessage")).contains("제한을 초과"); + assertThat(rows.get(1).get("caseId")).isEqualTo("EV-OK"); + assertThat(rows.get(1).get("errorMessage")).isEmpty(); + assertThat(rows.get(1).get("aiJobFit")).isEqualTo("90"); + assertThat(rows.get(1).get("aiImpact")).isEqualTo("80"); + assertThat(rows.get(1).get("aiCompleteness")).isEqualTo("70"); + } + @Test @DisplayName("평가 CSV 후보/decision 통계는 검증 후 결과 기준으로 기록한다") void runWritesValidatedCandidateDecisionCounts() throws Exception { - AnalysisAiClient analysisAiClient = mock(AnalysisAiClient.class); + EvaluationAnalysisGenerator generator = mock(EvaluationAnalysisGenerator.class); ObjectMapper objectMapper = new ObjectMapper(); - EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService( - analysisAiClient, - new JobCategoryEvaluationCriteriaProvider(objectMapper), - objectMapper - ); - when(analysisAiClient.analyzeForEvaluationResult(any(AnalysisPromptInput.class), any())) - .thenReturn(new AnalysisAiCallResult( + EvaluationAnalysisBatchService service = new EvaluationAnalysisBatchService(generator, objectMapper); + when(generator.generate(any())) + .thenReturn(new EvaluationGeneratedResult( new AnalysisLlmResponse( 80, 70, @@ -482,7 +534,6 @@ void runWritesValidatedCandidateDecisionCounts() throws Exception { 60, "피드백" ), - true, 10, 20 )); @@ -512,7 +563,7 @@ void runWritesValidatedCandidateDecisionCounts() throws Exception { assertThat(row.get("rejectionCodeCounts")).doesNotContain("NONE"); } - private AnalysisAiCallResult result(AnalysisLlmResponse response) { - return new AnalysisAiCallResult(response, null, null, null, false, 0, 1); + private EvaluationGeneratedResult result(AnalysisLlmResponse response) { + return new EvaluationGeneratedResult(response, null, null, null, 0, 1); } }