Skip to content

Refactor/#270 evaluation runtime decouple #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shinae1023 merged 3 commits into from
Aug 12, 2026
Merged

Refactor/#270 evaluation runtime decouple #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af05019
[Refactor] evaluation 전용 sanitization facade 도입 (#270)
shinae1023 Aug 12, 2026
beb77c3
[Refactor] evaluation 분석 AI port 도입 (#270)
shinae1023 Aug 12, 2026
3fc81ac
[Fix] evaluation 사례 deadline과 command 검증 보강 (#270)
shinae1023 Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion ...n/java/com/jobdri/jobdri_api/domain/analysis/infrastructure/ai/OpenAiAnalysisAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@
import com.jobdri.jobdri_api.global.config.LlmConcurrencyLimiter;
import com.jobdri.jobdri_api.global.metrics.AsyncMetricsRecorder;
import com.openai.client.OpenAIClient;
import com.openai.core.RequestOptions;
import com.openai.models.responses.ResponseCreateParams;
import com.openai.models.responses.StructuredResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.time.Duration;

@Component
@RequiredArgsConstructor
public class OpenAiAnalysisAdapter {
Expand All @@ -22,18 +25,32 @@ public class OpenAiAnalysisAdapter {
private String analysisModel;

public <T> T createStructuredResponse(String operationName, String prompt, Class<T> responseType) {
return createStructuredResponse(operationName, prompt, responseType, null);
}

public <T> T createStructuredResponse(
String operationName,
String prompt,
Class<T> responseType,
Duration timeout
) {
var params = ResponseCreateParams.builder()
.model(analysisModel)
.input(prompt)
.temperature(0.2)
.text(responseType)
.build();
RequestOptions requestOptions = timeout == null
? null
: RequestOptions.builder().timeout(timeout).build();
long startedAt = System.nanoTime();
boolean success = false;
try {
StructuredResponse<T> response = llmConcurrencyLimiter.execute(
operationName,
() -> openAIClient.responses().create(params)
() -> requestOptions == null
? openAIClient.responses().create(params)
: openAIClient.responses().create(params, requestOptions)
);
T structuredContent = analysisResponseParser.extractStructuredContent(response);
success = true;
Expand Down
88 changes: 69 additions & 19 deletions src/main/java/com/jobdri/jobdri_api/domain/analysis/service/ai/AnalysisAiClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
Expand Down Expand Up @@ -100,19 +102,22 @@ public AnalysisLlmResponse analyze(
promptInput,
referenceContext,
jobCategoryEvaluationCriteria,
"cover-letter-analysis"
"cover-letter-analysis",
null
).response();
case HYBRID_EXACT -> analyzeHybridExact(
promptInput,
referenceContext,
jobCategoryEvaluationCriteria,
"cover-letter-analysis"
"cover-letter-analysis",
null
).response();
case SINGLE_PASS -> analyzeSinglePass(
promptInput,
referenceContext,
jobCategoryEvaluationCriteria,
"cover-letter-analysis"
"cover-letter-analysis",
null
).response();
};
} catch (GeneralException e) {
Expand Down Expand Up @@ -155,26 +160,37 @@ public AnalysisLlmResponse analyzeForEvaluation(
public AnalysisAiCallResult analyzeForEvaluationResult(
AnalysisPromptInput promptInput,
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria
) {
return analyzeForEvaluationResult(promptInput, jobCategoryEvaluationCriteria, null);
}

public AnalysisAiCallResult analyzeForEvaluationResult(
AnalysisPromptInput promptInput,
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria,
Instant deadline
) {
try {
return switch (resolveAnalysisMode()) {
case TWO_PASS -> analyzeTwoPass(
promptInput,
emptyContext(),
jobCategoryEvaluationCriteria,
"cover-letter-analysis-evaluation"
"cover-letter-analysis-evaluation",
deadline
);
case HYBRID_EXACT -> analyzeHybridExact(
promptInput,
emptyContext(),
jobCategoryEvaluationCriteria,
"cover-letter-analysis-evaluation"
"cover-letter-analysis-evaluation",
deadline
);
case SINGLE_PASS -> analyzeSinglePass(
promptInput,
emptyContext(),
jobCategoryEvaluationCriteria,
"cover-letter-analysis-evaluation"
"cover-letter-analysis-evaluation",
deadline
);
};
} catch (GeneralException e) {
Expand All @@ -192,13 +208,15 @@ private AnalysisAiCallResult analyzeSinglePass(
AnalysisPromptInput promptInput,
RetrievalContext referenceContext,
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria,
String operationName
String operationName,
Instant deadline
) {
long startedAt = System.nanoTime();
AnalysisLlmResponse response = createStructuredResponse(
operationName,
buildPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria),
AnalysisLlmResponse.class
AnalysisLlmResponse.class,
deadline
);
response = analysisResponseParser.sanitizeSinglePassSubheadings(promptInput, response);
return AnalysisAiCallResult.singlePass(response, elapsedMillis(startedAt));
Expand All @@ -208,13 +226,15 @@ private AnalysisAiCallResult analyzeTwoPass(
AnalysisPromptInput promptInput,
RetrievalContext referenceContext,
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria,
String operationName
String operationName,
Instant deadline
) {
long candidateStartedAt = System.nanoTime();
AnalysisCandidateResponse rawCandidates = createStructuredResponse(
operationName + "-candidates",
buildCandidatePrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria),
AnalysisCandidateResponse.class
AnalysisCandidateResponse.class,
deadline
);
long candidateLatencyMs = elapsedMillis(candidateStartedAt);
AnalysisCandidateResponse sanitizedCandidates = sanitizeCandidates(promptInput, rawCandidates);
Expand All @@ -235,7 +255,8 @@ private AnalysisAiCallResult analyzeTwoPass(
CandidateReviewResponse reviewResponse = createStructuredResponse(
operationName + "-final",
buildFinalPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria, sanitizedCandidates),
CandidateReviewResponse.class
CandidateReviewResponse.class,
deadline
);
CandidateReviewResponse validatedReviewResponse = validateCandidateReview(
promptInput,
Expand All @@ -257,7 +278,8 @@ private AnalysisAiCallResult analyzeTwoPass(
jobCategoryEvaluationCriteria,
sanitizedCandidates,
validatedReviewResponse,
operationName
operationName,
deadline
);
AnalysisLlmResponse response = buildFinalResponse(promptInput, sanitizedCandidates, recheckedReviewResponse);
long finalLatencyMs = elapsedMillis(finalStartedAt);
Expand Down Expand Up @@ -294,19 +316,22 @@ private AnalysisAiCallResult analyzeHybridExact(
AnalysisPromptInput promptInput,
RetrievalContext referenceContext,
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria,
String operationName
String operationName,
Instant deadline
) {
AnalysisAiCallResult singlePassResult = analyzeSinglePass(
promptInput,
referenceContext,
jobCategoryEvaluationCriteria,
operationName + "-single-pass"
operationName + "-single-pass",
deadline
);
AnalysisAiCallResult twoPassResult = analyzeTwoPass(
promptInput,
referenceContext,
jobCategoryEvaluationCriteria,
operationName + "-two-pass"
operationName + "-two-pass",
deadline
);
AnalysisLlmResponse merged = mergeHybridExact(
singlePassResult.response(),
Expand Down Expand Up @@ -335,6 +360,29 @@ private <T> T createStructuredResponse(String operationName, String prompt, Clas
return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType);
}

private <T> T createStructuredResponse(
String operationName,
String prompt,
Class<T> responseType,
Instant deadline
) {
if (deadline == null) {
return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType);
}
Duration remaining = Duration.between(Instant.now(), deadline);
if (remaining.isZero() || remaining.isNegative()) {
throw new GeneralException(
GeneralErrorCode.EXTERNAL_SERVICE_TIMEOUT,
"평가 사례 처리 시간이 제한을 초과했습니다."
);
}
return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType, remaining);
}

private <T> T createStructuredResponse(String operationName, String prompt, Class<T> responseType, Duration timeout) {
return openAiAnalysisAdapter.createStructuredResponse(operationName, prompt, responseType);
}

String buildPrompt(
JobPosting jobPosting,
List<Question> questions,
Expand Down Expand Up @@ -553,7 +601,8 @@ CandidateReviewResponse recheckWhenAllCandidatesRejected(
JobCategoryEvaluationCriteria jobCategoryEvaluationCriteria,
AnalysisCandidateResponse sanitizedCandidates,
CandidateReviewResponse reviewResponse,
String operationName
String operationName,
Instant deadline
) {
int firstPassCandidates = sanitizedCandidates == null || sanitizedCandidates.analysisCandidates() == null
? 0
Expand All @@ -573,7 +622,8 @@ CandidateReviewResponse recheckWhenAllCandidatesRejected(
CandidateRecheckResponse recheckResponse = createStructuredResponse(
operationName + "-recheck",
buildRecheckPrompt(promptInput, referenceContext, jobCategoryEvaluationCriteria, sanitizedCandidates, reviewResponse),
CandidateRecheckResponse.class
CandidateRecheckResponse.class,
deadline
);
CandidateReviewResponse rechecked = applyRecheckResponse(promptInput, sanitizedCandidates, reviewResponse, recheckResponse);
int recoveredMentionedCount = recoveredDecisionCount(rechecked, QuestionAnalysisStatus.MENTIONED);
Expand Down Expand Up @@ -1635,7 +1685,7 @@ private RetrievalContext emptyContext() {
return new RetrievalContext(List.of(), List.of());
}

AnalysisMode resolveAnalysisMode() {
public AnalysisMode resolveAnalysisMode() {
if (StringUtils.hasText(analysisMode)) {
String normalized = analysisMode.trim().replace('-', '_').toUpperCase(java.util.Locale.ROOT);
try {
Expand All @@ -1647,7 +1697,7 @@ AnalysisMode resolveAnalysisMode() {
return twoPassEnabled ? AnalysisMode.TWO_PASS : AnalysisMode.SINGLE_PASS;
}

enum AnalysisMode {
public enum AnalysisMode {
SINGLE_PASS,
TWO_PASS,
HYBRID_EXACT
Expand Down
Loading
Loading