from skills.plan_mode_arc_gsm8k_improvement import ARCGSM8KImprovementPlan

# ARC/GSM8K改善Planの作成
improvement_plan = ARCGSM8KImprovementPlan()

config = {
    "target_model": "AEGIS-Phi3.5mini-jp-v2.4",
    "baseline_model": "Phi-3.5-mini-instruct",
    "analysis_focus": ["arc_timeout_analysis", "arc_extraction_analysis", "gsm8k_contamination_check"],
    "sample_sizes": {"arc_challenge": 500, "gsm8k": 300},
    "evaluation_seeds": [42, 123, 456, 789, 999],  # 複数seedでの評価
    "timeout_settings": {
        "arc_challenge": 180,
        "gsm8k": 120
    }
}

# 改善分析実行
analysis_results = improvement_plan.execute_improvement_analysis(config)
print(f"ARC timeout rate: {analysis_results['arc_analysis']['timeout_rate']:.1%}")
print(f"ARC extraction failure rate: {analysis_results['arc_analysis']['extraction_failure_rate']:.1%}")
print(f"GSM8K contamination detected: {analysis_results['gsm8k_analysis']['contamination_found']}")

# GSM8Kの詳細健全性チェック
sanity_check = improvement_plan.execute_gsm8k_sanity_check({
    "contamination_check": True,
    "multi_seed_evaluation": True,
    "zero_shot_evaluation": True,
    "scoring_validation": True
})

print("GSM8K Sanity Check Results:")
print(f"- Data contamination: {'Detected' if sanity_check['contamination']['found'] else 'Not found'}")
print(f"- Multi-seed variance: {sanity_check['multi_seed']['variance']:.2f}")
print(f"- 0-shot performance: {sanity_check['zero_shot']['accuracy']:.1%}")
print(f"- Scoring consistency: {sanity_check['scoring']['consistency_score']:.2f}")

# 分析結果に基づく改善策生成
improvement_recommendations = improvement_plan.generate_improvement_recommendations(analysis_results)

print("Recommended Improvements:")
for i, rec in enumerate(improvement_recommendations, 1):
    print(f"{i}. {rec['action']}: {rec['expected_impact']}")

# 改善策の自動実行
if improvement_recommendations:
    improvement_plan.execute_recommended_improvements(improvement_recommendations[:2])  # 優先度の高いものを実行

# タイムアウト率分析
timeout_analysis = improvement_plan.analyze_arc_timeout_rates({
    "sample_size": 1000,
    "timeout_threshold": 180,
    "models": ["AEGIS", "Phi-3.5", "Borea"]
})

print(f"AEGIS ARC timeout rate: {timeout_analysis['AEGIS']['timeout_rate']:.1%}")
print(f"Phi-3.5 ARC timeout rate: {timeout_analysis['Phi-3.5']['timeout_rate']:.1%}")

# 抽出失敗率分析
extraction_analysis = improvement_plan.analyze_arc_extraction_failures({
    "sample_size": 500,
    "extraction_logic": "robust",  # 頑健な抽出を使用
    "failure_patterns": ["empty_response", "invalid_format", "no_choice_mentioned"]
})

# データ汚染検査
contamination_check = improvement_plan.check_gsm8k_data_contamination({
    "training_data_sample": "so8t_training_sample_50k.jsonl",
    "test_questions": "gsm8k_test_1000.jsonl",
    "contamination_threshold": 0.8,  # 80%以上の類似度
    "check_types": ["exact_match", "n_gram_overlap", "semantic_similarity"]
})

# 複数seed評価
multi_seed_results = improvement_plan.evaluate_gsm8k_multi_seed({
    "seeds": [42, 123, 456, 789],
    "shot_counts": [8, 4, 0],  # 8-shot, 4-shot, 0-shot
    "sample_size_per_seed": 200
})

# 結果の安定性分析
stability_analysis = improvement_plan.analyze_gsm8k_stability(multi_seed_results)
print(f"Performance variance across seeds: {stability_analysis['variance']:.2f}")
print(f"8-shot dependency: {'High' if stability_analysis['shot_dependency'] > 0.3 else 'Low'}")

# ARC-Challenge改善策
arc_improvements = {
    "extraction_logic_upgrade": True,
    "timeout_extension": 240,  # 180秒 → 240秒
    "response_format_fine_tuning": True,
    "forced_choice_prompting": True
}

# GSM8K改善策
gsm8k_improvements = {
    "data_deduplication": True,
    "few_shot_diversification": True,
    "scoring_logic_validation": True,
    "zero_shot_capability_enhancement": True
}

# 改善策の適用と再評価
improvement_results = improvement_plan.apply_and_re_evaluate({
    "arc_improvements": arc_improvements,
    "gsm8k_improvements": gsm8k_improvements,
    "re_evaluation_samples": 300,
    "statistical_validation": True
})

print("Improvement Results:")
print(f"ARC score improvement: {improvement_results['arc_improvement']['score_gain']:.1f} points")
print(f"GSM8K stability improvement: {improvement_results['gsm8k_improvement']['variance_reduction']:.2f}")

class ARCTimeoutAnalyzer:
    def analyze_timeout_patterns(self, evaluation_results):
        """タイムアウト発生パターンの詳細分析"""
        timeout_patterns = {
            'by_question_length': self.group_by_question_length(results),
            'by_choice_count': self.group_by_choice_count(results),
            'by_complexity': self.group_by_reasoning_complexity(results),
            'temporal_distribution': self.analyze_temporal_distribution(results)
        }

        # 推奨タイムアウト設定
        recommended_timeout = self.calculate_optimal_timeout(timeout_patterns)

        return {
            'patterns': timeout_patterns,
            'recommended_timeout': recommended_timeout,
            'bottleneck_questions': self.identify_bottleneck_questions(results)
        }

class ARCExtractionAnalyzer:
    def analyze_extraction_failures(self, evaluation_results):
        """回答抽出失敗の詳細分析"""
        failure_analysis = {
            'format_violations': self.categorize_format_violations(results),
            'response_patterns': self.analyze_response_patterns(results),
            'extraction_logic_coverage': self.evaluate_extraction_coverage(results),
            'model_specific_issues': self.identify_model_specific_issues(results)
        }

        # 改善された抽出ロジック生成
        improved_extractor = self.generate_improved_extractor(failure_analysis)

        return {
            'failure_analysis': failure_analysis,
            'improved_extractor': improved_extractor,
            'expected_improvement': self.estimate_improvement_gain(failure_analysis)
        }

class GSM8KContaminationChecker:
    def check_data_contamination(self, training_data, test_questions):
        """学習データとテストデータの重複検査"""
        contamination_analysis = {
            'exact_matches': self.find_exact_matches(training_data, test_questions),
            'near_duplicates': self.find_near_duplicates(training_data, test_questions, threshold=0.8),
            'n_gram_overlaps': self.analyze_n_gram_overlaps(training_data, test_questions),
            'semantic_similarities': self.calculate_semantic_similarities(training_data, test_questions)
        }

        # 汚染リスク評価
        contamination_risk = self.assess_contamination_risk(contamination_analysis)

        return {
            'contamination_analysis': contamination_analysis,
            'contamination_risk': contamination_risk,
            'recommended_actions': self.generate_remediation_actions(contamination_risk)
        }

class GSM8KStabilityAnalyzer:
    def analyze_multi_seed_stability(self, multi_seed_results):
        """複数seedでの評価安定性分析"""
        stability_metrics = {
            'performance_variance': self.calculate_performance_variance(multi_seed_results),
            'shot_dependency': self.assess_shot_count_dependency(multi_seed_results),
            'example_sensitivity': self.evaluate_example_sensitivity(multi_seed_results),
            'scoring_consistency': self.check_scoring_consistency(multi_seed_results)
        }

        # 安定性スコア計算
        stability_score = self.compute_overall_stability_score(stability_metrics)

        return {
            'stability_metrics': stability_metrics,
            'stability_score': stability_score,
            'stability_interpretation': self.interpret_stability_score(stability_score),
            'improvement_recommendations': self.generate_stability_improvements(stability_metrics)
        }

arc_improvement_analysis = {
    'timeout_analysis': {
        'timeout_rate': 0.023,  # 2.3% of questions timed out
        'bottleneck_questions': ['complex_reasoning_q1', 'multi_step_q45'],
        'recommended_timeout': 240,  # Increase to 4 minutes
        'timeout_patterns': {
            'by_complexity': {'high': 0.15, 'medium': 0.08, 'low': 0.02}
        }
    },
    'extraction_analysis': {
        'failure_rate': 0.453,  # 45.3% extraction failures
        'failure_patterns': {
            'empty_response': 0.12,
            'invalid_format': 0.23,
            'no_choice_mentioned': 0.098
        },
        'improved_extraction_expected_gain': 0.25  # 25 point improvement expected
    },
    'response_pattern_analysis': {
        'explicit_answer_format': 0.15,
        'choice_mentioned': 0.42,
        'no_choice': 0.43,
        'recommended_prompt_adjustments': [
            'Add explicit choice instruction',
            'Include format examples',
            'Consider forced-choice prompting'
        ]
    }
}

gsm8k_sanity_analysis = {
    'contamination_check': {
        'found': False,
        'exact_matches': 0,
        'near_duplicates': 0,
        'max_similarity': 0.45,
        'contamination_risk': 'low'
    },
    'multi_seed_stability': {
        'performance_variance': 0.023,
        'shot_dependency': 0.67,  # High dependency on 8-shot examples
        'stability_score': 0.78,
        'stability_interpretation': 'moderately_stable'
    },
    'zero_shot_performance': {
        'accuracy': 0.234,
        'vs_8shot_drop': 0.746,  # Significant drop without examples
        'reasoning_capability': 'limited_without_examples'
    },
    'scoring_validation': {
        'consistency_score': 0.92,
        'edge_cases_handled': 0.89,
        'recommended_improvements': ['Handle special number formats', 'Validate extraction accuracy']
    }
}

# ARC-Challengeのタイムアウト・抽出失敗分析
python scripts/evaluation/arc_gsm8k_improvement_analyzer.py \
  --analysis_type arc_improvement \
  --model_path AEGIS-Phi3.5mini-jp \
  --sample_size 500 \
  --timeout_threshold 180 \
  --output_path analysis_results/arc_improvement_analysis.json

# GSM8Kのデータ汚染・複数seed評価
python scripts/evaluation/arc_gsm8k_improvement_analyzer.py \
  --analysis_type gsm8k_sanity \
  --training_data_path so8t_training_data.jsonl \
  --test_questions_path gsm8k_test.jsonl \
  --seeds 42,123,456,789 \
  --output_path analysis_results/gsm8k_sanity_analysis.json

# ARC+GSM8K統合改善分析
python scripts/evaluation/arc_gsm8k_improvement_analyzer.py \
  --analysis_type comprehensive \
  --model_path AEGIS-Phi3.5mini-jp \
  --training_data_path so8t_training_data.jsonl \
  --sample_sizes "arc:500,gsm8k:300" \
  --seeds 42,123,456 \
  --output_path analysis_results/comprehensive_improvement_analysis.json