from skills.plan_mode_official_leaderboard_abctest import OfficialABCTestPlan

# 3モデル公式準拠比較テスト
abc_test = OfficialABCTestPlan()

test_config = {
    "models": {
        "Phi-3.5-mini-instruct": "microsoft/Phi-3.5-mini-instruct",
        "Borea-phi3.5-instinct-jp": "path/to/borea/model",
        "AEGIS-Phi3.5mini-jp-v2.4": "your-username/AEGIS-Phi3.5mini-jp"
    },
    "benchmarks": ["gsm8k", "math", "arc_challenge"],
    "sample_sizes": {"gsm8k": 1000, "math": 500, "arc_challenge": 1000},
    "runs_per_model": 3,  # 統計的安定性のための複数回実行
    "statistical_tests": ["t_test", "cohen_d", "confidence_intervals"],
    "significance_level": 0.05
}

# 公式準拠A/B/Cテスト実行
results = abc_test.execute_official_abctest(test_config)
print(f"A/B/C Test completed: {results['summary']['winner']}")

# 詳細な統計分析を含む評価
statistical_analysis = abc_test.perform_statistical_analysis(results)

print("=== STATISTICAL SIGNIFICANCE RESULTS ===")
for benchmark in results['benchmarks']:
    print(f"{benchmark.upper()}:")
    for comparison in results['comparisons'][benchmark]:
        print(f"  {comparison['model_a']} vs {comparison['model_b']}:")
        print(f"    t-statistic: {comparison['t_statistic']:.3f}")
        print(f"    p-value: {comparison['p_value']:.4f}")
        print(f"    Cohen's d: {comparison['cohen_d']:.3f}")
        print(f"    Significant: {'Yes' if comparison['significant'] else 'No'}")

# 比較チャートとレポート生成
visualization = abc_test.generate_comparison_visualization(results)
report = abc_test.generate_official_report(results)

# 保存
abc_test.save_results(results, "official_abctest_results.json")
abc_test.save_visualizations(visualization, "charts/")
abc_test.save_report(report, "reports/official_abctest_report.pdf")

# モデル読み込みと基本検証
models_ready = abc_test.verify_models_availability(test_config['models'])
benchmarks_ready = abc_test.verify_benchmarks_availability(test_config['benchmarks'])

if not (models_ready and benchmarks_ready):
    raise ValueError("Models or benchmarks not available")

# リソース割り当て最適化
resource_plan = abc_test.optimize_resource_allocation(test_config)
print(f"GPU allocation: {resource_plan['gpu_allocation']}")
print(f"Estimated time: {resource_plan['estimated_hours']} hours")

# 複数GPUでの並行評価
evaluation_jobs = abc_test.create_evaluation_jobs(test_config)
results = abc_test.execute_parallel_evaluations(evaluation_jobs)

# 進捗監視
with tqdm(total=len(evaluation_jobs), desc="A/B/C Test Progress") as pbar:
    for completed_job in abc_test.monitor_evaluation_progress():
        pbar.update(1)
        print(f"Completed: {completed_job['model']} on {completed_job['benchmark']}")

# 統計的有意性検定
statistical_tests = abc_test.perform_statistical_tests(results, test_config['significance_level'])

# 効果サイズ計算
effect_sizes = abc_test.calculate_effect_sizes(results)

# 信頼区間推定
confidence_intervals = abc_test.calculate_confidence_intervals(results, confidence_level=0.95)

# 多重比較補正
corrected_p_values = abc_test.apply_multiple_comparison_correction(statistical_tests)

# 結果集約
final_results = abc_test.aggregate_results(results, statistical_tests, effect_sizes)

# 勝者決定（統計的有意性ベース）
winner_analysis = abc_test.determine_winner(final_results, test_config)

# 詳細レポート生成
comprehensive_report = abc_test.generate_comprehensive_report(final_results, winner_analysis)

class ABCTestDesign:
    def __init__(self, models, benchmarks, significance_level=0.05):
        self.models = models  # 3つのモデル
        self.benchmarks = benchmarks
        self.alpha = significance_level
        self.adjusted_alpha = self.alpha / 3  # Bonferroni補正

    def calculate_required_sample_size(self, effect_size, power=0.8):
        """必要なサンプルサイズ計算"""
        # Cohen's dに基づくpower analysis
        return self.power_analysis_sample_size(effect_size, power)

    def create_comparison_matrix(self):
        """3モデル間の全比較ペア生成"""
        return [
            (model_a, model_b)
            for i, model_a in enumerate(self.models[:-1])
            for model_b in self.models[i+1:]
        ]

class StatisticalTester:
    def perform_pairwise_t_tests(self, results_a, results_b):
        """対応のないt-test実行"""
        from scipy.stats import ttest_ind

        t_stat, p_value = ttest_ind(results_a, results_b, equal_var=False)

        # Cohen's d効果サイズ
        mean_diff = np.mean(results_a) - np.mean(results_b)
        pooled_std = np.sqrt((np.std(results_a)**2 + np.std(results_b)**2) / 2)
        cohen_d = mean_diff / pooled_std if pooled_std > 0 else 0

        return {
            't_statistic': t_stat,
            'p_value': p_value,
            'cohen_d': cohen_d,
            'significant': p_value < self.adjusted_alpha,
            'effect_size_interpretation': self.interpret_cohen_d(cohen_d)
        }

    def interpret_cohen_d(self, d):
        """Cohen's dの解釈"""
        abs_d = abs(d)
        if abs_d < 0.2:
            return "negligible"
        elif abs_d < 0.5:
            return "small"
        elif abs_d < 0.8:
            return "medium"
        else:
            return "large"

class ConfidenceIntervalCalculator:
    def calculate_bootstrap_ci(self, data, n_bootstrap=1000, ci_level=0.95):
        """ブートストラップ法による信頼区間"""
        bootstrap_means = []
        n = len(data)

        for _ in range(n_bootstrap):
            sample = np.random.choice(data, size=n, replace=True)
            bootstrap_means.append(np.mean(sample))

        lower_percentile = (1 - ci_level) / 2 * 100
        upper_percentile = (1 + ci_level) / 2 * 100

        return {
            'mean': np.mean(data),
            'ci_lower': np.percentile(bootstrap_means, lower_percentile),
            'ci_upper': np.percentile(bootstrap_means, upper_percentile),
            'ci_level': ci_level
        }

OfficialABCTestResults = {
    'metadata': {
        'timestamp': '2026-01-17 03:00:00',
        'models_tested': ['Phi-3.5-mini-instruct', 'Borea-phi3.5-instinct-jp', 'AEGIS-Phi3.5mini-jp-v2.4'],
        'benchmarks': ['gsm8k', 'math', 'arc_challenge'],
        'sample_sizes': {'gsm8k': 1000, 'math': 500, 'arc_challenge': 1000},
        'runs_per_model': 3,
        'evaluation_protocols': {
            'gsm8k': '8-shot CoT',
            'math': '0-shot CoT',
            'arc_challenge': '10-shot'
        }
    },
    'raw_results': {
        'model_name': {
            'benchmark_name': {
                'run_1': {'accuracy': 0.xxx, 'individual_scores': [...]},
                'run_2': {...},
                'run_3': {...},
                'aggregate': {'mean': 0.xxx, 'std': 0.xxx, '95ci': [0.xxx, 0.xxx]}
            }
        }
    },
    'statistical_analysis': {
        'pairwise_comparisons': [
            {
                'model_a': 'AEGIS-Phi3.5mini-jp-v2.4',
                'model_b': 'Phi-3.5-mini-instruct',
                'benchmark': 'gsm8k',
                't_statistic': 2.345,
                'p_value': 0.023,
                'cohen_d': 0.678,
                'significant': True,
                'effect_size': 'medium',
                'confidence_intervals': {'model_a': [0.85, 0.87], 'model_b': [0.83, 0.85]}
            }
        ],
        'overall_rankings': {
            'gsm8k': ['AEGIS-Phi3.5mini-jp-v2.4', 'Borea-phi3.5-instinct-jp', 'Phi-3.5-mini-instruct'],
            'math': ['AEGIS-Phi3.5mini-jp-v2.4', 'Phi-3.5-mini-instruct', 'Borea-phi3.5-instinct-jp'],
            'arc_challenge': ['Phi-3.5-mini-instruct', 'AEGIS-Phi3.5mini-jp-v2.4', 'Borea-phi3.5-instinct-jp']
        }
    },
    'summary': {
        'winner_by_benchmark': {
            'gsm8k': 'AEGIS-Phi3.5mini-jp-v2.4',
            'math': 'AEGIS-Phi3.5mini-jp-v2.4',
            'arc_challenge': 'Phi-3.5-mini-instruct'
        },
        'overall_winner': 'AEGIS-Phi3.5mini-jp-v2.4',
        'confidence_level': 'high',
        'key_findings': [
            'AEGIS shows significant improvement in GSM8K and MATH',
            'Phi-3.5-mini-instruct performs best on ARC-Challenge',
            'All differences are statistically significant (p < 0.05)'
        ]
    }
}

# 公式準拠A/B/Cテスト実行
python scripts/plan_mode_official_abctest.py \
  --models-config scripts/evaluation/models_config.json \
  --benchmarks gsm8k math arc_challenge \
  --sample-sizes "gsm8k:1000,math:500,arc_challenge:1000" \
  --runs-per-model 3 \
  --significance-level 0.05 \
  --output-dir evaluation_results/official_abctest/

# 統計分析とレポート生成
python scripts/analyze_abctest_results.py \
  --results-dir evaluation_results/official_abctest/ \
  --generate-plots \
  --create-pdf-report \
  --significance-analysis

# 結果比較チャート生成
python scripts/visualize_abctest_comparison.py \
  --results-file evaluation_results/official_abctest/final_results.json \
  --output-dir charts/abctest_comparison/ \
  --chart-types "bar,ranking,effect_size,confidence_intervals"