OpenCode
68a9071cf1
Local branch had diverged from origin/master (sibling commits on the same base). Rewrote local history linearly on top of origin/master, folding in all local content: exa/hot-platform discovery hunters, leaks ledger, nightly sweep orchestrator, updated .gitignore and skill docs, plus the latest hunt output and vault state. Remote-only files (vault keys, channel scripts) were restored rather than dropped, so the resulting tree is a full union of both sides.
1 line
18 KiB
JSON
1 line
18 KiB
JSON
{"repo": "cartert27/faster", "files": {"tests/test_auto_mpg_integration.py": {"36b2f184a4936da048398c98e08ed77c5a92915f": {"t": 1786068623.4543006, "n": 17013, "h": "74754c277b2470c4", "c": "\"\"\"Integration test using the Auto MPG dataset to compare FASTER with baseline models for regression.\"\"\"\n\nimport pytest\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import cross_validate\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestRegressor\nimport warnings\nfrom typing import Dict, List, Tuple\nfrom unittest.mock import patch\nimport os\nimport openai\nimport logging\nfrom sklearn.datasets import fetch_openml\nfrom sklearn.impute import SimpleImputer\n\nfrom faster.pipeline import Pipeline, PipelineConfig\nfrom faster.feature_selection import SelectionCriteria\nfrom faster.domain_knowledge import DomainKnowledgeExtractor\nfrom tests.conftest import colored_metric_output\n\nlogging.basicConfig(\n level=logging.INFO, format=\"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n)\nlogger = logging.getLogger(__name__)\n\nMOCK_LLM_RESPONSE = [\n {\n \"feature_name\": \"horsepower\",\n \"importance\": 0.9,\n \"relationships\": [\"weight\", \"cylinders\", \"displacement\"],\n \"suggested_transformations\": [\"log\", \"zscore\", \"polynomial\", \"square_root\"],\n \"rationale\": \"Horsepower has an inverse relationship with MPG. More powerful engines consume more fuel.\",\n },\n {\n \"feature_name\": \"weight\",\n \"importance\": 0.85,\n \"relationships\": [\"horsepower\", \"displacement\"],\n \"suggested_transformations\": [\"log\", \"zscore\", \"polynomial\", \"reciprocal\"],\n \"rationale\": \"Heavier cars require more energy to move, directly impacting fuel efficiency.\",\n },\n {\n \"feature_name\": \"displacement\",\n \"importance\": 0.8,\n \"relationships\": [\"cylinders\", \"horsepower\"],\n \"suggested_transformations\": [\"log\", \"zscore\", \"polynomial\", \"square_root\"],\n \"rationale\": \"Engine displacement affects fuel consumption; larger engines typically consume more fuel.\",\n },\n {\n \"feature_name\": \"cylinders\",\n \"importance\": 0.75,\n \"relationships\": [\"displacement\", \"horsepower\"],\n \"suggested_transformations\": [\"one_hot\", \"zscore\", \"polynomial\"],\n \"rationale\": \"Number of cylinders correlates with engine size and fuel consumption.\",\n },\n {\n \"feature_name\": \"model_year\",\n \"importance\": 0.7,\n \"relationships\": [\"weight\", \"horsepower\"],\n \"suggested_transformations\": [\"zscore\", \"binning\", \"time_since\"],\n \"rationale\": \"Newer models tend to have improved fuel efficiency due to technological advancements.\",\n },\n {\n \"feature_name\": \"origin\",\n \"importance\": 0.6,\n \"relationships\": [\"model_year\", \"weight\"],\n \"suggested_transformations\": [\"one_hot\", \"label\"],\n \"rationale\": \"Cars from different regions had different design philosophies affecting fuel efficiency.\",\n },\n {\n \"feature_name\": \"acceleration\",\n \"importance\": 0.5,\n \"relationships\": [\"horsepower\", \"weight\"],\n \"suggested_transformations\": [\"zscore\", \"polynomial\", \"reciprocal\"],\n \"rationale\": \"Acceleration capability relates to power-to-weight ratio, indirectly impacting fuel efficiency.\",\n },\n]\n\n\ndef load_and_preprocess_auto_mpg() -> Tuple[pd.DataFrame, pd.Series]:\n \"\"\"Load and preprocess the Auto MPG dataset for regression (predicting MPG).\"\"\"\n try:\n auto_mpg = fetch_openml(name=\"auto-mpg\", version=1, as_frame=True)\n\n df = auto_mpg.data\n df[\"mpg\"] = auto_mpg.target\n\n except Exception as e:\n logger.warning(f\"Error fetching from OpenML: {str(e)}. Falling back to local loading...\")\n\n url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\"\n column_names = [\n \"mpg\",\n \"cylinders\",\n \"displacement\",\n \"horsepower\",\n \"weight\",\n \"acceleration\",\n \"model_year\",\n \"origin\",\n \"car_name\",\n ]\n df = pd.read_csv(url, delim_whitespace=True, header=None, names=column_names, na_values=\"?\")\n\n if \"car_name\" in df.columns:\n df = df.drop(\"car_name\", axis=1)\n\n numeric_columns = [\"horsepower\"] # Only horsepower has missing values typically\n imputer = SimpleImputer(strategy=\"median\")\n df[numeric_columns] = imputer.fit_transform(df[numeric_columns])\n\n origin_mapping = {1: \"american\", 2: \"european\", 3: \"asian\"}\n df[\"origin\"] = df[\"origin\"].map(origin_mapping)\n\n y = df[\"mpg\"]\n X = df.drop(\"mpg\", axis=1)\n\n X = pd.get_dummies(X, columns=[\"origin\"], drop_first=False)\n\n numeric_features = [\n \"cylinders\",\n \"displacement\",\n \"horsepower\",\n \"weight\",\n \"acceleration\",\n \"model_year\",\n ]\n scaler = StandardScaler()\n X[numeric_features] = scaler.fit_transform(X[numeric_features])\n\n return X, y\n\n\ndef evaluate_model(X: pd.DataFrame, y: pd.Series) -> Dict[str, float]:\n \"\"\"Evaluate regression model using cross-validation.\"\"\"\n regressor = RandomForestRegressor(n_estimators=100, random_state=42)\n\n scoring = {\"r2\": \"r2\", \"mae\": \"neg_mean_absolute_error\", \"rmse\": \"neg_root_mean_squared_error\"}\n\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n cv_results = cross_validate(regressor, X, y, cv=5, scoring=scoring, return_train_score=True)\n\n return {\n \"test_r2\": cv_results[\"test_r2\"].mean(),\n \"test_mae\": -cv_results[\"test_mae\"].mean(),\n \"test_rmse\": -cv_results[\"test_rmse\"].mean(),\n \"train_r2\": cv_results[\"train_r2\"].mean(),\n \"train_mae\": -cv_results[\"train_mae\"].mean(),\n \"train_rmse\": -cv_results[\"train_rmse\"].mean(),\n }\n\n\ndef verify_openrouter_api_key() -> str:\n \"\"\"\n Verify and retrieve the OpenRouter API key.\n\n Returns:\n str: Validated API key or raises a pytest.skip exception\n \"\"\"\n\n api_key = os.environ.get(\"OPENROUTER_API_KEY\", \"\").strip()\n\n if not api_key:\n pytest.skip(\"Skipping non-mock test: OPENROUTER_API_KEY environment variable not set\")\n\n if not api_key.startswith(\"sk-\"):\n logger.warning(\"Warning: OpenRouter API key does not start with 'sk-'. Key may be invalid.\")\n\n placeholder_terms = [\"dummy\", \"test\", \"placeholder\", \"your_key\", \"example\"]\n if any(term in api_key.lower() for term in placeholder_terms):\n pytest.skip(f\"Skipping non-mock test: OPENROUTER_API_KEY appears to be a placeholder value\")\n\n masked_key = api_key[:4] + \"...\" + api_key[-4:] if len(api_key) > 8 else \"***\"\n logger.info(f\"Using OpenRouter API key: {masked_key} (length: {len(api_key)})\")\n\n return api_key\n\n\ndef configure_openai_client(api_key: str) -> openai.OpenAI:\n \"\"\"\n Configure and return an OpenAI client configured for OpenRouter.\n\n Args:\n api_key: The OpenRouter API key\n\n Returns:\n openai.OpenAI: Configured client\n \"\"\"\n\n logger.info(f\"Configuring OpenAI client with base URL: https://openrouter.ai/api/v1\")\n\n return openai.OpenAI(\n base_url=\"https://openrouter.ai/api/v1\",\n api_key=api_key,\n )\n\n\n@pytest.mark.parametrize(\"use_mock\", [True, False])\ndef test_auto_mpg_integration(use_mock):\n \"\"\"Integration test comparing baseline and FASTER-enhanced models on Auto MPG dataset for regression.\"\"\"\n\n if not use_mock:\n try:\n api_key = verify_openrouter_api_key()\n\n client = configure_openai_client(api_key)\n\n try:\n response = client.chat.completions.create(\n model=\"deepseek/deepseek-chat:free\",\n messages=[{\"role\": \"user\", \"content\": \"Hello, this is a test\"}],\n temperature=0.0,\n max_tokens=10,\n )\n logger.info(\"OpenRouter API test successful\")\n except Exception as e:\n logger.error(f\"OpenRouter API test failed: {str(e)}\")\n pytest.skip(f\"Skipping non-mock test: OpenRouter API request failed: {str(e)}\")\n\n except Exception as e:\n logger.error(f\"Error setting up OpenRouter credentials: {str(e)}\")\n pytest.skip(f\"Skipping non-mock test: {str(e)}\")\n\n X, y = load_and_preprocess_auto_mpg()\n data = X.copy()\n data[\"mpg\"] = y\n\n baseline_scores = evaluate_model(X, y)\n\n config = PipelineConfig(\n model_name=\"deepseek/deepseek-chat:free\",\n temperature=0.0,\n max_interaction_degree=2,\n text_feature_method=\"tfidf\",\n alpha=0.05,\n correction_method=\"fdr_bh\",\n selection_criteria=SelectionCriteria(),\n output_dir=None,\n save_intermediate=False,\n categorical_encoding=\"dummy\",\n keep_all_features=True, # Keep all features without dropping any\n )\n\n if use_mock:\n with patch(\n \"faster.domain_knowledge.DomainKnowledgeExtractor._query_llm_with_retry\"\n ) as mock_query:\n mock_query.return_value = MOCK_LLM_RESPONSE\n\n pipeline_no_domain = Pipeline(config)\n result_no_domain = pipeline_no_domain.run(\n data=data,\n target_column=\"mpg\",\n problem_description=\"Regression problem predicting miles per gallon (MPG) of automobiles\",\n is_classification=False,\n keep_all_features=True, # Keep all features without dropping any\n )\n\n pipeline_with_domain = Pipeline(config)\n result_with_domain = pipeline_with_domain.run(\n data=data,\n target_column=\"mpg\",\n problem_description=\"\"\"\n Regression problem predicting miles per gallon (MPG) of automobiles.\n Features:\n - cylinders: Number of cylinders in the engine (integer)\n - displacement: Engine displacement in cubic inches (continuous)\n - horsepower: Engine horsepower (continuous)\n - weight: Vehicle weight in pounds (continuous)\n - acceleration: Time to accelerate from 0 to 60 mph in seconds (continuous)\n - model_year: Vehicle model year (integer, 70-82 representing 1970-1982)\n - origin: Origin of car (categorical: american, european, asian)\n \n Domain context:\n - Fuel efficiency (MPG) generally decreases with vehicle weight\n - Higher horsepower engines typically consume more fuel\n - Larger engine displacement generally correlates with lower MPG\n - Cars with more cylinders typically have lower fuel efficiency\n - Technological improvements over model years generally increased fuel efficiency\n - There are tradeoffs between performance (acceleration) and fuel efficiency\n - Different regions (origins) had different emission standards and design philosophies\n - The dataset contains cars from the 1970s and early 1980s during fuel crises\n \"\"\",\n is_classification=False,\n keep_all_features=True, # Keep all features without dropping any\n )\n else:\n domain_extractor = DomainKnowledgeExtractor(\n model_name=config.model_name, temperature=config.temperature, api_key=api_key\n )\n\n pipeline_no_domain = Pipeline(config)\n pipeline_no_domain.domain_extractor = domain_extractor\n result_no_domain = pipeline_no_domain.run(\n data=data,\n target_column=\"mpg\",\n problem_description=\"Regression problem predicting miles per gallon (MPG) of automobiles\",\n is_classification=False,\n keep_all_features=True, # Keep all features without dropping any\n )\n\n pipeline_with_domain = Pipeline(config)\n pipeline_with_domain.domain_extractor = domain_extractor\n result_with_domain = pipeline_with_domain.run(\n data=data,\n target_column=\"mpg\",\n problem_description=\"\"\"\n Regression problem predicting miles per gallon (MPG) of automobiles.\n Features:\n - cylinders: Number of cylinders in the engine (integer)\n - displacement: Engine displacement in cubic inches (continuous)\n - horsepower: Engine horsepower (continuous)\n - weight: Vehicle weight in pounds (continuous)\n - acceleration: Time to accelerate from 0 to 60 mph in seconds (continuous)\n - model_year: Vehicle model year (integer, 70-82 representing 1970-1982)\n - origin: Origin of car (categorical: american, european, asian)\n \n Domain context:\n - Fuel efficiency (MPG) generally decreases with vehicle weight\n - Higher horsepower engines typically consume more fuel\n - Larger engine displacement generally correlates with lower MPG\n - Cars with more cylinders typically have lower fuel efficiency\n - Technological improvements over model years generally increased fuel efficiency\n - There are tradeoffs between performance (acceleration) and fuel efficiency\n - Different regions (origins) had different emission standards and design philosophies\n - The dataset contains cars from the 1970s and early 1980s during fuel crises\n \"\"\",\n is_classification=False,\n keep_all_features=True, # Keep all features without dropping any\n )\n\n print(\"\\nAvailable attributes in result_no_domain:\", dir(result_no_domain))\n\n no_domain_features = result_no_domain.transformed_data\n with_domain_features = result_with_domain.transformed_data\n\n if \"mpg\" in no_domain_features.columns:\n no_domain_features = no_domain_features.drop(\"mpg\", axis=1)\n if \"mpg\" in with_domain_features.columns:\n with_domain_features = with_domain_features.drop(\"mpg\", axis=1)\n\n print(\"\\nNo Domain Features Shape:\", no_domain_features.shape)\n print(\"With Domain Features Shape:\", with_domain_features.shape)\n print(\"\\nBaseline X Columns:\")\n print(sorted(X.columns.tolist()))\n print(\"\\nFASTER (No Domain Knowledge) X Columns:\")\n print(sorted(no_domain_features.columns.tolist()))\n print(\"\\nFASTER (With Domain Knowledge) X Columns:\")\n print(sorted(with_domain_features.columns.tolist()))\n\n faster_no_domain_scores = evaluate_model(no_domain_features, y)\n\n faster_with_domain_scores = evaluate_model(with_domain_features, y)\n\n print(f\"\\nModel Performance Comparison ({'Mock' if use_mock else 'Real'} LLM):\")\n\n print(\"\\nBaseline Model:\")\n print(\" Train Metrics:\")\n for metric, score in [(k, v) for k, v in baseline_scores.items() if k.startswith(\"train\")]:\n print(f\" {metric.replace('train_', '')}: {score:.4f}\")\n print(\" Test Metrics:\")\n for metric, score in [(k, v) for k, v in baseline_scores.items() if k.startswith(\"test\")]:\n print(f\" {metric.replace('test_', '')}: {score:.4f}\")\n\n print(\"\\nFASTER (No Domain Knowledge):\")\n print(\" Train Metrics:\")\n for metric, score in [\n (k, v) for k, v in faster_no_domain_scores.items() if k.startswith(\"train\")\n ]:\n metric_name = metric.replace(\"train_\", \"\")\n baseline_score = baseline_scores[metric]\n print(f\" {colored_metric_output(metric_name, score, baseline_score)}\")\n print(\" Test Metrics:\")\n for metric, score in [\n (k, v) for k, v in faster_no_domain_scores.items() if k.startswith(\"test\")\n ]:\n metric_name = metric.replace(\"test_\", \"\")\n baseline_score = baseline_scores[metric]\n print(f\" {colored_metric_output(metric_name, score, baseline_score)}\")\n\n print(\"\\nFASTER (With Domain Knowledge):\")\n print(\" Train Metrics:\")\n for metric, score in [\n (k, v) for k, v in faster_with_domain_scores.items() if k.startswith(\"train\")\n ]:\n metric_name = metric.replace(\"train_\", \"\")\n baseline_score = baseline_scores[metric]\n print(f\" {colored_metric_output(metric_name, score, baseline_score)}\")\n print(\" Test Metrics:\")\n for metric, score in [\n (k, v) for k, v in faster_with_domain_scores.items() if k.startswith(\"test\")\n ]:\n metric_name = metric.replace(\"test_\", \"\")\n baseline_score = baseline_scores[metric]\n print(f\" {colored_metric_output(metric_name, score, baseline_score)}\")\n\n print(\"\\nGenerated Features:\")\n print(\"\\nNo Domain Knowledge:\")\n print(result_no_domain.selected_features)\n print(\"\\nWith Domain Knowledge:\")\n print(result_with_domain.selected_features)\n\n assert result_no_domain.selected_features is not None\n assert result_with_domain.selected_features is not None\n assert len(result_no_domain.selected_features) > 0\n assert len(result_with_domain.selected_features) > 0\n\n all_scores = {\n \"baseline\": baseline_scores,\n \"faster_no_domain\": faster_no_domain_scores,\n \"faster_with_domain\": faster_with_domain_scores,\n }\n\n print(\"\\nAll performance metrics:\", all_scores)\n"}}}} |