{ "metadata": { "kernelspec": { "display_name": "Jupyter Notebook", "name": "jupyter" } }, "nbformat_minor": 5, "nbformat": 4, "cells": [ { "id": "bfbac53f-de24-4af3-9d30-ab03925027f2", "cell_type": "code", "metadata": { "language": "python" }, "source": "!pip install mlflow==2.22.1", "outputs": [], "execution_count": null }, { "cell_type": "code", "id": "568745d9-7b53-450f-b9fb-f4e47c5f1f1b", "metadata": { "language": "python", "name": "cell1" }, "source": "# Import python packages\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime, timedelta\nimport matplotlib.pyplot as plt\n\n# Import Snowpark\nfrom snowflake.snowpark.context import get_active_session\nsession = get_active_session()\n\n# import training_queries as tq\n# import model_evaluation as me\n# import x_cols\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport sklearn\n\n# from snowflake.ml.modeling.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.calibration import calibration_curve\nfrom sklearn.metrics import classification_report", "execution_count": null, "outputs": [] }, { "id": "c3b025f0-086c-48d0-90fa-0ad9767a11e7", "cell_type": "code", "metadata": { "language": "python" }, "source": "from snowflake.ml.registry import Registry\nregistry = Registry(session=session, database_name=\"FACTS\", schema_name=\"DEV\")\n\nfrom snowflake.ml.model import type_hints", "outputs": [], "execution_count": null }, { "id": "48dd1890-d93a-4c9e-b5fc-7f46c35bd2f7", "cell_type": "code", "metadata": { "language": "python" }, "source": "import mlflow\ntracking_server = 'http://mlflow.bf77.svc.spcs.internal:5000'\nmlflow.set_tracking_uri(uri=tracking_server)", "outputs": [], "execution_count": null }, { "id": "5bf2c41d-df08-4a93-85e8-a8217378a831", "cell_type": "code", "metadata": { "language": "python" }, "source": "# 1. Collect training data from 0925.\nq_t24 = \"\"\"select * from DEV_ENGINEERING.TSTOWE.TADAS_TRAINING_DATA_0925\"\"\"\ntraining_data_24_df = session.sql(q_t24).to_pandas()\n\n# 2. Add a unique_identifier, isrc_geo.\ntraining_data_24_df['ISRC_DATE'] = training_data_24_df['ISRC'] + \"_\" + training_data_24_df['REPORT_DATE_STR']", "outputs": [], "execution_count": null }, { "id": "09bacb23-5553-4c13-aa2b-a53ec544fa58", "cell_type": "code", "metadata": { "language": "python" }, "source": "# 3. Collect training data from 0825, but filter to USA.\n# 4. Collect labels from 0825 and apply above filter to USA.\n\nq_t25 = \"\"\"select * from DEV_ENGINEERING.TSTOWE.TADAS_TRAINING_DATA_0825\"\"\"\ntraining_data_25_df = session.sql(q_t25).to_pandas()\n\ntraining_data_25_df['ISRC'] = training_data_25_df['ISRC_GEO'].str[:-3]\ntraining_data_25_df['ISRC_DATE'] = training_data_25_df['ISRC'] + \"_\" + training_data_25_df['REPORT_DATE_STR']", "outputs": [], "execution_count": null }, { "id": "e1ea419d-57f0-4a90-9a9a-739002b8d6c1", "cell_type": "code", "metadata": { "language": "python" }, "source": "# 4. Collect labels from 0825 and apply above filter to USA.\nlabel_df = session.sql(\"\"\"select * from DEV_ENGINEERING.TSTOWE.TADAS_2025_08_LABELS\"\"\").to_pandas()\ntraining_data_25_df['isrc_geo_date'] = training_data_25_df['ISRC_GEO'] + '_' + training_data_25_df['REPORT_DATE_STR']\nlabel_df['REPORT_DATE_STR'] = label_df['REPORT_DATE'].astype(str)\nlabel_df['isrc_geo_date'] = label_df['ISRC_GEO'] + '_' + label_df['REPORT_DATE_STR']\nlabel_df_small = label_df[['isrc_geo_date', 'CONSECUTIVE_TREND', 'shift_30_34']]\ntraining_data_25_df = training_data_25_df.merge(label_df_small, how='left', on='isrc_geo_date')", "outputs": [], "execution_count": null }, { "id": "fc1d5d5a-7825-4fff-b632-1ed12afafae2", "cell_type": "code", "metadata": { "language": "python" }, "source": "# A) drop non-us stuff from 25.\n# need a geo column to do this.\ntraining_data_25_df['GEO'] = training_data_25_df['ISRC_GEO'].str[-2:]\ntraining_data_25_US_df = training_data_25_df[training_data_25_df['GEO'] == 'US']", "outputs": [], "execution_count": null }, { "id": "0ae09e4b-6c67-4699-83f6-dfb50861afbe", "cell_type": "code", "metadata": { "language": "python" }, "source": "# B) drop isrc_geo_date and geo from 25.\ndel training_data_25_US_df['isrc_geo_date']\ndel training_data_25_US_df['GEO']", "outputs": [], "execution_count": null }, { "id": "750f42c2-01fb-4ca3-ac9e-defbffdb0ab7", "cell_type": "code", "metadata": { "language": "python" }, "source": "# C) align \"consecutive_trend\" columns capitalizations.\ntraining_data_24_df.rename(columns={'consecutive_trend':'CONSECUTIVE_TREND'}, inplace=True)", "outputs": [], "execution_count": null }, { "id": "c8a73732-b8c9-491d-bb35-49a504611bcf", "cell_type": "code", "metadata": { "language": "python" }, "source": "# 5. Combine. Check size and dates of data.\ntraining_data_all_df = pd.concat([training_data_24_df, training_data_25_US_df], ignore_index=True)", "outputs": [], "execution_count": null }, { "id": "627f01ec-31c6-442d-8285-2d6fae7d5981", "cell_type": "code", "metadata": { "language": "python" }, "source": "training_data_all_df['DIVIDER'] = training_data_all_df['ISRC'].str[-7:].astype(int)", "outputs": [], "execution_count": null }, { "id": "ed1841d5-4be2-4418-9f04-0e7fe51b7aca", "cell_type": "code", "metadata": { "language": "python" }, "source": "##### MODELING V1: No training data adjustment\n##### [HOLD FOR LATER: --- V2: Oversample 2025.]\nv1_modeling_df = training_data_all_df.copy()\n\n# Make the data usable/modelable.\nv1_modeling_df = v1_modeling_df[v1_modeling_df['shift_30_34'].notnull()]\nv1_modeling_df = v1_modeling_df.replace([np.inf], 1000.0)\nv1_modeling_df = v1_modeling_df.replace([-np.inf], -1000.0)\nv1_modeling_df = v1_modeling_df.fillna(0.0)\nv1_modeling_df = v1_modeling_df.set_index('ISRC_DATE')\n\n# 6. Split based on isrc modulo, not split test.\ntraining_df = v1_modeling_df[v1_modeling_df['DIVIDER'].astype(int) % 10 <= 7]\ntest_df = v1_modeling_df[v1_modeling_df['DIVIDER'].astype(int) % 10 > 7]", "outputs": [], "execution_count": null }, { "id": "aed3b61d-fc14-46a7-ba39-8471aecb3c67", "cell_type": "code", "metadata": { "language": "python" }, "source": "# make X's from \"all\"\nX_train = training_df[x_cols.x_cols_no_amazon]\nX_test = test_df[x_cols.x_cols_no_amazon]\n\n# make y's\ny_train = training_df['shift_30_34']\ny_test = test_df['shift_30_34']", "outputs": [], "execution_count": null }, { "id": "09afd836-ceb8-4c4b-b0bd-d75e6a86fb96", "cell_type": "code", "metadata": { "language": "python" }, "source": "client = mlflow.tracking.MlflowClient()\nUSER_NAME = session.get_current_user().replace('\"', '')\n# Provide an Experiment description that will appear in the UI\nexperiment_description = (\n \"TADAS Update Model - Remove Amazon.\"\n)\n\n# Provide searchable tags that define characteristics of the Runs that\n# will be in this Experiment\nexperiment_tags = {\n \"project_name\": \"tadas\",\n \"team\": \"smeds\",\n \"mlflow.note.content\": experiment_description,\n}\n\n# Create the Experiment, providing a unique name\ntraining_experiment = client.get_experiment_by_name(\n name=f\"{USER_NAME}_TADAS30_FYQ225_NoAmazon\"\n)\n\nmy_experiment = mlflow.set_experiment(f\"{USER_NAME}_TADAS30_FYQ225_NoAmazon\")\n\n", "outputs": [], "execution_count": null }, { "id": "050fcaa8-0e17-4edb-8d97-8163fa49cc26", "cell_type": "code", "metadata": { "language": "python" }, "source": "# # 1. Model based on all of the available columns (because size of track may play a factor.)\n# run_name = \"random_forest_classifier_hypertuned\"\n\n# params = {\n# 'n_estimators': 300,\n# 'max_depth': 20,\n# 'min_samples_split': 5,\n# 'n_jobs': 4\n# }\n\n\n# # create and train a 30 day model.\n# rfc_model = RandomForestClassifier(class_weight='balanced_subsample', \n# n_estimators=params['n_estimators'], \n# max_depth=params['max_depth'], \n# min_samples_split=params['min_samples_split'], \n# n_jobs=params['n_jobs'],\n# random_state=42)\n# rfc_model.fit(X_train, y_train)\n\n# # predict for train and test\n# y_train_predict = rfc_model.predict(X_train)\n# y_test_predict = rfc_model.predict(X_test)\n\n# # predict_proba for train and test\n# y_train_proba = rfc_model.predict_proba(X_train)\n# y_test_proba = rfc_model.predict_proba(X_test)\n\nprob_true, prob_pred, this_fig = me.analyze_models(y_test, y_test_predict, y_test_proba, \"Model #1: RF with Class Balancing\", bins=10, visualize=True)\n\nece = me.expected_calibration_error(y_test, y_test_predict, n_bins=10)\n\nprint(ece)\nmetrics = {'ece': ece}\n\n# Initiate the MLflow run context\nwith mlflow.start_run(run_name=run_name) as run:\n # Log the parameters used for the model fit\n mlflow.log_params(params)\n\n # Log the error metrics that were calculated during validation\n mlflow.log_metrics(metrics)\n", "outputs": [], "execution_count": null }, { "id": "40403b8b-e502-44b6-9e4e-ff3783e4e0f1", "cell_type": "code", "metadata": { "language": "python" }, "source": "def analyze_models_temp(y_test, X_test_predict, X_test_proba, model_name, bins=10, visualize=True):\n ### Classification report for model\n report = classification_report(y_test, X_test_predict)\n\n print('Classification Report for', model_name,':\\n')\n # print(report)\n\n ### Compute calibration curve for model:\n # Access the predicted probabilities for the positive class\n X_test_proba_pos = X_test_proba[:, 1]\n prob_true, prob_pred = calibration_curve(y_test, X_test_proba_pos, n_bins=10)\n\n if visualize == True:\n # Plot reliability diagram\n fig, ax1 = plt.subplots(figsize=(10, 6))\n\n # Plot the calibration curve\n ax1.plot(prob_pred, prob_true, marker='o')\n ax1.plot([0, 1], [0, 1], linestyle='--', color='r')\n ax1.set_xlabel('Mean predicted probability')\n ax1.set_ylabel('Fraction of positives')\n ax1.set_title('Reliability Diagram for '+model_name)\n\n # Create a histogram to show the distribution of predicted probabilities\n ax2 = ax1.twinx()\n hist, bins, _ = ax2.hist(X_test_proba_pos, bins=bins, alpha=0.5)\n\n # Calculate bin centers\n bin_centers = 0.5 * (bins[:-1] + bins[1:])\n\n # Add count labels above the histogram bars\n for count, x in zip(hist, bin_centers):\n # Only put the label above non-zero bars\n if count > 0:\n ax2.text(x, count, str(int(count)), ha='center', va='bottom', color='grey')\n\n ax2.set_ylabel('Count')\n\n # Set the layout to prevent overlapping of plots\n fig.tight_layout()\n\n # Show the plot\n plt.show()\n\n return prob_true, prob_pred, fig", "outputs": [], "execution_count": null }, { "id": "27a99a40-17b5-4cee-b83b-f09b2ad1d605", "cell_type": "code", "metadata": { "language": "python" }, "source": "\nprob_true, prob_pred, fig = analyze_models_temp(y_test, y_test_predict, y_test_proba, \"Model #1: RF with Class Balancing\", bins=10, visualize=True)\n# with mlflow.start_run(run_name=run_name) as run:\n# mlflow.log_figure(fig, \"plots/tadas.png\")\n", "outputs": [], "execution_count": null }, { "id": "f8df27b2-80c7-49d1-ab70-32017ac265aa", "cell_type": "code", "metadata": { "language": "python" }, "source": "# 1. Model based on all of the available columns (because size of track may play a factor.)\nrun_name = \"random_forest_classifier_weighted\"\n\nparams = {\n 'n_jobs': 4\n}\n\n\n# create and train a 30 day model.\nrfc_model = RandomForestClassifier(class_weight='balanced_subsample',\n n_jobs=params['n_jobs'],\n random_state=42)\nrfc_model.fit(X_train, y_train)\n\n# predict for train and test\ny_train_predict = rfc_model.predict(X_train)\ny_test_predict = rfc_model.predict(X_test)\n\n# predict_proba for train and test\ny_train_proba = rfc_model.predict_proba(X_train)\ny_test_proba = rfc_model.predict_proba(X_test)\n\nprob_true, prob_pred, fig = analyze_models_temp(y_test, y_test_predict, y_test_proba, \"Model #2: RF with Class Balancing without hypertuning\", bins=10, visualize=True)\n\nece = me.expected_calibration_error(y_test, y_test_predict, n_bins=10)\n\nmetrics = {'ece': ece}\n\n# Initiate the MLflow run context\nwith mlflow.start_run(run_name=run_name) as run:\n # Log the parameters used for the model fit\n mlflow.log_params(params)\n\n # Log the error metrics that were calculated during validation\n mlflow.log_metrics(metrics)\n\n # log the callibration curve.\n mlflow.log_figure(fig, \"plots/tadas.png\")", "outputs": [], "execution_count": null }, { "id": "d158b302-c0d1-4245-a21b-cda8f610e70d", "cell_type": "code", "metadata": { "language": "python" }, "source": "\nregistry.log_model(\n model_name='TADAS_30_FYQ225_NoAmazon',\n model=rfc_model,\n version_name='dev_v1',\n target_platforms=[\"WAREHOUSE\", \"SNOWPARK_CONTAINER_SERVICES\"],\n task=type_hints.Task.TABULAR_REGRESSION,\n comment='Initial Re-trained TADAS30 Model.',\n sample_input_data = X_train.head()\n)", "outputs": [], "execution_count": null }, { "id": "17b3a07f-87fe-4e25-9734-6ff8278c39bc", "cell_type": "code", "metadata": { "language": "python" }, "source": "model_ver = registry.get_model('TADAS_30_FYQ225').version(\"DEV_V1\")\npredictions = model_ver.run(X_test.head(), function_name=\"PREDICT_PROBA\")", "outputs": [], "execution_count": null }, { "id": "aa2cc40c-3d01-43fc-8065-85fd5b72fc93", "cell_type": "code", "metadata": { "language": "python" }, "source": "runs = client.search_runs(\n experiment_ids=[my_experiment.experiment_id],\n filter_string=\"\", # Optional: filter by tags or metrics\n run_view_type=mlflow.entities.ViewType.ACTIVE_ONLY,\n max_results=100\n)\n\n# Extract run_id and accuracy from each run\ndata = []\nfor run in runs:\n run_id = run.info.run_id\n metrics = run.data.metrics\n accuracy = metrics.get(\"ece\", None) # or another metric name\n run_name = run.info.run_name\n if accuracy is not None:\n data.append({\"run_id\": run_id, \"ece\": accuracy, \"run_name\": run_name})\n\n# Convert to DataFrame and sort by accuracy\ndf = pd.DataFrame(data)\ndf_sorted = df.sort_values(by=\"ece\", ascending=False)\n\nprint(df_sorted)\ndf_sorted.plot(kind=\"bar\", x=\"run_name\", y=\"ece\")", "outputs": [], "execution_count": null } ] }