{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import dateparser\n",
    "import numpy as np\n",
    "import datetime\n",
    "from datetime import datetime\n",
    "\n",
    "def clean_features(df):\n",
    "    \"\"\" Replace blanks with \"real empty\" (NaN); replace None (and string \"None\") with numpy NaN \"\"\"\n",
    "    df = df.replace(r'', np.nan)\n",
    "    df.fillna(value=np.nan, inplace=True)\n",
    "    df.replace('None', np.nan, inplace=True)\n",
    "    df.replace('unknown', np.nan, inplace=True)\n",
    "    \n",
    "    # Add \"missing\" instead NaN so our plots would show those\n",
    "    df.fillna('missing', inplace=True)\n",
    "    \n",
    "    return df\n",
    "\n",
    "\n",
    "def generate_date_features(feature_vector, date_cols):\n",
    "\n",
    "    # TODO! why \"hour\" is missing sometimes?\n",
    "\n",
    "    # TODO! add extracted time cols diff with other date cols\n",
    "\n",
    "    \"\"\"\" Limit date cols, and pick cols that does not have \"birth\" or \"dob\" in it \"\"\"\n",
    "    try:\n",
    "        # TODO! Optimize this logic\n",
    "        dobs = [x.lower() for x in date_cols if any(y.lower() in x.lower() for y in ['birth', 'dob'])]\n",
    "        new_dates = [x for x in date_cols if x.lower() not in dobs]\n",
    "        if len(new_dates) > 0:\n",
    "            date_cols = [x for x in date_cols if x.lower() not in dobs] #[0:3]\n",
    "        else:\n",
    "            date_cols = []\n",
    "    except Exception as e:\n",
    "        pass\n",
    "\n",
    "\n",
    "    \"\"\" Do some feature engineering with dates - extract datetime part and datetime diff \"\"\"\n",
    "    if len(date_cols) > 0:\n",
    "        date_extract_dow_names = [f\"{x}_dow\" for x in date_cols]\n",
    "        date_extract_hour_names = [f\"{x}_hour\" for x in date_cols]\n",
    "        feature_vector[date_extract_dow_names] = feature_vector[date_cols].applymap(lambda x: extract_datetime_part(x, '%w'))\n",
    "        feature_vector[date_extract_hour_names] = feature_vector[date_cols].applymap(lambda x: extract_datetime_part(x, '%H'))\n",
    "\n",
    "        # Get difference between date and today # TODO! oon't need it when we have \"max_date\" already\n",
    "        # date_extract_datediff_now_names = [f\"{x}_days_diff_today\" for x in date_cols]\n",
    "        # feature_vector[date_extract_datediff_now_names] = feature_vector[date_cols].applymap(lambda x: extract_datediff(x, datetime.datetime.today()))\n",
    "\n",
    "        # Get difference between 2 dates in a list by creating pairs of all available dates\n",
    "        if len(date_cols) >= 2:\n",
    "            for pair in itertools.combinations(new_dates, 2):\n",
    "                field_name = f\"{pair[0]}_{pair[1]}_diff\"\n",
    "                feature_vector[field_name] = feature_vector[list(pair)].apply(lambda x: extract_datediff(x[0], x[1]), axis=1)\n",
    "\n",
    "    return feature_vector\n",
    "\n",
    "\n",
    "def extract_datetime_part(date_string, part_fmt):\n",
    "    \"\"\" Extract day of week. \"\"\"\n",
    "    # TODO! investigate why hour doesn't work\n",
    "    datetime_part = np.nan\n",
    "    try:\n",
    "        datetime_part = dateparser.parse(str(date_string), settings={'PREFER_DATES_FROM': 'past'}).strftime(part_fmt)\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    return datetime_part\n",
    "\n",
    "\n",
    "def extract_datediff(first_date_string, second_date_string):\n",
    "    \"\"\" Calculate difference between date and now.\n",
    "    # TODO! with sufficient labelling info, we could calculate diff between actual event date\n",
    "    \"\"\"\n",
    "    diff = np.nan\n",
    "    try:\n",
    "        diff = abs(dateparser.parse(str(first_date_string), settings={'PREFER_DATES_FROM': 'past'}) - dateparser.parse(\n",
    "            str(second_date_string), settings={'PREFER_DATES_FROM': 'past'})).days\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    return diff\n",
    "\n",
    "\n",
    "def calculate_diff(x, y):\n",
    "    \"\"\" Calculate difference between two values\n",
    "    \"\"\"\n",
    "    diff = np.nan\n",
    "    try:\n",
    "        diff = y - x\n",
    "    except Exception as e:\n",
    "        pass\n",
    "    return diff\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}