{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "aece24e2-ce8c-46d1-8220-29a29e9d8fdc", "metadata": { "tags": [] }, "outputs": [], "source": [ "import boto3\n", "import sagemaker\n", "\n", "original_boto3_version = boto3.__version__\n", "#%pip install 'boto3>1.17.21'" ] }, { "cell_type": "code", "execution_count": 2, "id": "8c19dc6b-9976-4d83-9e4f-d1c24518cfea", "metadata": { "tags": [] }, "outputs": [], "source": [ "from sagemaker.session import Session\n", "\n", "region = boto3.Session().region_name\n", "\n", "boto_session = boto3.Session(region_name=region)\n", "\n", "sagemaker_client = boto_session.client(service_name=\"sagemaker\", region_name=region)\n", "featurestore_runtime = boto_session.client(\n", " service_name=\"sagemaker-featurestore-runtime\", region_name=region\n", ")\n", "\n", "feature_store_session = Session(\n", " boto_session=boto_session,\n", " sagemaker_client=sagemaker_client,\n", " sagemaker_featurestore_runtime_client=featurestore_runtime,\n", ")" ] }, { "cell_type": "code", "execution_count": 3, "id": "157766db-657b-49ac-bab2-80fd8569c56c", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "sagemaker-us-east-1-103233932089\n" ] } ], "source": [ "# You can modify the following to use a bucket of your choosing\n", "default_s3_bucket_name = feature_store_session.default_bucket()\n", "prefix = \"sagemaker-us-east-1-103233932089\"\n", "\n", "print(default_s3_bucket_name)" ] }, { "cell_type": "code", "execution_count": 4, "id": "7654f0ce-44c0-4dcf-8a98-ca044af2f474", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "arn:aws:iam::103233932089:role/dev-sagemaker-notebook-audience-role\n" ] } ], "source": [ "from sagemaker import get_execution_role\n", "\n", "# You can modify the following to use a role of your choosing. See the documentation for how to create this.\n", "role = get_execution_role()\n", "print(role)" ] }, { "cell_type": "code", "execution_count": 5, "id": "153f7bb2-8874-4d39-8bcb-08f4c9cf4601", "metadata": { "tags": [] }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import io\n", "\n", "s3_client = boto3.client(\"s3\", region_name=region)\n", "\n", "fraud_detection_bucket_name = f\"sagemaker-example-files-prod-{region}\"\n", "identity_file_key = (\n", " \"datasets/tabular/fraud_detection/synthethic_fraud_detection_SA/sampled_identity.csv\"\n", ")\n", "transaction_file_key = (\n", " \"datasets/tabular/fraud_detection/synthethic_fraud_detection_SA/sampled_transactions.csv\"\n", ")\n", "\n", "identity_data_object = s3_client.get_object(\n", " Bucket=fraud_detection_bucket_name, Key=identity_file_key\n", ")\n", "transaction_data_object = s3_client.get_object(\n", " Bucket=fraud_detection_bucket_name, Key=transaction_file_key\n", ")\n", "\n", "identity_data = pd.read_csv(io.BytesIO(identity_data_object[\"Body\"].read()))\n", "transaction_data = pd.read_csv(io.BytesIO(transaction_data_object[\"Body\"].read()))\n", "\n", "identity_data = identity_data.round(5)\n", "transaction_data = transaction_data.round(5)\n", "\n", "identity_data = identity_data.fillna(0)\n", "transaction_data = transaction_data.fillna(0)\n", "\n", "# Feature transformations for this dataset are applied before ingestion into FeatureStore.\n", "# One hot encode card4, card6\n", "encoded_card_bank = pd.get_dummies(transaction_data[\"card4\"], prefix=\"card_bank\")\n", "encoded_card_type = pd.get_dummies(transaction_data[\"card6\"], prefix=\"card_type\")\n", "\n", "transformed_transaction_data = pd.concat(\n", " [transaction_data, encoded_card_type, encoded_card_bank], axis=1\n", ")\n", "# blank space is not allowed in feature name\n", "transformed_transaction_data = transformed_transaction_data.rename(\n", " columns={\"card_bank_american express\": \"card_bank_american_express\"}\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "3be71db6-8dfc-4f8a-98b5-b4874749e76d", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TransactionIDid_01id_02id_03id_04id_05id_06id_07id_08id_09...id_11id_12id_13id_14id_15id_16id_17id_18id_19id_20
02990130-538780.00.00.00.0-7001100.0...3280253241260125TFFT
12990266-1069246.00.00.00.0-6702100.0...4747122333860TFTF
22992553-45348819.00.00.00.0-7300100.0...211432681112135FFTF
32994568-15337170.00.00.00.0-1012100.0...5512725320213549FFTT
42994749-5680670.00.00.08.0-122100.0...5243257719254FFTT
\n", "

5 rows × 21 columns

\n", "
" ], "text/plain": [ " TransactionID id_01 id_02 id_03 id_04 id_05 id_06 id_07 id_08 \\\n", "0 2990130 -5 38780.0 0.0 0.0 0.0 -70 0 1 \n", "1 2990266 -10 69246.0 0.0 0.0 0.0 -67 0 2 \n", "2 2992553 -45 348819.0 0.0 0.0 0.0 -73 0 0 \n", "3 2994568 -15 337170.0 0.0 0.0 0.0 -10 1 2 \n", "4 2994749 -5 680670.0 0.0 0.0 8.0 -1 2 2 \n", "\n", " id_09 ... id_11 id_12 id_13 id_14 id_15 id_16 id_17 id_18 id_19 \\\n", "0 100.0 ... 32 80 253 241 260 125 T F F \n", "1 100.0 ... 47 47 122 33 38 60 T F T \n", "2 100.0 ... 21 143 268 111 2 135 F F T \n", "3 100.0 ... 55 127 253 202 135 49 F F T \n", "4 100.0 ... 52 43 257 7 19 254 F F T \n", "\n", " id_20 \n", "0 T \n", "1 F \n", "2 F \n", "3 T \n", "4 T \n", "\n", "[5 rows x 21 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "identity_data.head()" ] }, { "cell_type": "code", "execution_count": 7, "id": "dafe0cc8-ef58-4603-be39-a4712fc56d01", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TransactionIDisFraudTransactionDTTransactionAmtcard1card2card3card4card5card6...N8N9card_type_0card_type_creditcard_type_debitcard_bank_0card_bank_american_expresscard_bank_discovercard_bank_mastercardcard_bank_visa
033430870881085529.0012469360.0150.0mastercard126.0debit...FT00100010
1330731807955295107.9516188178.0150.0mastercard224.0debit...FT00100010
23555327015084339159.951825555.0150.0visa226.0debit...TF00100001
3331073608017157159.9510057225.0150.0mastercard224.0debit...FF00100010
4303471101127470117.0011444555.0150.0visa226.0debit...FF00100001
\n", "

5 rows × 56 columns

\n", "
" ], "text/plain": [ " TransactionID isFraud TransactionDT TransactionAmt card1 card2 card3 \\\n", "0 3343087 0 8810855 29.00 12469 360.0 150.0 \n", "1 3307318 0 7955295 107.95 16188 178.0 150.0 \n", "2 3555327 0 15084339 159.95 1825 555.0 150.0 \n", "3 3310736 0 8017157 159.95 10057 225.0 150.0 \n", "4 3034711 0 1127470 117.00 11444 555.0 150.0 \n", "\n", " card4 card5 card6 ... N8 N9 card_type_0 card_type_credit \\\n", "0 mastercard 126.0 debit ... F T 0 0 \n", "1 mastercard 224.0 debit ... F T 0 0 \n", "2 visa 226.0 debit ... T F 0 0 \n", "3 mastercard 224.0 debit ... F F 0 0 \n", "4 visa 226.0 debit ... F F 0 0 \n", "\n", " card_type_debit card_bank_0 card_bank_american_express \\\n", "0 1 0 0 \n", "1 1 0 0 \n", "2 1 0 0 \n", "3 1 0 0 \n", "4 1 0 0 \n", "\n", " card_bank_discover card_bank_mastercard card_bank_visa \n", "0 0 1 0 \n", "1 0 1 0 \n", "2 0 0 1 \n", "3 0 1 0 \n", "4 0 0 1 \n", "\n", "[5 rows x 56 columns]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "transformed_transaction_data.head()" ] }, { "cell_type": "markdown", "id": "ed4c90c6-6b97-4e0b-b8d5-db0d849c0806", "metadata": {}, "source": [ "\n", "In this step we will create the FeatureGroups representing the transaction and identity tables.\n", "" ] }, { "cell_type": "code", "execution_count": 9, "id": "803319e7-8f52-4f05-be2a-1ecd9b67bbe1", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "identity-feature-group-13-08-41-38\n", "transaction-feature-group-13-08-41-38\n" ] } ], "source": [ "from time import gmtime, strftime, sleep\n", "\n", "identity_feature_group_name = \"identity-feature-group-\" + strftime(\"%d-%H-%M-%S\", gmtime())\n", "transaction_feature_group_name = \"transaction-feature-group-\" + strftime(\"%d-%H-%M-%S\", gmtime())\n", "print(identity_feature_group_name)\n", "print(transaction_feature_group_name)" ] }, { "cell_type": "code", "execution_count": 11, "id": "9f37fb7c-e0a7-4f99-877a-1b14f7d59faf", "metadata": { "tags": [] }, "outputs": [], "source": [ "from sagemaker.feature_store.feature_group import FeatureGroup\n", "\n", "if identity_feature_group_name == '':\n", " print('manual')\n", " identity_feature_group_name = 'identity-feature-group-12-14-54-54'\n", " transaction_feature_group_name = 'transaction-feature-group-12-14-54-54'\n", "\n", "identity_feature_group = FeatureGroup(\n", " name=identity_feature_group_name, sagemaker_session=feature_store_session\n", ")\n", "transaction_feature_group = FeatureGroup(\n", " name=transaction_feature_group_name, sagemaker_session=feature_store_session\n", ")" ] }, { "cell_type": "code", "execution_count": 12, "id": "80672a0e-09dc-4658-b15f-a425f0f40453", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "[FeatureDefinition(feature_name='TransactionID', feature_type=),\n", " FeatureDefinition(feature_name='isFraud', feature_type=),\n", " FeatureDefinition(feature_name='TransactionDT', feature_type=),\n", " FeatureDefinition(feature_name='TransactionAmt', feature_type=),\n", " FeatureDefinition(feature_name='card1', feature_type=),\n", " FeatureDefinition(feature_name='card2', feature_type=),\n", " FeatureDefinition(feature_name='card3', feature_type=),\n", " FeatureDefinition(feature_name='card4', feature_type=),\n", " FeatureDefinition(feature_name='card5', feature_type=),\n", " FeatureDefinition(feature_name='card6', feature_type=),\n", " FeatureDefinition(feature_name='B1', feature_type=),\n", " FeatureDefinition(feature_name='B2', feature_type=),\n", " FeatureDefinition(feature_name='B3', feature_type=),\n", " FeatureDefinition(feature_name='B4', feature_type=),\n", " FeatureDefinition(feature_name='B5', feature_type=),\n", " FeatureDefinition(feature_name='B6', feature_type=),\n", " FeatureDefinition(feature_name='B7', feature_type=),\n", " FeatureDefinition(feature_name='B8', feature_type=),\n", " FeatureDefinition(feature_name='B9', feature_type=),\n", " FeatureDefinition(feature_name='B10', feature_type=),\n", " FeatureDefinition(feature_name='B11', feature_type=),\n", " FeatureDefinition(feature_name='B12', feature_type=),\n", " FeatureDefinition(feature_name='F1', feature_type=),\n", " FeatureDefinition(feature_name='F2', feature_type=),\n", " FeatureDefinition(feature_name='F3', feature_type=),\n", " FeatureDefinition(feature_name='F4', feature_type=),\n", " FeatureDefinition(feature_name='F5', feature_type=),\n", " FeatureDefinition(feature_name='F6', feature_type=),\n", " FeatureDefinition(feature_name='F7', feature_type=),\n", " FeatureDefinition(feature_name='F8', feature_type=),\n", " FeatureDefinition(feature_name='F9', feature_type=),\n", " FeatureDefinition(feature_name='F10', feature_type=),\n", " FeatureDefinition(feature_name='F11', feature_type=),\n", " FeatureDefinition(feature_name='F12', feature_type=),\n", " FeatureDefinition(feature_name='F13', feature_type=),\n", " FeatureDefinition(feature_name='F14', feature_type=),\n", " FeatureDefinition(feature_name='F15', feature_type=),\n", " FeatureDefinition(feature_name='F16', feature_type=),\n", " FeatureDefinition(feature_name='F17', feature_type=),\n", " FeatureDefinition(feature_name='N1', feature_type=),\n", " FeatureDefinition(feature_name='N2', feature_type=),\n", " FeatureDefinition(feature_name='N3', feature_type=),\n", " FeatureDefinition(feature_name='N4', feature_type=),\n", " FeatureDefinition(feature_name='N5', feature_type=),\n", " FeatureDefinition(feature_name='N6', feature_type=),\n", " FeatureDefinition(feature_name='N7', feature_type=),\n", " FeatureDefinition(feature_name='N8', feature_type=),\n", " FeatureDefinition(feature_name='N9', feature_type=),\n", " FeatureDefinition(feature_name='card_type_0', feature_type=),\n", " FeatureDefinition(feature_name='card_type_credit', feature_type=),\n", " FeatureDefinition(feature_name='card_type_debit', feature_type=),\n", " FeatureDefinition(feature_name='card_bank_0', feature_type=),\n", " FeatureDefinition(feature_name='card_bank_american_express', feature_type=),\n", " FeatureDefinition(feature_name='card_bank_discover', feature_type=),\n", " FeatureDefinition(feature_name='card_bank_mastercard', feature_type=),\n", " FeatureDefinition(feature_name='card_bank_visa', feature_type=),\n", " FeatureDefinition(feature_name='EventTime', feature_type=)]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import time\n", "\n", "current_time_sec = int(round(time.time()))\n", "\n", "\n", "def cast_object_to_string(data_frame):\n", " for label in data_frame.columns:\n", " if data_frame.dtypes[label] == \"object\":\n", " data_frame[label] = data_frame[label].astype(\"str\").astype(\"string\")\n", "\n", "\n", "# cast object dtype to string. The SageMaker FeatureStore Python SDK will then map the string dtype to String feature type.\n", "cast_object_to_string(identity_data)\n", "cast_object_to_string(transformed_transaction_data)\n", "\n", "# record identifier and event time feature names\n", "record_identifier_feature_name = \"TransactionID\"\n", "event_time_feature_name = \"EventTime\"\n", "\n", "# append EventTime feature\n", "identity_data[event_time_feature_name] = pd.Series(\n", " [current_time_sec] * len(identity_data), dtype=\"float64\"\n", ")\n", "transformed_transaction_data[event_time_feature_name] = pd.Series(\n", " [current_time_sec] * len(transaction_data), dtype=\"float64\"\n", ")\n", "\n", "# load feature definitions to the feature group. SageMaker FeatureStore Python SDK will auto-detect the data schema based on input data.\n", "identity_feature_group.load_feature_definitions(data_frame=identity_data)\n", "# output is suppressed\n", "transaction_feature_group.load_feature_definitions(data_frame=transformed_transaction_data)\n", "# output is suppressed" ] }, { "cell_type": "code", "execution_count": 13, "id": "41b63395-dc50-410a-96c8-7dc610372275", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "FeatureGroup identity-feature-group-13-08-41-38 successfully created.\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "Waiting for Feature Group Creation\n", "FeatureGroup transaction-feature-group-13-08-41-38 successfully created.\n" ] } ], "source": [ "def wait_for_feature_group_creation_complete(feature_group):\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " while status == \"Creating\":\n", " print(\"Waiting for Feature Group Creation\")\n", " time.sleep(5)\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " if status != \"Created\":\n", " raise RuntimeError(f\"Failed to create feature group {feature_group.name}\")\n", " print(f\"FeatureGroup {feature_group.name} successfully created.\")\n", "\n", "\n", "identity_feature_group.create(\n", " s3_uri=f\"s3://{default_s3_bucket_name}/{prefix}\",\n", " record_identifier_name=record_identifier_feature_name,\n", " event_time_feature_name=event_time_feature_name,\n", " role_arn=role,\n", " enable_online_store=False,\n", ")\n", "\n", "transaction_feature_group.create(\n", " s3_uri=f\"s3://{default_s3_bucket_name}/{prefix}\",\n", " record_identifier_name=record_identifier_feature_name,\n", " event_time_feature_name=event_time_feature_name,\n", " role_arn=role,\n", " enable_online_store=False,\n", ")\n", "\n", "wait_for_feature_group_creation_complete(feature_group=identity_feature_group)\n", "wait_for_feature_group_creation_complete(feature_group=transaction_feature_group)" ] }, { "cell_type": "code", "execution_count": 15, "id": "4e0387d4-b973-4f2b-b523-9fc6ffa124fd", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-13-08-41-38',\n", " 'FeatureGroupName': 'identity-feature-group-13-08-41-38',\n", " 'RecordIdentifierFeatureName': 'TransactionID',\n", " 'EventTimeFeatureName': 'EventTime',\n", " 'FeatureDefinitions': [{'FeatureName': 'TransactionID',\n", " 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_01', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_02', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'id_03', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'id_04', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'id_05', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'id_06', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_07', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_08', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_09', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'id_10', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_11', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_12', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_13', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_14', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_15', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_16', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'id_17', 'FeatureType': 'String'},\n", " {'FeatureName': 'id_18', 'FeatureType': 'String'},\n", " {'FeatureName': 'id_19', 'FeatureType': 'String'},\n", " {'FeatureName': 'id_20', 'FeatureType': 'String'},\n", " {'FeatureName': 'EventTime', 'FeatureType': 'Fractional'}],\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 39, 727000, tzinfo=tzlocal()),\n", " 'OfflineStoreConfig': {'S3StorageConfig': {'S3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-us-east-1-103233932089',\n", " 'ResolvedOutputS3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-us-east-1-103233932089/103233932089/sagemaker/us-east-1/offline-store/identity-feature-group-13-08-41-38-1686645759/data'},\n", " 'DisableGlueTableCreation': False,\n", " 'DataCatalogConfig': {'TableName': 'identity_feature_group_13_08_41_38_1686645759',\n", " 'Catalog': 'AwsDataCatalog',\n", " 'Database': 'sagemaker_featurestore'}},\n", " 'RoleArn': 'arn:aws:iam::103233932089:role/dev-sagemaker-notebook-audience-role',\n", " 'FeatureGroupStatus': 'Created',\n", " 'ResponseMetadata': {'RequestId': '9a48054c-ed93-48b3-9838-6eef9fffde1e',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '9a48054c-ed93-48b3-9838-6eef9fffde1e',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '2003',\n", " 'date': 'Tue, 13 Jun 2023 08:44:23 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "identity_feature_group.describe()" ] }, { "cell_type": "code", "execution_count": 16, "id": "757a9901-4e2c-4949-b5f3-06c645828372", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-13-08-41-38',\n", " 'FeatureGroupName': 'transaction-feature-group-13-08-41-38',\n", " 'RecordIdentifierFeatureName': 'TransactionID',\n", " 'EventTimeFeatureName': 'EventTime',\n", " 'FeatureDefinitions': [{'FeatureName': 'TransactionID',\n", " 'FeatureType': 'Integral'},\n", " {'FeatureName': 'isFraud', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'TransactionDT', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'TransactionAmt', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'card1', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card2', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'card3', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'card4', 'FeatureType': 'String'},\n", " {'FeatureName': 'card5', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'card6', 'FeatureType': 'String'},\n", " {'FeatureName': 'B1', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B2', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B3', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B4', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B5', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B6', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B7', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B8', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B9', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B10', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B11', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'B12', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F1', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F2', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F3', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F4', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F5', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F6', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F7', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F8', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F9', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F10', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F11', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F12', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F13', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F14', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F15', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F16', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'F17', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'N1', 'FeatureType': 'String'},\n", " {'FeatureName': 'N2', 'FeatureType': 'String'},\n", " {'FeatureName': 'N3', 'FeatureType': 'String'},\n", " {'FeatureName': 'N4', 'FeatureType': 'String'},\n", " {'FeatureName': 'N5', 'FeatureType': 'String'},\n", " {'FeatureName': 'N6', 'FeatureType': 'String'},\n", " {'FeatureName': 'N7', 'FeatureType': 'String'},\n", " {'FeatureName': 'N8', 'FeatureType': 'String'},\n", " {'FeatureName': 'N9', 'FeatureType': 'String'},\n", " {'FeatureName': 'card_type_0', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_type_credit', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_type_debit', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_bank_0', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_bank_american_express', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_bank_discover', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_bank_mastercard', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'card_bank_visa', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'EventTime', 'FeatureType': 'Fractional'}],\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 40, 870000, tzinfo=tzlocal()),\n", " 'OfflineStoreConfig': {'S3StorageConfig': {'S3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-us-east-1-103233932089',\n", " 'ResolvedOutputS3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-us-east-1-103233932089/103233932089/sagemaker/us-east-1/offline-store/transaction-feature-group-13-08-41-38-1686645760/data'},\n", " 'DisableGlueTableCreation': False,\n", " 'DataCatalogConfig': {'TableName': 'transaction_feature_group_13_08_41_38_1686645760',\n", " 'Catalog': 'AwsDataCatalog',\n", " 'Database': 'sagemaker_featurestore'}},\n", " 'RoleArn': 'arn:aws:iam::103233932089:role/dev-sagemaker-notebook-audience-role',\n", " 'FeatureGroupStatus': 'Created',\n", " 'ResponseMetadata': {'RequestId': 'ac1817a2-7a48-4215-8b17-d682b96c4233',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': 'ac1817a2-7a48-4215-8b17-d682b96c4233',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '3720',\n", " 'date': 'Tue, 13 Jun 2023 08:44:27 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "transaction_feature_group.describe()" ] }, { "cell_type": "code", "execution_count": 17, "id": "9e428b1d-3632-41c0-986a-5d82adb3dde1", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupSummaries': [{'FeatureGroupName': 'transaction-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 40, 870000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'transaction-feature-group-12-14-54-54',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-12-14-54-54',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 14, 58, 27, 979000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'identity-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 39, 727000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'identity-feature-group-12-14-54-54',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-12-14-54-54',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 14, 58, 26, 297000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'auto-mpg-2023-06-12-13-28-33',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/auto-mpg-2023-06-12-13-28-33',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 13, 40, 54, 714000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}}],\n", " 'ResponseMetadata': {'RequestId': '9ef94d95-cb92-42ef-8d60-68edf15d6513',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '9ef94d95-cb92-42ef-8d60-68edf15d6513',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '1222',\n", " 'date': 'Tue, 13 Jun 2023 08:44:38 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sagemaker_client.list_feature_groups() # use boto client to list FeatureGroups" ] }, { "cell_type": "markdown", "id": "0e44357a-1a22-4a32-8378-9621293fbbce", "metadata": {}, "source": [ "ingest data" ] }, { "cell_type": "markdown", "id": "9053c40b-e344-47c7-90ac-7e57dee96c6f", "metadata": {}, "source": [ "https://sagemaker-examples.readthedocs.io/en/latest/sagemaker-featurestore/feature_store_introduction.html" ] }, { "cell_type": "code", "execution_count": 20, "id": "c782257b-e796-4e8e-b697-45e47bf2f865", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "IngestionManagerPandas(feature_group_name='transaction-feature-group-13-08-41-38', sagemaker_fs_runtime_client_config=, sagemaker_session=, max_workers=5, max_processes=1, profile_name=None, _async_result=, _processing_pool=, _failed_indices=[])" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "identity_feature_group.ingest(data_frame=identity_data, max_workers=3, wait=True)\n", "transaction_feature_group.ingest(data_frame=transformed_transaction_data, max_workers=5, wait=True)" ] }, { "cell_type": "code", "execution_count": 32, "id": "eac666f5-6935-4bfb-9f7c-c404c4551ef2", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-13-08-41-38',\n", " 'FeatureGroupName': 'identity-feature-group-13-08-41-38',\n", " 'FeatureName': 'id_17',\n", " 'FeatureType': 'String',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 39, 727000, tzinfo=tzlocal()),\n", " 'LastModifiedTime': datetime.datetime(2023, 6, 13, 9, 2, 33, 375000, tzinfo=tzlocal()),\n", " 'Description': 'Something stringy.',\n", " 'Parameters': [],\n", " 'ResponseMetadata': {'RequestId': '5301f77b-31e2-4dad-87c7-dea020266f72',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '5301f77b-31e2-4dad-87c7-dea020266f72',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '331',\n", " 'date': 'Tue, 13 Jun 2023 09:02:33 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sagemaker.feature_store.inputs import FeatureParameter\n", "\n", "identity_feature_group.update_feature_metadata(\n", " feature_name=\"id_17\",\n", " description=\"Something stringy.\",\n", " #parameter_additions=[FeatureParameter(\"idType\", \"primaryKey\")],\n", ")\n", "\n", "identity_feature_group.describe_feature_metadata(feature_name=\"id_17\")" ] }, { "cell_type": "markdown", "id": "fea00a7c-c3d9-4d85-9ea2-a2eca4574b3e", "metadata": {}, "source": [ "Test online feature retreival" ] }, { "cell_type": "code", "execution_count": 18, "id": "6dcfab4d-b512-4907-8dff-c59171c1a366", "metadata": { "tags": [] }, "outputs": [], "source": [ "# record_identifier_value = str(2990130)\n", "\n", "# featurestore_runtime.get_record(\n", "# FeatureGroupName=transaction_feature_group_name,\n", "# RecordIdentifierValueAsString=record_identifier_value,\n", "# )" ] }, { "cell_type": "code", "execution_count": 18, "id": "e1aedcb2-7d1f-465d-9c0b-a2737cdd320d", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CREATE EXTERNAL TABLE IF NOT EXISTS sagemaker_featurestore.identity-feature-group-13-08-41-38 (\n", " TransactionID INT\n", " id_01 INT\n", " id_02 FLOAT\n", " id_03 FLOAT\n", " id_04 FLOAT\n", " id_05 FLOAT\n", " id_06 INT\n", " id_07 INT\n", " id_08 INT\n", " id_09 FLOAT\n", " id_10 INT\n", " id_11 INT\n", " id_12 INT\n", " id_13 INT\n", " id_14 INT\n", " id_15 INT\n", " id_16 INT\n", " id_17 STRING\n", " id_18 STRING\n", " id_19 STRING\n", " id_20 STRING\n", " EventTime FLOAT\n", " write_time TIMESTAMP\n", " event_time TIMESTAMP\n", " is_deleted BOOLEAN\n", ")\n", "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'\n", " STORED AS\n", " INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n", " OUTPUTFORMAT 'parquet.hive.DeprecatedParquetOutputFormat'\n", "LOCATION 's3://sagemaker-us-east-1-103233932089/sagemaker-us-east-1-103233932089/103233932089/sagemaker/us-east-1/offline-store/identity-feature-group-13-08-41-38-1686645759/data'\n" ] } ], "source": [ "print(identity_feature_group.as_hive_ddl())" ] }, { "cell_type": "code", "execution_count": 28, "id": "2c48b2ec-72a5-4076-b407-1dbd443018db", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FeatureGroup identity-feature-group-13-08-41-38 successfully created.\n", "FeatureGroup transaction-feature-group-13-08-41-38 successfully created.\n" ] } ], "source": [ "def check_feature_group_status(feature_group):\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " while status == \"Creating\":\n", " print(\"Waiting for Feature Group to be Created\")\n", " time.sleep(5)\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " print(f\"FeatureGroup {feature_group.name} successfully created.\")\n", "\n", "\n", "check_feature_group_status(identity_feature_group)\n", "check_feature_group_status(transaction_feature_group)" ] }, { "cell_type": "code", "execution_count": 19, "id": "b7770b33-789c-464e-88be-603bb0e3aa84", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "103233932089\n" ] } ], "source": [ "account_id = boto3.client(\"sts\").get_caller_identity()[\"Account\"]\n", "print(account_id)" ] }, { "cell_type": "code", "execution_count": 21, "id": "895bfba8-7caf-45e4-ae83-d9dcc338d65d", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Waiting for data in offline store...\n", "\n", "Waiting for data in offline store...\n", "\n", "Waiting for data in offline store...\n", "\n", "Waiting for data in offline store...\n", "\n", "Waiting for data in offline store...\n", "\n", "Data available.\n" ] } ], "source": [ "identity_feature_group_resolved_output_s3_uri = (\n", " identity_feature_group.describe()\n", " .get(\"OfflineStoreConfig\")\n", " .get(\"S3StorageConfig\")\n", " .get(\"ResolvedOutputS3Uri\")\n", ")\n", "transaction_feature_group_resolved_output_s3_uri = (\n", " transaction_feature_group.describe()\n", " .get(\"OfflineStoreConfig\")\n", " .get(\"S3StorageConfig\")\n", " .get(\"ResolvedOutputS3Uri\")\n", ")\n", "\n", "identity_feature_group_s3_prefix = identity_feature_group_resolved_output_s3_uri.replace(\n", " f\"s3://{default_s3_bucket_name}/\", \"\"\n", ")\n", "transaction_feature_group_s3_prefix = transaction_feature_group_resolved_output_s3_uri.replace(\n", " f\"s3://{default_s3_bucket_name}/\", \"\"\n", ")\n", "\n", "offline_store_contents = None\n", "while offline_store_contents is None:\n", " objects_in_bucket = s3_client.list_objects(\n", " Bucket=default_s3_bucket_name, Prefix=transaction_feature_group_s3_prefix\n", " )\n", " if \"Contents\" in objects_in_bucket and len(objects_in_bucket[\"Contents\"]) > 1:\n", " offline_store_contents = objects_in_bucket[\"Contents\"]\n", " else:\n", " print(\"Waiting for data in offline store...\\n\")\n", " sleep(60)\n", "\n", "print(\"Data available.\")" ] }, { "cell_type": "code", "execution_count": 22, "id": "c131a2fe-dd23-46c3-b9e3-2eb43c0a3b90", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running SELECT * FROM \"identity_feature_group_13_08_41_38_1686645759\" limit 10\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
transactionidid_01id_02id_03id_04id_05id_06id_07id_08id_09...id_15id_16id_17id_18id_19id_20eventtimewrite_timeapi_invocation_timeis_deleted
03017327-562496.00.00.017.0-7100100.0...273141TFTT1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:00.000False
13036658-76101232.00.00.01.0-6420100.0...20158TFTF1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:00.000False
23045171-5209707.00.00.015.0-9712100.0...100139TFFF1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:00.000False
33060389057200.00.00.010.0-6111100.0...130185FFTT1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:00.000False
43300166-5157431.00.00.01.0-5102100.0...45105FFTF1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:01.000False
53560111-15236894.00.00.02.0-3712100.0...6294FFTF1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:01.000False
63572028-592780.00.00.00.0-1921100.0...56220TFTF1.686646e+092023-06-13 08:52:55.2892023-06-13 08:48:01.000False
731092530393459.00.00.0-4.0-4010100.0...4050FFTT1.686646e+092023-06-13 08:52:55.2772023-06-13 08:47:59.000False
83374903-5196188.00.00.00.0-322100.0...112298FFTF1.686646e+092023-06-13 08:52:55.2772023-06-13 08:48:00.000False
93115165-5265299.00.00.0-27.0-7922100.0...5184FFTT1.686646e+092023-06-13 08:52:55.2772023-06-13 08:48:00.000False
\n", "

10 rows × 25 columns

\n", "
" ], "text/plain": [ " transactionid id_01 id_02 id_03 id_04 id_05 id_06 id_07 id_08 \\\n", "0 3017327 -5 62496.0 0.0 0.0 17.0 -71 0 0 \n", "1 3036658 -76 101232.0 0.0 0.0 1.0 -64 2 0 \n", "2 3045171 -5 209707.0 0.0 0.0 15.0 -97 1 2 \n", "3 3060389 0 57200.0 0.0 0.0 10.0 -61 1 1 \n", "4 3300166 -5 157431.0 0.0 0.0 1.0 -51 0 2 \n", "5 3560111 -15 236894.0 0.0 0.0 2.0 -37 1 2 \n", "6 3572028 -5 92780.0 0.0 0.0 0.0 -19 2 1 \n", "7 3109253 0 393459.0 0.0 0.0 -4.0 -40 1 0 \n", "8 3374903 -5 196188.0 0.0 0.0 0.0 -3 2 2 \n", "9 3115165 -5 265299.0 0.0 0.0 -27.0 -79 2 2 \n", "\n", " id_09 ... id_15 id_16 id_17 id_18 id_19 id_20 eventtime \\\n", "0 100.0 ... 273 141 T F T T 1.686646e+09 \n", "1 100.0 ... 201 58 T F T F 1.686646e+09 \n", "2 100.0 ... 100 139 T F F F 1.686646e+09 \n", "3 100.0 ... 130 185 F F T T 1.686646e+09 \n", "4 100.0 ... 45 105 F F T F 1.686646e+09 \n", "5 100.0 ... 62 94 F F T F 1.686646e+09 \n", "6 100.0 ... 56 220 T F T F 1.686646e+09 \n", "7 100.0 ... 40 50 F F T T 1.686646e+09 \n", "8 100.0 ... 112 298 F F T F 1.686646e+09 \n", "9 100.0 ... 51 84 F F T T 1.686646e+09 \n", "\n", " write_time api_invocation_time is_deleted \n", "0 2023-06-13 08:52:55.289 2023-06-13 08:48:00.000 False \n", "1 2023-06-13 08:52:55.289 2023-06-13 08:48:00.000 False \n", "2 2023-06-13 08:52:55.289 2023-06-13 08:48:00.000 False \n", "3 2023-06-13 08:52:55.289 2023-06-13 08:48:00.000 False \n", "4 2023-06-13 08:52:55.289 2023-06-13 08:48:01.000 False \n", "5 2023-06-13 08:52:55.289 2023-06-13 08:48:01.000 False \n", "6 2023-06-13 08:52:55.289 2023-06-13 08:48:01.000 False \n", "7 2023-06-13 08:52:55.277 2023-06-13 08:47:59.000 False \n", "8 2023-06-13 08:52:55.277 2023-06-13 08:48:00.000 False \n", "9 2023-06-13 08:52:55.277 2023-06-13 08:48:00.000 False \n", "\n", "[10 rows x 25 columns]" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "identity_feature_group = 'identity-feature-group-13-08-41-38'\n", "\n", "from sagemaker.feature_store.feature_group import FeatureGroup\n", "\n", "identity_feature_group = FeatureGroup(\n", " name=identity_feature_group, sagemaker_session=feature_store_session\n", ")\n", "\n", "\n", "identity_query = identity_feature_group.athena_query()\n", "identity_table = identity_query.table_name\n", "\n", "query_string = ('SELECT * FROM \"%s\" limit 10' % identity_table)\n", "print(\"Running \" + query_string)\n", "\n", "identity_query.run(\n", " query_string=query_string,\n", " output_location=\"s3://\" + default_s3_bucket_name + \"/\" + prefix + \"/query_results/\",\n", ")\n", "identity_query.wait()\n", "dataset = identity_query.as_dataframe()\n", "\n", "dataset" ] }, { "cell_type": "code", "execution_count": 23, "id": "98356b07-ccce-473b-b3a7-3c9e72d051fe", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running SELECT * FROM \"transaction_feature_group_13_08_41_38_1686645760\" LEFT JOIN \"identity_feature_group_13_08_41_38_1686645759\" ON \"transaction_feature_group_13_08_41_38_1686645760\".transactionid = \"identity_feature_group_13_08_41_38_1686645759\".transactionid\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
transactionidisfraudtransactiondttransactionamtcard1card2card3card4card5card6...id_15id_16id_17id_18id_19id_20eventtime.1write_time.1api_invocation_time.1is_deleted.1
0356655101544830959.001546111.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1357218401564080582.006231302.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2336475409459519107.9512501490.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
331763730422888477.001039391.0150.0visa166.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
4350204801347329059.0018018452.0150.0mastercard117.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
..................................................................
1995323489005888022311.9512686215.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1996353083001434044877.0016630242.0150.0visa166.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
199732743210706709649.0010313258.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1998332557208343387117.003277111.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
1999353098001434309845.004436174.0150.0visa226.0debit...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", "

2000 rows × 85 columns

\n", "
" ], "text/plain": [ " transactionid isfraud transactiondt transactionamt card1 card2 \\\n", "0 3566551 0 15448309 59.00 1546 111.0 \n", "1 3572184 0 15640805 82.00 6231 302.0 \n", "2 3364754 0 9459519 107.95 12501 490.0 \n", "3 3176373 0 4228884 77.00 1039 391.0 \n", "4 3502048 0 13473290 59.00 18018 452.0 \n", "... ... ... ... ... ... ... \n", "1995 3234890 0 5888022 311.95 12686 215.0 \n", "1996 3530830 0 14340448 77.00 16630 242.0 \n", "1997 3274321 0 7067096 49.00 10313 258.0 \n", "1998 3325572 0 8343387 117.00 3277 111.0 \n", "1999 3530980 0 14343098 45.00 4436 174.0 \n", "\n", " card3 card4 card5 card6 ... id_15 id_16 id_17 id_18 id_19 \\\n", "0 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "1 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "2 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "3 150.0 visa 166.0 debit ... NaN NaN NaN NaN NaN \n", "4 150.0 mastercard 117.0 debit ... NaN NaN NaN NaN NaN \n", "... ... ... ... ... ... ... ... ... ... ... \n", "1995 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "1996 150.0 visa 166.0 debit ... NaN NaN NaN NaN NaN \n", "1997 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "1998 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "1999 150.0 visa 226.0 debit ... NaN NaN NaN NaN NaN \n", "\n", " id_20 eventtime.1 write_time.1 api_invocation_time.1 is_deleted.1 \n", "0 NaN NaN NaN NaN NaN \n", "1 NaN NaN NaN NaN NaN \n", "2 NaN NaN NaN NaN NaN \n", "3 NaN NaN NaN NaN NaN \n", "4 NaN NaN NaN NaN NaN \n", "... ... ... ... ... ... \n", "1995 NaN NaN NaN NaN NaN \n", "1996 NaN NaN NaN NaN NaN \n", "1997 NaN NaN NaN NaN NaN \n", "1998 NaN NaN NaN NaN NaN \n", "1999 NaN NaN NaN NaN NaN \n", "\n", "[2000 rows x 85 columns]" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "identity_query = identity_feature_group.athena_query()\n", "transaction_query = transaction_feature_group.athena_query()\n", "\n", "identity_table = identity_query.table_name\n", "transaction_table = transaction_query.table_name\n", "\n", "query_string = (\n", " 'SELECT * FROM \"'\n", " + transaction_table\n", " + '\" LEFT JOIN \"'\n", " + identity_table\n", " + '\" ON \"'\n", " + transaction_table\n", " + '\".transactionid = \"'\n", " + identity_table\n", " + '\".transactionid'\n", ")\n", "print(\"Running \" + query_string)\n", "\n", "# run Athena query. The output is loaded to a Pandas dataframe.\n", "# dataset = pd.DataFrame()\n", "identity_query.run(\n", " query_string=query_string,\n", " output_location=\"s3://\" + default_s3_bucket_name + \"/\" + prefix + \"/query_results/\",\n", ")\n", "identity_query.wait()\n", "dataset = identity_query.as_dataframe()\n", "\n", "dataset" ] }, { "cell_type": "code", "execution_count": 43, "id": "1fa91880-67da-4419-9d04-e3f3d4cab5d0", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'Results': [{'FeatureGroup': {'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/auto-mpg-2023-06-12-13-28-33',\n", " 'FeatureGroupName': 'auto-mpg-2023-06-12-13-28-33',\n", " 'RecordIdentifierFeatureName': 'car_name',\n", " 'EventTimeFeatureName': 'event_time',\n", " 'FeatureDefinitions': [{'FeatureName': 'mpg', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'cylinders', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'displacement', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'horsepower', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'weight', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'acceleration', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'model_year', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'origin', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'car_name', 'FeatureType': 'String'},\n", " {'FeatureName': 'event_time', 'FeatureType': 'Fractional'}],\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 13, 40, 54, tzinfo=tzlocal()),\n", " 'OfflineStoreConfig': {'S3StorageConfig': {'S3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-studio-book/chapter04',\n", " 'ResolvedOutputS3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-studio-book/chapter04/103233932089/sagemaker/us-east-1/offline-store/auto-mpg-2023-06-12-13-28-33-1686577254/data'},\n", " 'DisableGlueTableCreation': False,\n", " 'DataCatalogConfig': {'TableName': 'auto_mpg_2023_06_12_13_28_33_1686577254',\n", " 'Catalog': 'AwsDataCatalog',\n", " 'Database': 'sagemaker_featurestore'}},\n", " 'RoleArn': 'arn:aws:iam::103233932089:role/dev-sagemaker-notebook-audience-role',\n", " 'FeatureGroupStatus': 'Created',\n", " 'Description': 'This feature group tracks the vehicle information such as mpg, and horsepower between 1970 and 1982.'}}],\n", " 'ResponseMetadata': {'RequestId': '6cc5834f-bc17-4ad8-888b-31bf61ac0bc5',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '6cc5834f-bc17-4ad8-888b-31bf61ac0bc5',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '1560',\n", " 'date': 'Tue, 13 Jun 2023 10:39:59 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sagemaker_client.search(\n", " Resource=\"FeatureGroup\",\n", " SearchExpression={\n", " 'Filters': [\n", " {\n", " 'Name': 'FeatureGroupName',\n", " 'Operator': 'Contains',\n", " 'Value': 'mpg'\n", " },\n", " ]\n", " }\n", ")" ] }, { "cell_type": "markdown", "id": "ddf3cdbc-deae-444e-a824-2136bf08e5ff", "metadata": {}, "source": [ " Delete old feature groups " ] }, { "cell_type": "code", "execution_count": 33, "id": "0147fcab-3229-4789-8e35-c2a5b55f0909", "metadata": { "tags": [] }, "outputs": [], "source": [ "identity_feature_group_name_for_del = 'identity-feature-group-12-14-54-54'\n", "transaction_feature_group_name_for_del = 'transaction-feature-group-12-14-54-54'\n", "\n", "identity_feature_group_for_del = FeatureGroup(\n", " name=identity_feature_group_name_for_del, sagemaker_session=feature_store_session\n", ")\n", "transaction_feature_group_for_del = FeatureGroup(\n", " name=identity_feature_group_name_for_del, sagemaker_session=feature_store_session\n", ")" ] }, { "cell_type": "code", "execution_count": 34, "id": "2887a629-fa40-46b3-a4c1-1ad80982da82", "metadata": { "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupSummaries': [{'FeatureGroupName': 'transaction-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 40, 870000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'transaction-feature-group-12-14-54-54',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-12-14-54-54',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 14, 58, 27, 979000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'identity-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 39, 727000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'identity-feature-group-12-14-54-54',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-12-14-54-54',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 14, 58, 26, 297000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'auto-mpg-2023-06-12-13-28-33',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/auto-mpg-2023-06-12-13-28-33',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 13, 40, 54, 714000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}}],\n", " 'ResponseMetadata': {'RequestId': 'c7628431-b865-42a7-995f-7ab6e05c1814',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': 'c7628431-b865-42a7-995f-7ab6e05c1814',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '1304',\n", " 'date': 'Tue, 13 Jun 2023 09:05:33 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sagemaker_client.list_feature_groups() # use boto client to list FeatureGroups" ] }, { "cell_type": "code", "execution_count": null, "id": "0c5782ed-4a08-4526-891e-7ffbe93a0109", "metadata": { "tags": [] }, "outputs": [], "source": [ "identity_feature_group_for_del.delete()\n", "transaction_feature_group_for_del.delete()\n", "sagemaker_client.list_feature_groups() # use boto client to list FeatureGroups" ] }, { "cell_type": "code", "execution_count": null, "id": "51e00110-e33b-41e2-b66c-b477bf09a127", "metadata": {}, "outputs": [], "source": [ "import sagemaker\n", "from sagemaker.feature_store.feature_group import FeatureGroup\n", "\n", "sagemaker_session = sagemaker.Session()\n", "fg_name = 'your-feature-group-name'\n", "\n", "my_fg = FeatureGroup(name=fg_name, sagemaker_session=sagemaker_session)\n", "my_fg.delete() " ] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.10" } }, "nbformat": 4, "nbformat_minor": 5 }