{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./model_abtest.log','r') as f:\n",
    "    raw = f.read().split('\\n')[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list = []\n",
    "for event in raw:\n",
    "    obs = json.loads(event)\n",
    "    obs['extra_data'] = obs.get('extra_data',None)\n",
    "    data_list.append(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>time</th>\n",
       "      <th>salt</th>\n",
       "      <th>inputs</th>\n",
       "      <th>params</th>\n",
       "      <th>event</th>\n",
       "      <th>checksum</th>\n",
       "      <th>extra_data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>1602739669</td>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>{'userid': '431'}</td>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>exposure</td>\n",
       "      <td>796b9a12</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>1602739720</td>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>{'userid': '431'}</td>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>exposure</td>\n",
       "      <td>796b9a12</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>1602739722</td>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>{'userid': '431'}</td>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>exposure</td>\n",
       "      <td>796b9a12</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>1602739722</td>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>{'userid': '431'}</td>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>exposure</td>\n",
       "      <td>796b9a12</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>1602739724</td>\n",
       "      <td>ModelExperiment</td>\n",
       "      <td>{'userid': '637'}</td>\n",
       "      <td>{'use_pytorch': 0, 'model_type': 'surprise'}</td>\n",
       "      <td>exposure</td>\n",
       "      <td>796b9a12</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              name        time             salt             inputs  \\\n",
       "0  ModelExperiment  1602739669  ModelExperiment  {'userid': '431'}   \n",
       "1  ModelExperiment  1602739720  ModelExperiment  {'userid': '431'}   \n",
       "2  ModelExperiment  1602739722  ModelExperiment  {'userid': '431'}   \n",
       "3  ModelExperiment  1602739722  ModelExperiment  {'userid': '431'}   \n",
       "4  ModelExperiment  1602739724  ModelExperiment  {'userid': '637'}   \n",
       "\n",
       "                                         params     event  checksum extra_data  \n",
       "0   {'use_pytorch': 1, 'model_type': 'pytorch'}  exposure  796b9a12       None  \n",
       "1   {'use_pytorch': 1, 'model_type': 'pytorch'}  exposure  796b9a12       None  \n",
       "2   {'use_pytorch': 1, 'model_type': 'pytorch'}  exposure  796b9a12       None  \n",
       "3   {'use_pytorch': 1, 'model_type': 'pytorch'}  exposure  796b9a12       None  \n",
       "4  {'use_pytorch': 0, 'model_type': 'surprise'}  exposure  796b9a12       None  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame.from_dict(data_list) \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_df = df.loc[df.event=='rate',['params','extra_data']]\n",
    "exp_df['variant'] = exp_df['params'].apply(lambda x: x['use_pytorch'])\n",
    "exp_df['rating'] = exp_df['extra_data'].apply(lambda x: x['rate_val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>params</th>\n",
       "      <th>extra_data</th>\n",
       "      <th>variant</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>{'use_pytorch': 0, 'model_type': 'surprise'}</td>\n",
       "      <td>{'rate_val': 3}</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>{'rate_val': 7}</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>{'use_pytorch': 1, 'model_type': 'pytorch'}</td>\n",
       "      <td>{'rate_val': 0}</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>{'use_pytorch': 0, 'model_type': 'surprise'}</td>\n",
       "      <td>{'rate_val': 7}</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>{'use_pytorch': 0, 'model_type': 'surprise'}</td>\n",
       "      <td>{'rate_val': 3}</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          params       extra_data  variant  \\\n",
       "32  {'use_pytorch': 0, 'model_type': 'surprise'}  {'rate_val': 3}        0   \n",
       "36   {'use_pytorch': 1, 'model_type': 'pytorch'}  {'rate_val': 7}        1   \n",
       "41   {'use_pytorch': 1, 'model_type': 'pytorch'}  {'rate_val': 0}        1   \n",
       "44  {'use_pytorch': 0, 'model_type': 'surprise'}  {'rate_val': 7}        0   \n",
       "48  {'use_pytorch': 0, 'model_type': 'surprise'}  {'rate_val': 3}        0   \n",
       "\n",
       "    rating  \n",
       "32       3  \n",
       "36       7  \n",
       "41       0  \n",
       "44       7  \n",
       "48       3  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = exp_df.loc[exp_df.variant==0,'rating'].values\n",
    "d2 = exp_df.loc[exp_df.variant==1,'rating'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9650813700331365 0.37882815062317066\n"
     ]
    }
   ],
   "source": [
    "import scipy.stats\n",
    "statistic, pval = scipy.stats.ttest_ind(d1,d2)\n",
    "print(statistic,pval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "d1 = random.choices(range(1,6),k=50)\n",
    "d2 = random.choices(range(3,10),k=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=-10.876400571212649, pvalue=1.5325813144977069e-18)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scipy.stats.ttest_ind(d1,d2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}