{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2be1b7ab",
   "metadata": {},
   "source": [
    "# Efficiency - Test"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2f03db12",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "135430fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from iqual import  tests"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "17eefba0",
   "metadata": {},
   "source": [
    "### Load datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "285f4255",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>data_round</th>\n",
       "      <th>refugee_status</th>\n",
       "      <th>bootstrap_run</th>\n",
       "      <th>annotated</th>\n",
       "      <th>religious_act</th>\n",
       "      <th>secular_act</th>\n",
       "      <th>no_ambition_act</th>\n",
       "      <th>vague_job_act</th>\n",
       "      <th>job_secular_act</th>\n",
       "      <th>...</th>\n",
       "      <th>ability_high</th>\n",
       "      <th>ability_low</th>\n",
       "      <th>budget_high</th>\n",
       "      <th>budget_low</th>\n",
       "      <th>awareness_information_high</th>\n",
       "      <th>awareness_information_low</th>\n",
       "      <th>camp_regulations</th>\n",
       "      <th>covid_impacts</th>\n",
       "      <th>public_assistance</th>\n",
       "      <th>worries_anxieties</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C601001009R2</td>\n",
       "      <td>R2</td>\n",
       "      <td>refugee</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C601001009R3</td>\n",
       "      <td>R3</td>\n",
       "      <td>refugee</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.294118</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C601002009R2</td>\n",
       "      <td>R2</td>\n",
       "      <td>refugee</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C601003005R3</td>\n",
       "      <td>R3</td>\n",
       "      <td>refugee</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C602004004R3</td>\n",
       "      <td>R3</td>\n",
       "      <td>refugee</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            uid data_round refugee_status  bootstrap_run  annotated  \\\n",
       "0  C601001009R2         R2        refugee              1          1   \n",
       "1  C601001009R3         R3        refugee              1          1   \n",
       "2  C601002009R2         R2        refugee              1          1   \n",
       "3  C601003005R3         R3        refugee              1          1   \n",
       "4  C602004004R3         R3        refugee              1          1   \n",
       "\n",
       "   religious_act  secular_act  no_ambition_act  vague_job_act  \\\n",
       "0           0.00     0.166667              0.0       0.333333   \n",
       "1           0.00     0.117647              0.0       0.058824   \n",
       "2           0.00     0.000000              0.0       0.125000   \n",
       "3           0.00     0.000000              0.0       0.000000   \n",
       "4           0.05     0.000000              0.0       0.000000   \n",
       "\n",
       "   job_secular_act  ...  ability_high  ability_low  budget_high  budget_low  \\\n",
       "0         0.000000  ...           0.0     0.000000          0.0    0.333333   \n",
       "1         0.294118  ...           0.0     0.058824          0.0    0.000000   \n",
       "2         0.000000  ...           0.0     0.125000          0.0    0.125000   \n",
       "3         0.250000  ...           0.0     0.000000          0.0    0.000000   \n",
       "4         0.200000  ...           0.1     0.000000          0.0    0.000000   \n",
       "\n",
       "   awareness_information_high  awareness_information_low  camp_regulations  \\\n",
       "0                         0.0                   0.000000              0.00   \n",
       "1                         0.0                   0.117647              0.00   \n",
       "2                         0.0                   0.000000              0.25   \n",
       "3                         0.0                   0.000000              0.00   \n",
       "4                         0.0                   0.000000              0.05   \n",
       "\n",
       "   covid_impacts  public_assistance  worries_anxieties  \n",
       "0          0.000                0.0                0.0  \n",
       "1          0.000                0.0                0.0  \n",
       "2          0.000                0.0                0.0  \n",
       "3          0.125                0.0                0.0  \n",
       "4          0.000                0.0                0.0  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir         = \"../../data\"\n",
    "\n",
    "### Enhanced bootstrap data\n",
    "bootstrap_df = pd.read_csv(os.path.join(data_dir,\"enhanced_bootstrap_mean.csv\"))\n",
    "bootstrap_df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cb9fc0f1",
   "metadata": {},
   "source": [
    "### Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3ddf1950",
   "metadata": {},
   "outputs": [],
   "source": [
    "annotation_vars = [\n",
    "    'ability_high', 'ability_low', \n",
    "    'awareness_information_high', 'awareness_information_low', \n",
    "    'budget_high', 'budget_low', 'covid_impacts', \n",
    "    'education_high', 'education_low', 'education_neutral', 'education_religious', \n",
    "    'entrepreneur', 'job_secular', 'marriage', 'migration', \n",
    "    'no_ambition', 'public_assistance', 'reliance_on_god', \n",
    "    'religious', 'secular', 'vague_job', 'vague_non_specific', \n",
    "    'vocational_training', 'worries_anxieties',\n",
    "]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7eac3f4d",
   "metadata": {},
   "source": [
    "### Separate Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67c55028",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Create three datasets\n",
    "\n",
    "# > Human DF\n",
    "human_df   = bootstrap_df[bootstrap_df.annotated==1]\n",
    "\n",
    "# Machine DF\n",
    "machine_df = bootstrap_df[bootstrap_df.annotated==0]\n",
    "\n",
    "# Out-sample data / In-sample average predictions\n",
    "\n",
    "## >  Out-Sample\n",
    "# test_df  = bootstrap_df[bootstrap_df.split=='test']] \n",
    "\n",
    "## >  In-sample average predictions\n",
    "med_df     = bootstrap_df[bootstrap_df.annotated==1].groupby(['uid','refugee_status','data_round']).mean(numeric_only=True).reset_index()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "607a7d39",
   "metadata": {},
   "source": [
    "### Use `tests.Efficiency` from `iQual` to get `Measurement Error Variances`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c44f7bb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Measurement Error Variances\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>annotation</th>\n",
       "      <th>sig2_h</th>\n",
       "      <th>sig2_y</th>\n",
       "      <th>sig2_eps</th>\n",
       "      <th>se_h</th>\n",
       "      <th>se_enh</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ability_high</td>\n",
       "      <td>0.0064</td>\n",
       "      <td>0.0098</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.0021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ability_low</td>\n",
       "      <td>0.0057</td>\n",
       "      <td>0.0050</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0027</td>\n",
       "      <td>0.0018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>awareness_information_high</td>\n",
       "      <td>0.0091</td>\n",
       "      <td>0.0096</td>\n",
       "      <td>0.0070</td>\n",
       "      <td>0.0034</td>\n",
       "      <td>0.0024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>awareness_information_low</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0008</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0011</td>\n",
       "      <td>0.0008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>budget_high</td>\n",
       "      <td>0.0046</td>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>0.0024</td>\n",
       "      <td>0.0017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>budget_low</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0116</td>\n",
       "      <td>0.0060</td>\n",
       "      <td>0.0044</td>\n",
       "      <td>0.0026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>covid_impacts</td>\n",
       "      <td>0.0037</td>\n",
       "      <td>0.0030</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0.0022</td>\n",
       "      <td>0.0013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education_high</td>\n",
       "      <td>0.0093</td>\n",
       "      <td>0.0090</td>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.0034</td>\n",
       "      <td>0.0023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>education_low</td>\n",
       "      <td>0.0027</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>education_neutral</td>\n",
       "      <td>0.0245</td>\n",
       "      <td>0.0267</td>\n",
       "      <td>0.0108</td>\n",
       "      <td>0.0056</td>\n",
       "      <td>0.0037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>education_religious</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0024</td>\n",
       "      <td>0.0016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>0.0053</td>\n",
       "      <td>0.0075</td>\n",
       "      <td>0.0015</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>job_secular</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0175</td>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.0045</td>\n",
       "      <td>0.0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>marriage</td>\n",
       "      <td>0.0133</td>\n",
       "      <td>0.0127</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.0024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>migration</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>no_ambition</td>\n",
       "      <td>0.0015</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>public_assistance</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0017</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0.0015</td>\n",
       "      <td>0.0010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>reliance_on_god</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.0043</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>religious</td>\n",
       "      <td>0.0060</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0027</td>\n",
       "      <td>0.0018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>secular</td>\n",
       "      <td>0.0090</td>\n",
       "      <td>0.0084</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0034</td>\n",
       "      <td>0.0022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>vague_job</td>\n",
       "      <td>0.0129</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0047</td>\n",
       "      <td>0.0040</td>\n",
       "      <td>0.0026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>vague_non_specific</td>\n",
       "      <td>0.0062</td>\n",
       "      <td>0.0073</td>\n",
       "      <td>0.0049</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>vocational_training</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.0013</td>\n",
       "      <td>0.0007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>worries_anxieties</td>\n",
       "      <td>0.0061</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0033</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    annotation  sig2_h  sig2_y  sig2_eps   se_h  se_enh\n",
       "0                 ability_high  0.0064  0.0098    0.0036 0.0029  0.0021\n",
       "1                  ability_low  0.0057  0.0050    0.0038 0.0027  0.0018\n",
       "2   awareness_information_high  0.0091  0.0096    0.0070 0.0034  0.0024\n",
       "3    awareness_information_low  0.0010  0.0008    0.0010 0.0011  0.0008\n",
       "4                  budget_high  0.0046  0.0055    0.0025 0.0024  0.0017\n",
       "5                   budget_low  0.0156  0.0116    0.0060 0.0044  0.0026\n",
       "6                covid_impacts  0.0037  0.0030    0.0009 0.0022  0.0013\n",
       "7               education_high  0.0093  0.0090    0.0055 0.0034  0.0023\n",
       "8                education_low  0.0027  0.0023    0.0014 0.0019  0.0012\n",
       "9            education_neutral  0.0245  0.0267    0.0108 0.0056  0.0037\n",
       "10         education_religious  0.0047  0.0049    0.0023 0.0024  0.0016\n",
       "11                entrepreneur  0.0053  0.0075    0.0015 0.0026  0.0018\n",
       "12                 job_secular  0.0156  0.0175    0.0055 0.0045  0.0029\n",
       "13                    marriage  0.0133  0.0127    0.0016 0.0041  0.0024\n",
       "14                   migration  0.0042  0.0026    0.0007 0.0023  0.0012\n",
       "15                 no_ambition  0.0015  0.0010    0.0010 0.0014  0.0009\n",
       "16           public_assistance  0.0019  0.0017    0.0009 0.0015  0.0010\n",
       "17             reliance_on_god  0.0041  0.0043    0.0020 0.0023  0.0015\n",
       "18                   religious  0.0060  0.0073    0.0020 0.0027  0.0018\n",
       "19                     secular  0.0090  0.0084    0.0042 0.0034  0.0022\n",
       "20                   vague_job  0.0129  0.0136    0.0047 0.0040  0.0026\n",
       "21          vague_non_specific  0.0062  0.0073    0.0049 0.0028  0.0021\n",
       "22         vocational_training  0.0014  0.0010    0.0003 0.0013  0.0007\n",
       "23           worries_anxieties  0.0061  0.0042    0.0033 0.0028  0.0017"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.float_format = '{:.4f}'.format\n",
    "\n",
    "efficiency_df  = tests.Efficiency(human_df,med_df,machine_df,annotation_vars).get_results_all()\n",
    "print(\"Measurement Error Variances\")\n",
    "efficiency_df[['annotation','sig2_h','sig2_y','sig2_eps','se_h','se_enh']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}