!pip install iqual --quiet

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from iqual import iqualnlp

# Install the ConvoKit library and download the Stanford Politeness Corpus
!pip install convokit --quiet

# Import the necessary functions
from convokit import Corpus, download

# Download the Stack Exchange Politeness Corpus
print("Downloading the Stack Exchange Politeness Corpus...")
corpus = Corpus(filename=download("stack-exchange-politeness-corpus"))
print(f"Download complete. Corpus contains {len(corpus.get_utterance_ids())} utterances.")

# Process the corpus into a DataFrame
rows = []

# Iterate through the utterances
for utt in corpus.iter_utterances():
    # Extract request text
    request_text = utt.text
    
    # Get metadata
    meta = utt.meta
    
    # Extract the binary politeness label (1="polite", 0="neutral", -1="impolite")
    if 'Binary' in meta:
        binary_label = meta['Binary']
        # Convert to our binary format (1=polite, 0=impolite/neutral)
        is_polite = 1 if binary_label == 1 else 0
        
        # For Stack Exchange, create a simplified context
        if len(request_text.split('.')) > 1:
            # Split at the first period
            parts = request_text.split('.', 1)
            context_text = parts[0] + '.'
            request_text = parts[1].strip()
        else:
            # For short texts, use a generic context
            context_text = "Stack Exchange discussion:"
        
        # Add to our list of data points
        rows.append({
            'context': context_text,
            'answer': request_text,
            'is_polite': is_polite
        })

# Convert to DataFrame
data = pd.DataFrame(rows)

# Remove any rows where question or answer is empty
data = data[data['context'].str.len() > 0]
data = data[data['answer'].str.len() > 0]

# Print stats
print(f"Created DataFrame with {len(data)} records")
print(f"Polite requests: {sum(data['is_polite'] == 1)}")
print(f"Impolite requests: {sum(data['is_polite'] == 0)}")

data.sample(10)

train_data, test_data = train_test_split(data, test_size=0.3)

# Create feature matrix (X) and target vector (y) for training
X_train = train_data[['context', 'answer']]
y_train = train_data['is_polite']

# Create feature matrix for testing
X_test = test_data[['context', 'answer']]
y_test = test_data['is_polite']

print(f"Training data size: {len(X_train)}")
print(f"Testing data size: {len(X_test)}")

# Create a model
model = iqualnlp.Model()

# Add text features. Define upto 2 text columns, usually a context/question and another for response/answer.
model.add_text_features(q_col='context', 
                        a_col='answer',
                        model='TfidfVectorizer',
                        env='scikit-learn',
                       )

# Add a classifier (Defaults to Logistic Regression)
model.add_classifier(name='LogisticRegression')

# Add a threshold component that will optimize for F1 score
model.add_threshold(scoring_metric='f1')

# Compile the model to finalize the pipeline structure
model.compile()

# Train the model on our training data
model.fit(X_train, y_train)

print("Model fit complete!")

model.model

y_pred = model.predict(X_test)
y_prob = model.predict_proba(X_test)

from sklearn.metrics import (
            f1_score, 
            accuracy_score,
            precision_score, 
            recall_score,
)


# Individual metrics
metrics = {
    'Accuracy': accuracy_score(y_test, y_pred),
    'Precision': precision_score(y_test,y_pred),
    'Recall': recall_score(y_test, y_pred),
    'F1 Score': f1_score(y_test, y_pred)
}

# Display metrics as a dataframe
metric_df = pd.DataFrame([metrics])
print("Performance Metrics (Out-Sample):")
metric_df

results = pd.DataFrame({
    'Context': X_test['context'],
    'Answer': X_test['answer'],
    'True Label': y_test,
    'Predicted Label': y_pred,
    'Probability': y_prob
})

# Display some examples of correct and incorrect predictions
print("Correct Predictions (True Positives):")
results[(results['True Label'] == 1) & (results['Predicted Label'] == 1)].head(3)

print("Incorrect Predictions (False Negatives):")
results[(results['True Label'] == 1) & (results['Predicted Label'] == 0)].head(3)