PCA.py

import pandas as pd
import numpy as np
import re, nltk        
from nltk.stem.porter import PorterStemmer
import random
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.decomposition import PCA

train_data_df = pd.read_csv('train_set.csv', header=None, delimiter="\t", quoting=3)
train_data_df.columns = ["Domain","Text"]
stemmer = PorterStemmer()

def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize(text):    
	text = re.sub("[^a-zA-Z]", " ", text)
	text = re.sub("(http://)", " ", text)
	text = re.sub(" +"," ", text)
	text = re.sub("\\b[a-zA-Z0-9]{10,100}\\b"," ",text)
	text = re.sub("\\b[a-zA-Z0-9]{0,1}\\b"," ",text)
	tokens = nltk.word_tokenize(text)
	stems = stem_tokens(tokens, stemmer)
	return stems

vectorizer = CountVectorizer(
    analyzer = 'word',
    tokenizer = tokenize,
    lowercase = True,
    stop_words = 'english',
    max_features = 85
)
       
X = vectorizer.fit_transform(train_data_df.Text.tolist()).todense()
pca = PCA(n_components=2).fit(X)
data2D = pca.transform(X)
print len(data2D[:,0])
print len(data2D[:,1])
plt.scatter(data2D[:,0], data2D[:,1], c=train_data_df.Domain)
plt.show()