英文自然语言处理——电影评论情感判别
目录
1、导入所需的库
import os
import re
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
import nltk
from nltk.corpus import stopwords 2、用Pandas读入训练数据
#用pandas读入训练数据
datafile=os.path.join('E:\\english_data','labeledTrainData.tsv')
df=pd.read_csv(datafile,sep='\t',escapechar='\\')
print('Number of reviews:{}'.format(len(df)))
df.head() 3、构建停用词列表数据
#words_nostop=[w for w in words if w not in stopwords.words('english')]
stopwords={}.fromkeys([line.rstrip() for line in open('E:\\english_data\\stopwords.txt')])
eng_stopwords=set(stopwords) 4、对数据做预处理
（1）去掉html标签
（2）移除标点符号
（3）将句子切分成词
（4）去掉停用词
（5）重组为新的句子
def clean_text(text):
text=BeautifulSoup(text,'html.parser').get_text()
text=re.sub('[^a-zA-Z]',' ',text)
words=text.lower().split()
words=[w for w in words if w not in eng_stopwords]
return ' '.join(words)
5、将清洗的数据添加到DataFrame里
df['clean_review']=df.review.apply(clean_text)
df.head() 6、计算训练集中每条评论数据的向量
（1）使用sklearn的CountVectorizer抽取bag of words特征
vectorizer=CountVectorizer(max_features=5000)
train_data_features=vectorizer.fit_transform(df.clean_review).toarray()
train_data_features.shape （2）使用Gensim的Word2Vec训练词嵌入模型
from gensim.models.word2vec import Word2Vec
# 设定词向量训练的参数
num_features = 300 # Word vector dimensionality
min_word_count = 40 # Minimum word count
num_workers = 4 # Number of threads to run in parallel
context = 10 # Context window size
downsampling = 1e-3 # Downsample setting for frequent words
model = Word2Vec(sentences, workers=num_workers, \
size=num_features, min_count = min_word_count, \
window = context, sample = downsampling)
# If you don't plan to train the model any further, calling
# init_sims will make the model much more memory-efficient.
model.init_sims(replace=True)
# It can be helpful to create a meaningful model name and
# save the model for later use. You can load it later using Word2Vec.load()
model.save(os.path.join('..', 'models', model_name)) 7、构建随机森林分类器并训练
forest=RandomForestClassifier(n_estimators=100)
forest=forest.fit(train_data_features,df.sentiment)
#删除不用的占内容变量
del df
del train_data_features 8、读取测试数据并进行预测
datafile=os.path.join('E:\\english_data','testData.tsv')
df=pd.read_csv(datafile,sep='\t',escapechar='\\')
print('Number of reviews:{}'.format(len(df)))
df['clean_review']=df.review.apply(clean_text)
df.head()
test_data_features=vectorizer.transform(df.clean_review).toarray()
test_data_features.shape
result=forest.predict(test_data_features)
output=pd.DataFrame({'id':df.id,'sentiment':result})
output.head() 9、将预测结果写入csv文件
output.to_csv(os.path.join('E:\\english_data','Bag_of_Words_model.csv'),index=False)
del df
del test_data_features
