Rachel Yang Text Mining Project

movie_subtitles.py

@@ -0,0 +1,109 @@
+import re
+import pickle
+import string
+from string import digits
+from nltk.corpus import stopwords
+
+"""
+Parse through already downloaded movie scripts/subtitles to reduce to only words
+"""
+
+def edited_file_name(movie_script):	
+	"""
+	Ensure new movie file name is clear
+
+	>>> edited_file_name('StarWars.srt')
+	StarWars_edited.txt
+
+	>>> edited_file_name('BTTF.srt')
+	BTTF_edited.txt
+	"""
+
+	movie_clear = movie_script.replace('.srt', '')
+	return '%s' % movie_clear + '_edited.txt'
+
+def edit_all(movie_list):
+	"""
+	Edit all movie names and all movie subtitles in movie_list
+	Increase efficiency
+	"""
+
+	for movie in movie_list:
+		edit_script(movie)
+		new_name = edited_file_name(movie)
+		hist = process_script(new_name)
+
+def edit_script(movie_script):
+	"""
+	Edit original movie subtitles to extract any unnecessary text
+	Save as new file
+	"""
+
+	new_name = edited_file_name(movie_script)
+
+	original_script = file(movie_script)				# name variable to original version of movie script
+
+	new_script = open(new_name, 'w')					# open new file for writing
+	for line in original_script:						# read through original script
+		line = line.translate(None, digits)				# remove numbers (0123456789)
+		line = re.sub('<.*?>', '', line)				# remove any text between '<>', including the symbols themselves (if text on screen)
+
+
+		if '-->' not in line and 'Subtitle' not in line and '()' not in line and '^' not in line:		# further parse to reduce to scripted words only
+			new_script.write(line)						# write edited lines onto new file
+	new_script.close()									# close file
+
+def process_script(file_name):
+	"""
+	Opens edited script for further parsing
+	"""
+
+	hist = dict()
+	f1_script = open(file_name)
+	for line in f1_script:
+		process_line(line, hist)
+	return hist
+
+def process_line(line, hist):
+	"""
+	Reads words in edited script to further extract unnecessary characters
+	"""
+
+	stop = set(stopwords.words('english'))				# sets up to remove most common boring words (ex: 'the', 'at', 'me', etc)
+	line = line.replace('-', ' ')						# replace hyphens with spaces
+
+	for word in line.split():
+		word = word.strip(string.punctuation + string.whitespace)		# remove punctuation and redundant whitespace
+		word = word.lower()								# make all letters lowercase to avoid technical difficulties
+		if word not in stop:							# filter out stopwords
+			return word
+		else:
+			pass
+
+		hist[word] = hist.get(word, 0) + 1				# observe frequency of words
+
+# def most_common(hist):								# sorts hist by frequency of words (largest to smallest) instead of by word itself
+# 	t = []
+# 	for key,value in hist.items():
+# 		t.append((value,key))
+
+# 	t.sort(reverse = True)
+# 	return t
+
+# def print_most_common(hist, num=10):					# prints out most common words
+# 	t = most_common(hist)
+# 	print 'The most common words are:'
+# 	for freq,word in t[:num]:
+# 		print word, '\t', freq
+
+
+
+#edit_script('StarWars.srt')
+#hist = process_script('blahblah.txt')
+#print_most_common(hist, 30)
+
+
+movies1 = ['StarWars.srt', 'TheGodfather.srt', 'TheMatrix.srt', 'Rocky.srt', 'JurassicPark.srt', 'KillBill1.srt', 'LOTR1.srt', 'ForrestGump.srt']
+movies2 = ['Frozen.srt', 'HSM.srt', 'Mulan.srt', 'HarryPotter1.srt', 'FindingNemo.srt', 'Up.srt', 'BTTF.srt']
+edit_all(movies1)
+edit_all(movies2)

word_cloud_2.py

@@ -0,0 +1,66 @@
+"""
+Constructs word cloud from edited movie scripts/subtitles
+"""
+
+from os import path
+import matplotlib.pyplot as plt
+from movie_subtitles import edited_file_name, movies1, movies2
+
+from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
+from PIL import Image
+
+from scipy.misc import imread
+
+def make_RD_WC(movie_script, image_name):
+	"""
+	Construct rough draft of word cloud 
+	"""
+
+	d = path.dirname(__file__)
+
+	text = open(path.join(d, movie_script)).read()		# open and read movie script for most frequent words excluding stopwords
+
+	color_WC = imread(path.join(d, image_name))			# open and read image shape/color
+	wordcloud = WordCloud(background_color="white", max_words=2000, mask=color_WC, max_font_size=40, random_state=42)
+
+	wordcloud.generate(text)							# generate word cloud
+
+	image_colors = ImageColorGenerator(color_WC)		# specify font colors based on image colors
+
+	plt.imshow(wordcloud.recolor(color_func=image_colors))		# recolor word cloud from default colors to image colors
+	plt.axis("off")										# hide axis (numbers on x and y axis)
+	plt.show()											# show final word clouds
+
+def RD_WC_all(movie_list, image_list):
+	"""
+	Parse through all movie files and all image files to run code for whole list instead of one at a time
+	Increase efficiency
+	"""
+
+ 	for movie in movie_list:
+ 		i = movie_list.index(movie)						# retrieve index from movie list to retrieve correct image
+ 		image = image_list[i]							# retrieve corresponding image
+  		RD_WC = make_RD_WC(movie, image) 				# call word cloud generator
+
+def edit_all_names(movie_list):	
+	"""
+	Create new list of edited movie file names to load updated files into word cloud generator
+
+	>>> edit_all_names(['StarWars.srt', 'BTTF.srt'])
+	['StarWars_edited.txt', 'BTTF_edited.txt']
+	"""
+
+	movies_edited = []
+	for movie in movie_list:
+		m = edited_file_name(movie)
+		movies_edited.append(m)
+	return movies_edited
+
+movies_edited_1 = edit_all_names(movies1)
+movies_edited_2 = edit_all_names(movies2)
+
+images1 = ['StarWars.jpg', 'TheGodfather.jpg', 'TheMatrix.jpg', 'Rocky.jpg', 'JurassicPark.jpg', 'KillBill1.png', 'LOTR1.jpg', 'ForrestGump.jpg']
+images2 = ['Frozen.jpg', 'HSM.jpg', 'Mulan.jpg', 'HarryPotter1.jpg', 'FindingNemo.jpg', 'Up.jpg', 'BTTF.jpg']
+
+RD_WC_all(movies_edited_1, images1)
+RD_WC_all(movies_edited_2, images2)