Repository navigation
Expand file tree
/
Copy pathbot.py
More file actions
292 lines (198 loc) · 7.64 KB
/
Copy pathbot.py
File metadata and controls
292 lines (198 loc) · 7.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
"""
A Reddit bot that replies to unread messages with newly generated markov chains.
"""
import pickle
import random
import praw
import config
MODEL_FILE = "./model.pickle"
COMMENTS_LOG = "./processed_comments.txt"
# These users will be ignored to avoid errors and infinite replies.
IGNORED_USERS = ["HuachiBot", "reddit", "AutoModerator", None]
# The stop words files.
ES_STOPWORDS_FILE = "./assets/stopwords-es.txt"
EN_STOPWORDS_FILE = "./assets/stopwords-en.txt"
STOP_WORDS = set()
def add_extra_words():
"""Adds the title and uppercase version of all words to STOP_WORDS.
We parse local copies of stop words downloaded from the following repositories:
https://github.com/stopwords-iso/stopwords-es
https://github.com/stopwords-iso/stopwords-en
"""
with open(ES_STOPWORDS_FILE, "r", encoding="utf-8") as temp_file:
for word in temp_file.read().splitlines():
STOP_WORDS.add(word)
with open(EN_STOPWORDS_FILE, "r", encoding="utf-8") as temp_file:
for word in temp_file.read().splitlines():
STOP_WORDS.add(word)
extra_words = list()
for word in STOP_WORDS:
extra_words.append(word.title())
extra_words.append(word.upper())
for word in extra_words:
STOP_WORDS.add(word)
def load_log():
"""Reads the processed comments log file and creates it if it doesn't exist.
Returns
-------
list
A list of Reddit comments ids.
"""
try:
with open(COMMENTS_LOG, "r", encoding="utf-8") as log_file:
return log_file.read().splitlines()
except FileNotFoundError:
with open(COMMENTS_LOG, "a", encoding="utf-8") as log_file:
return []
def update_log(comment_id):
"""Updates the processed comments log with the given comments id.
Parameters
----------
comment_id : str
A Reddit post id.
"""
with open(COMMENTS_LOG, "a", encoding="utf-8") as log_file:
log_file.write("{}\n".format(comment_id))
def init():
"""Inits the bot by fetching the inbox and replying with newly generated comments."""
# Complete our stop words set.
add_extra_words()
# Load the model and remove prefixes that are commonly used by other bots.
model = read_model(MODEL_FILE)
for key in list(model.keys()):
if "^#" in key or "|" in key or "*****" in key or "^^" in key:
del model[key]
# We start the Reddit bot.
reddit = praw.Reddit(client_id=config.APP_ID, client_secret=config.APP_SECRET,
user_agent=config.USER_AGENT, username=config.REDDIT_USERNAME,
password=config.REDDIT_PASSWORD)
processed_comments = load_log()
for comment in reddit.inbox.all(limit=100):
if comment.author not in IGNORED_USERS and comment.id not in processed_comments:
new_comment = generate_comment(model=model, order=2,
number_of_sentences=2,
initial_prefix=get_prefix_with_context(model, comment.body))
# Small clean up when the bot uses Markdown and making sure the first letter is uppercase.
new_comment = new_comment.replace(
" > ", "\n\n > ").replace(" * ", "\n\n* ")
new_comment = new_comment[0].upper() + new_comment[1:]
if "[" not in new_comment and "]" in new_comment:
new_comment = "[" + new_comment
new_comment = new_comment.replace("U/", "u/").replace("R/", "r/")
comment.reply(new_comment)
update_log(comment.id)
print("Replied to:", comment.id)
def read_model(file_name):
"""Loads the specified pickle file.
Parameters
----------
file_name : str
The location the pickle file.
Returns
-------
dict
The dictionary inside the pickle.
"""
with open(file_name, "rb") as model_file:
return pickle.load(model_file)
def get_prefix(model_keys):
"""Get a random prefix that starts in uppercase.
Parameters
----------
model_keys : list
A list containing all the model keys.
Returns
-------
str
The randomly selected prefix.
"""
# We give it a maximum of 10,000 tries.
for _ in range(10000):
random_prefix = random.choice(model_keys)
if random_prefix[0].isupper():
ends_with_punctuation = False
stripped_suffix = random_prefix.strip()
for char in [".", "?", "!"]:
if stripped_suffix[-1] == char:
ends_with_punctuation = True
break
if not ends_with_punctuation:
break
return random_prefix
def get_prefix_with_context(model, context):
"""Get a random prefix that matches the given context.
Parameters
----------
model : dict
The dictionary containing all the pairs and their possible outcomes.
context : str
A sentence which will be separated into keywords.
Returns
-------
str
The randomly selected context-aware prefix.
"""
model_keys = list(model.keys())
# Some light cleanup.
context = context.replace("?", "").replace("!", "").replace(".", "")
context_keywords = list(set(context.split()))
# We remove stop words from the context.
# We use reversed() to remove items from the list without affecting the sequence.
for word in reversed(context_keywords):
if len(word) <= 3 or word in STOP_WORDS:
context_keywords.remove(word)
# If our context has no keywords left we return a random prefix.
if len(context_keywords) == 0:
return get_prefix(model_keys)
# We are going to sample one prefix for each available keyword and return only one.
random.shuffle(model_keys)
sampled_prefixes = list()
for word in context_keywords:
for prefix in model_keys:
if word in prefix or word.lower() in prefix or word.title() in prefix:
sampled_prefixes.append(prefix)
break
# If we don't get any samples we fallback to the random prefix method.
if len(sampled_prefixes) == 0:
return get_prefix(model_keys)
else:
return random.choice(sampled_prefixes)
def generate_comment(model, number_of_sentences, initial_prefix, order):
"""Generates a new comment using the model and an initial prefix.
Parameters
----------
model : dict
The dictionary containing all the pairs and their possible outcomes.
number_of_Sentences : int
The maximum number of sentences.
initial_prefix : str
The word(s) that will start the chain.
order : int
The number of words in the state, this must match the order number in step2.py
Returns
-------
str
The newly generated text.
"""
counter = 0
model_keys = list(model.keys())
latest_suffix = initial_prefix
final_sentence = latest_suffix + " "
# We add a maximum sentence length to avoid going infinite in edge cases.
for _ in range(500):
try:
latest_suffix = random.choice(model[latest_suffix])
except:
# If we don't get another word we take another one randomly and continue the chain.
latest_suffix = get_prefix(model_keys)
final_sentence += latest_suffix + " "
latest_suffix = " ".join(final_sentence.split()[-order:]).strip()
for char in [".", "?", "!"]:
if latest_suffix[-1] == char:
counter += 1
break
if counter >= number_of_sentences:
break
return final_sentence
if __name__ == "__main__":
init()