From e421c73811eeb519291f2c0e12ffb9038d191117 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 18:26:30 +0100 Subject: [PATCH 01/10] PEP8 --- twitter_follow_bot.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/twitter_follow_bot.py b/twitter_follow_bot.py index 53c8e33..73d9d61 100644 --- a/twitter_follow_bot.py +++ b/twitter_follow_bot.py @@ -5,15 +5,16 @@ The Twitter Follow Bot library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) any -later version. +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. -The Twitter Follow Bot library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +The Twitter Follow Bot library is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +License for more details. -You should have received a copy of the GNU General Public License along with the Twitter -Follow Bot library. If not, see http://www.gnu.org/licenses/. +You should have received a copy of the GNU General Public License along with +the Twitter Follow Bot library. If not, see http://www.gnu.org/licenses/. """ from twitter import Twitter, OAuth, TwitterHTTPError @@ -26,8 +27,8 @@ CONSUMER_SECRET = "" TWITTER_HANDLE = "" -# put the full path and file name of the file you want to store your "already followed" -# list in +# put the full path and file name of the file you want to store your "already +# followed" list in ALREADY_FOLLOWED_FILE = "already-followed.csv" t = Twitter(auth=OAuth(OAUTH_TOKEN, OAUTH_SECRET, @@ -36,7 +37,7 @@ def search_tweets(q, count=100, result_type="recent"): """ - Returns a list of tweets matching a certain phrase (hashtag, word, etc.) + Returns a list of tweets matching a certain phrase (hashtag, word, etc.) """ return t.search.tweets(q=q, result_type=result_type, count=count) From 6d0933bf3674f51bccfbc25cb4f6d3ad4e1d2880 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 18:29:12 +0100 Subject: [PATCH 02/10] Refactor auto_follow --- twitter_follow_bot.py | 51 ++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/twitter_follow_bot.py b/twitter_follow_bot.py index 73d9d61..686b632 100644 --- a/twitter_follow_bot.py +++ b/twitter_follow_bot.py @@ -17,6 +17,8 @@ the Twitter Follow Bot library. If not, see http://www.gnu.org/licenses/. """ +from __future__ import unicode_literals + from twitter import Twitter, OAuth, TwitterHTTPError import os @@ -91,41 +93,30 @@ def auto_follow(q, count=100, result_type="recent"): """ result = search_tweets(q, count, result_type) - following = set(t.friends.ids(screen_name=TWITTER_HANDLE)["ids"]) - # make sure the "already followed" file exists - if not os.path.isfile(ALREADY_FOLLOWED_FILE): - with open(ALREADY_FOLLOWED_FILE, "w") as out_file: - out_file.write("") - - # read in the list of user IDs that the bot has already followed in the - # past - do_not_follow = set() - dnf_list = [] - with open(ALREADY_FOLLOWED_FILE) as in_file: - for line in in_file: - dnf_list.append(int(line)) - - do_not_follow.update(set(dnf_list)) - del dnf_list + to_follow = set() for tweet in result["statuses"]: + if tweet["user"]["screen_name"] == TWITTER_HANDLE: + continue + print('@{}:\n{}\n'.format( + tweet['user']['screen_name'], + tweet['text'])) + to_follow.add(tweet['user']['id']) + + already_following = set(t.friends.ids(screen_name=TWITTER_HANDLE)["ids"]) + + to_follow -= already_following + print("Following {} users".format(len(to_follow))) + for twitter_id in to_follow: + print(twitter_id) try: - if (tweet["user"]["screen_name"] != TWITTER_HANDLE and - tweet["user"]["id"] not in following and - tweet["user"]["id"] not in do_not_follow): - - t.friendships.create(user_id=tweet["user"]["id"], follow=True) - following.update(set([tweet["user"]["id"]])) - - print("followed %s" % (tweet["user"]["screen_name"])) - + t.friendships.create(user_id=twitter_id, follow=True) except TwitterHTTPError as e: - print("error: %s" % (str(e))) - - # quit on error unless it's because someone blocked me - if "blocked" not in str(e).lower(): - quit() + if 'blocked' not in str(e).lower(): # ignore block errors + print(repr(e)) + else: + raise def auto_follow_followers(): From abebb0a8f6f1b9341e717f2a7794ade0cde5c677 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 18:29:57 +0100 Subject: [PATCH 03/10] Add .gitignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d18402d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +.*.swp From 1bd2f7700248629c57cdf4eaaa446b0976fef3e3 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 18:30:11 +0100 Subject: [PATCH 04/10] Add requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eed830d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +twitter==1.14.3 From 002db64d3c0c4bf63b2e7f4863c6dbc459afdd08 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 19:33:46 +0100 Subject: [PATCH 05/10] Add scheduling for auto-follow Define some dates and keywords in schedule.csv then run main periodically. --- .gitignore | 1 + main.py | 38 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + schedule.csv.example | 2 ++ 4 files changed, 42 insertions(+) create mode 100755 main.py create mode 100644 schedule.csv.example diff --git a/.gitignore b/.gitignore index d18402d..d51e922 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc .*.swp +/schedule.csv diff --git a/main.py b/main.py new file mode 100755 index 0000000..ee6d41e --- /dev/null +++ b/main.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import csv +import datetime +import codecs +import sys + +from twitter_follow_bot import auto_follow + + +def job_valid_now(job): + start = parse_date(job['start_date']) + end = parse_date(job['end_date']) + return start <= datetime.date.today() <= end + + +def parse_date(date_string): + return datetime.datetime.strptime( + date_string, '%Y-%m-%d').date() + + +def run_job(job): + print("Running {}".format(job)) + + auto_follow_query = job.get('auto_follow') + if auto_follow_query is not None: + auto_follow(auto_follow_query) + + +def main(): + with open('schedule.csv', 'r') as f: + for row in csv.DictReader(f): + if job_valid_now(row): + run_job(row) + +if __name__ == '__main__': + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + main() diff --git a/requirements.txt b/requirements.txt index eed830d..4faa106 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ twitter==1.14.3 +pytz==2014.7 diff --git a/schedule.csv.example b/schedule.csv.example new file mode 100644 index 0000000..caa74b3 --- /dev/null +++ b/schedule.csv.example @@ -0,0 +1,2 @@ +start_date,end_date,auto_follow +2014-09-03,2014-09-04,"@SomeEvent excited" From 1a044d8d231fc2ede890edad5d62a69b693ccc37 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 19:39:51 +0100 Subject: [PATCH 06/10] Add run.sh script to wrap main.py for cron --- .gitignore | 2 ++ run.sh | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100755 run.sh diff --git a/.gitignore b/.gitignore index d51e922..6db1294 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc .*.swp /schedule.csv +/log/* +/venv diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..e1a629f --- /dev/null +++ b/run.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +THIS_SCRIPT=$(readlink -f $0) +THIS_DIR=$(dirname ${THIS_SCRIPT}) + +DATE_NOW=$(date +%Y-%m-%d_%H-%M-%S) +LOG_DIR=${THIS_DIR}/log + +function setup_log_output { + mkdir -p ${LOG_DIR} + LOG_FILE=${LOG_DIR}/${DATE_NOW}.log + ln -sf ${LOG_FILE} ${LOG_DIR}/latest +} + +function setup_virtualenv { + VENV_DIR=${THIS_DIR}/venv + + if [ ! -d "${VENV_DIR}" ]; then + if [ -s "${THIS_DIR}/.python_version" ]; then + virtualenv ${VENV_DIR} -p "$(cat ${THIS_DIR}/.python_version)" >> ${LOG_FILE} + else + virtualenv ${VENV_DIR} >> ${LOG_FILE} + fi + fi + source ${THIS_DIR}/venv/bin/activate +} + +function install_dependencies { + pip install -r ${THIS_DIR}/requirements.txt >> ${LOG_FILE} +} + +function source_settings_and_credentials { + for ENV_FILE in settings.sh credentials.sh + do + if [ -s "${THIS_DIR}/${ENV_FILE}" ]; then + source "${THIS_DIR}/${ENV_FILE}" + fi + done +} + +function delete_old_logs { + find ${LOG_DIR} -type f -iname '*.log' -mtime +30 -delete +} + +function run_main_code { + export PYTHONIOENCODING="utf-8" + command=${THIS_DIR}/main.py + # Line buffering, see http://unix.stackexchange.com/a/25378 + stdbuf -oL -eL run-one ${command} >> ${LOG_FILE} 2>&1 + RETCODE=$? + if [ ${RETCODE} != 0 ]; then + echo "$@ exited with code: ${RETCODE}" + git remote -v + tail -v -n 100 ${LOG_FILE} + exit 2 + fi + + grep -n '^ERROR:' ${LOG_FILE} +} + +setup_log_output +setup_virtualenv +install_dependencies +source_settings_and_credentials +run_main_code +delete_old_logs From 8a9d10c0af71c681837889a3dd2f61def095fbaa Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 19:47:27 +0100 Subject: [PATCH 07/10] Load tokens from credentials.sh, not python file --- .gitignore | 1 + credentials.sh.example | 9 +++++++++ twitter_follow_bot.py | 15 +++++++-------- 3 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 credentials.sh.example diff --git a/.gitignore b/.gitignore index 6db1294..8dd5aaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp /schedule.csv +/credentials.sh /log/* /venv diff --git a/credentials.sh.example b/credentials.sh.example new file mode 100644 index 0000000..5dacd11 --- /dev/null +++ b/credentials.sh.example @@ -0,0 +1,9 @@ +export TWITTER_HANDLE="MyScreenName" + +# Go to https://apps.twitter.com/ and create a new app with read/write +# access, then generate a user access token. + +export API_KEY='this comes from your app' +export API_SECRET='this comes from your app' +export ACCESS_TOKEN='this is for your user' +export ACCESS_TOKEN_SECRET='this is for your user' diff --git a/twitter_follow_bot.py b/twitter_follow_bot.py index 686b632..83c4118 100644 --- a/twitter_follow_bot.py +++ b/twitter_follow_bot.py @@ -22,19 +22,18 @@ from twitter import Twitter, OAuth, TwitterHTTPError import os -# put your tokens, keys, secrets, and Twitter handle in the following variables -OAUTH_TOKEN = "" -OAUTH_SECRET = "" -CONSUMER_KEY = "" -CONSUMER_SECRET = "" -TWITTER_HANDLE = "" +API_KEY = os.environ['API_KEY'] +API_SECRET = os.environ['API_SECRET'] +ACCESS_TOKEN = os.environ['ACCESS_TOKEN'] +ACCESS_TOKEN_SECRET = os.environ['ACCESS_TOKEN_SECRET'] +TWITTER_HANDLE = os.environ['TWITTER_HANDLE'] # put the full path and file name of the file you want to store your "already # followed" list in ALREADY_FOLLOWED_FILE = "already-followed.csv" -t = Twitter(auth=OAuth(OAUTH_TOKEN, OAUTH_SECRET, - CONSUMER_KEY, CONSUMER_SECRET)) +t = Twitter(auth=OAuth(ACCESS_TOKEN, ACCESS_TOKEN_SECRET, + API_KEY, API_SECRET)) def search_tweets(q, count=100, result_type="recent"): From ab6bf08cfe6e356faec3754f45c80d0e0d9da569 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Wed, 3 Sep 2014 19:55:25 +0100 Subject: [PATCH 08/10] Always load schedule.csv from directory of main.py --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index ee6d41e..884ddfc 100755 --- a/main.py +++ b/main.py @@ -5,6 +5,8 @@ import codecs import sys +from os.path import dirname, join as pjoin + from twitter_follow_bot import auto_follow @@ -28,7 +30,7 @@ def run_job(job): def main(): - with open('schedule.csv', 'r') as f: + with open(pjoin(dirname(__file__), 'schedule.csv'), 'r') as f: for row in csv.DictReader(f): if job_valid_now(row): run_job(row) From 9ce5380217779c36ebe5f0c878a136bc93512768 Mon Sep 17 00:00:00 2001 From: Paul M Furley Date: Thu, 9 Oct 2014 15:41:24 +0100 Subject: [PATCH 09/10] Add a log of who we've followed / unfollowed It's a record of who we automatically followed and when. This allows us to automatically unfollow after a certain period, if required, and avoid pestering them again in the future. Stored as a plain old CSV. --- twitter_follow_bot.py | 126 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 9 deletions(-) diff --git a/twitter_follow_bot.py b/twitter_follow_bot.py index 83c4118..605371b 100644 --- a/twitter_follow_bot.py +++ b/twitter_follow_bot.py @@ -19,8 +19,18 @@ from __future__ import unicode_literals +import csv +import datetime + +from collections import OrderedDict + from twitter import Twitter, OAuth, TwitterHTTPError import os +from os.path import dirname, join as pjoin + +import logging +logger = logging.getLogger(__name__) + API_KEY = os.environ['API_KEY'] API_SECRET = os.environ['API_SECRET'] @@ -28,6 +38,8 @@ ACCESS_TOKEN_SECRET = os.environ['ACCESS_TOKEN_SECRET'] TWITTER_HANDLE = os.environ['TWITTER_HANDLE'] +DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + # put the full path and file name of the file you want to store your "already # followed" list in ALREADY_FOLLOWED_FILE = "already-followed.csv" @@ -36,6 +48,92 @@ API_KEY, API_SECRET)) +class FollowLog(object): + + FOLLOW_LOG = pjoin(dirname(__file__), 'follow_log.csv') + FOLLOW_LOG_FIELDS = ('twitter_id', 'screen_name', 'follow_datetime', + 'unfollow_datetime', 'follow_reason') + + def __init__(self): + self._following = OrderedDict() + + @staticmethod + def _empty_row(twitter_id): + row = {k: None for k in FollowLog.FOLLOW_LOG_FIELDS} + row['twitter_id'] = twitter_id + return row + + @staticmethod + def _serialize_row(row): + row['follow_datetime'] = row['follow_datetime'].strftime( + DATETIME_FORMAT) + return row + + @staticmethod + def _deserialize_row(row): + row['follow_datetime'] = datetime.datetime.strptime( + row['follow_datetime'], DATETIME_FORMAT) + return row + + def __enter__(self): + self._load_from_csv() + return self + + def __exit__(self, exception_type, exception_value, traceback): + if exception_type is None: + self._save_to_csv() + + def _load_from_csv(self): + if not os.path.exists(self.FOLLOW_LOG): + self._following = OrderedDict() + return + + with open(self.FOLLOW_LOG, 'r') as f: + self._following = OrderedDict( + [(int(row['twitter_id']), self._deserialize_row(row)) + for row in csv.DictReader(f)]) + + def _save_to_csv(self): + tmp_filename = self.FOLLOW_LOG + '.tmp' + with open(tmp_filename, 'w') as f: + writer = csv.DictWriter(f, self.FOLLOW_LOG_FIELDS) + writer.writeheader() + for twitter_id, row in self._following.items(): + row['twitter_id'] = twitter_id + writer.writerow(self._serialize_row(row)) + + os.rename(tmp_filename, self.FOLLOW_LOG) + + def _get_or_create(self, twitter_id): + if twitter_id not in self._following: + self._following[twitter_id] = self._empty_row(twitter_id) + return self._following[twitter_id] + + def save_follow(self, twitter_id, reason=None): + entry = self._get_or_create(twitter_id) + + entry['follow_datetime'] = datetime.datetime.now() + entry['follow_reason'] = reason + + def save_unfollow(self, twitter_id): + entry = self._get_or_create(twitter_id) + entry['unfollow_datetime'] = datetime.datetime.now() + + def have_followed_before(self, twitter_id): + entry = self._following.get(twitter_id) + if entry is None: # no record of this twitter id. + return False + + if entry['follow_datetime'] is not None: + return True + + return False + + +def get_follow_log(): + return FollowLog() + + def search_tweets(q, count=100, result_type="recent"): """ Returns a list of tweets matching a certain phrase (hashtag, word, etc.) @@ -107,15 +205,10 @@ def auto_follow(q, count=100, result_type="recent"): to_follow -= already_following print("Following {} users".format(len(to_follow))) - for twitter_id in to_follow: - print(twitter_id) - try: - t.friendships.create(user_id=twitter_id, follow=True) - except TwitterHTTPError as e: - if 'blocked' not in str(e).lower(): # ignore block errors - print(repr(e)) - else: - raise + with get_follow_log() as follow_log: + for twitter_id in to_follow: + _follow(follow_log, twitter_id, + 'Tweet: `{}`'.format(tweet['text'])) def auto_follow_followers(): @@ -172,3 +265,18 @@ def auto_unfollow_nonfollowers(): if user_id not in users_keep_following: t.friendships.destroy(user_id=user_id) print("unfollowed %d" % (user_id)) + + +def _follow(follow_log, twitter_id, reason=None): + print(twitter_id) + try: + t.friendships.create(user_id=twitter_id, follow=True) + except TwitterHTTPError as e: + if 'blocked' in str(e).lower(): # ignore block errors + # details: {"errors":[{"code":162, + # "message":"You have been blocked from following this account + # at the request of the user."}]} + logging.info('Ignoring "blocked" exception') + logging.exception(e) + else: + follow_log.save_follow(twitter_id, reason) From 3412ea3711ea84a217d123708ed38767b05be70f Mon Sep 17 00:00:00 2001 From: "requires.io" Date: Thu, 22 Jan 2015 11:34:43 +0000 Subject: [PATCH 10/10] [requires.io] dependency update --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4faa106..c7ce879 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -twitter==1.14.3 -pytz==2014.7 +twitter==1.15.0 +pytz==2014.10