Instantly share code, notes, and snippets.

israelshirk / aconnection.py
Last active 4 years ago

Star

Code Revisions 2 Stars 2

Boto SES + Python Twisted

aconnection.py

1 # Overrides parts of Boto to provide async call/response. Derived

2 # SESConnection and boto/connection.
3
4 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
5 # Copyright (c) 2011 Harry Marr http://hmarr.com/
6 #
7 # Permission is hereby granted, free of charge, to any person obta
8 # copy of this software and associated documentation files (the
9 # "Software"), to deal in the Software without restriction, includ
10 # without limitation the rights to use, copy, modify, merge, publi
11 # tribute, sublicense, and/or sell copies of the Software, and to
12 # persons to whom the Software is furnished to do so, subject to t
13 # lowing conditions:
14 #
15 # The above copyright notice and this permission notice shall be i
16 # in all copies or substantial portions of the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCH
20 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
21 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABI
22 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FRO
23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DE
24 # IN THE SOFTWARE.
25 import re
26 import urllib
27 import base64
28
29 from boto.connection import AWSAuthConnection, HTTPRequest, HTTPRe
30 from boto.exception import BotoServerError
31 from boto.regioninfo import RegionInfo
32 import boto
33 import boto.jsonresponse
34 from boto.ses import exceptions as ses_exceptions
35 from boto.ses.connection import *
36 from urllib import urlencode
37
38 from pprint import pprint
39
40 from twisted.internet import reactor
41 from twisted.internet.defer import Deferred, DeferredList, succeed
42 from twisted.internet.protocol import Protocol
43 from twisted.web.client import Agent, HTTPConnectionPool
44 from twisted.web.http_headers import Headers
45 from twisted.web.iweb import IBodyProducer
46 from zope.interface import implements
47
48 class StringProducer(object):
49 implements(IBodyProducer)
50
51 def __init__(self, body):
52 self.body = body
53 self.length = len(body)
54
55 def startProducing(self, consumer):
56 consumer.write(self.body)
57 return succeed(None)
58
59 def pauseProducing(self):
60 pass
61
62 def stopProducing(self):
63 pass
64
 65 class ResponseReceiver(Protocol):
66 def __init__(self, finished, callback, response, data):

67 self.finished = finished
68 self.body = ""
69 self.callback = callback
70 self.response = response
71 self.data = data
72
73 def dataReceived(self, bytes):
74 self.body += bytes
75
76 def connectionLost(self, reason):
77 # print 'Finished receiving body:', reason.getErrorMessage
78
79 self.callback(self.response, self.body, self.data)
80
81 self.finished.callback(None)
82
83 class ASESConnection(SESConnection):
84 ResponseError = BotoServerError
85 DefaultRegionName = 'us-east-1'
86 DefaultRegionEndpoint = 'email.us-east-1.amazonaws.com'
87 # DefaultRegionEndpoint = 'coast.dev'
88 APIVersion = '2010-12-01'
89
90 host_override = None
91
92 def __init__(self, aws_access_key_id=None, aws_secret_access_k
93 is_secure=True, port=None, proxy=None, proxy_port
94 proxy_user=None, proxy_pass=None, debug=0,
95 https_connection_factory=None, region=None, path=
96 security_token=None, validate_certs=True, agent=N
97 if not region:
98 region = RegionInfo(self, self.DefaultRegionName,
99 self.DefaultRegionEndpoint)
100 self.region = region
101 SESConnection.__init__(self,
102 aws_access_key_id=aws_access_ke
103 is_secure=is_secure, port=port,
104 proxy_user=proxy_user, proxy_pa
105 https_connection_factory=None,
106 security_token=security_token,
107 validate_certs=validate_certs)

108 self.agent = agent

109 self.callback = callback
110 self.errback = errback
111 self.reactor = reactor
112
113 def _make_request(self, action, params=None):
114 """Make a call to the SES API.
115
116 :type action: string
117 :param action: The API method to use (e.g. SendRawEmail)
118
119 :type params: dict
120 :param params: Parameters that will be sent as POST data w
121 call.
122 """
123 ct = 'application/x-www-form-urlencoded; charset=UTF-8'
124 headers = {'Content-Type': ct}
125 params = params or {}
126 params['Action'] = action
127
128 for k, v in params.items():
129 if isinstance(v, unicode): # UTF-8 encode only if it'
130 params[k] = v.encode('utf-8')
131
132 self.make_request(
133 'POST',
134 '/',
135 headers=headers,
136 data=urllib.urlencode(params)
137 )
138
139 def make_request(self, method, path, headers=None, data='', ho
140 auth_path=None, sender=None, override_num_ret
141 params=None):
142 """Makes a request to the server, with stock multiple-retr
143 if params is None:
144 params = {}
145
146 if self.host_override != None:
147 host = self.host_override
148

149 self.http_request = self.build_base_http_request(method, p

150 params, header
151
152 self.http_request.authorize(connection=self)
153
154 bodyProducer = StringProducer(self.http_request.body)
155
156 url = self.http_request.protocol + '://' + self.http_reque
157
158 url += self.http_request.path
159
160 # Twisted sets this again - doing it twice results in a 40
161 del(self.http_request.headers['Content-Length'])
162
163 for (title, value) in self.http_request.headers.iteritems(
164 if type(value) == type("string"):
165 value = [ value ]
166
167 self.http_request.headers[title] = value
168
169 # print self.http_request.method, url, Headers(self.http_r
170
171 d = self.agent.request(
172 self.http_request.method,
173 url,
174 Headers(self.http_request.headers),
175 bodyProducer=bodyProducer
176 )
177
178 d.addCallback(self.http_request_callback, data)
179 d.addErrback(self.http_request_errback, data)
180
181 def http_request_callback(self, response, data):
182 finished = Deferred()
183
184 response.deliverBody(ResponseReceiver(finished, self.http_
185
186 return finished
187
188 def http_body_received_callback(self, response, body, data):
189 if response.code == 200:
190 list_markers = ('VerifiedEmailAddresses', 'Identities'
191 'VerificationAttributes', 'SendDataPoi
192 item_markers = ('member', 'item', 'entry')
193
194 try:
195 e = boto.jsonresponse.Element(list_marker=list_mar
196 item_marker=item_mar
197 h = boto.jsonresponse.XmlHandler(e, None)
198 h.parse(body)
199 self.callback( e, data )
200
201 return
202 except Exception, e:
203 # print e
204 self.errback( self._handle_error(response, body),
205
206 return
207 else:
208 # HTTP codes other than 200 are considered errors. Go
209 # some error handling to determine which exception get
210 self.errback( self._handle_error(response, body), data
211
212 return
213
214 def http_request_errback(self, failure, data):
215 print failure, data
216 self.errback( "HTTP Error", failure )
217
218 def _handle_error(self, response, body):
219 """
220 Handle raising the correct exception, depending on the err
221 errors share the same HTTP response code, meaning we have
222 kludgey and do string searches to figure out what went wro
223 """
224
225 if "Address blacklisted." in body:
226 # Delivery failures happened frequently enough with th
227 # email address for Amazon to blacklist it. After a da
228 # they'll be automatically removed, and delivery can b
229 # again (if you write the code to do so in your applic
230 ExceptionToRaise = ses_exceptions.SESAddressBlackliste
231 exc_reason = "Address blacklisted."

232 elif "Email address is not verified." in body:

233 # This error happens when the "Reply-To" value passed
234 # send_email() hasn't been verified yet.
235 ExceptionToRaise = ses_exceptions.SESAddressNotVerifie
236 exc_reason = "Email address is not verified."
237 elif "Daily message quota exceeded." in body:
238 # Encountered when your account exceeds the maximum to
239 # of emails per 24 hours.
240 ExceptionToRaise = ses_exceptions.SESDailyQuotaExceede
241 exc_reason = "Daily message quota exceeded."
242 elif "Maximum sending rate exceeded." in body:
243 # Your account has sent above its allowed requests a s
244 ExceptionToRaise = ses_exceptions.SESMaxSendingRateExc
245 exc_reason = "Maximum sending rate exceeded."
246 elif "Domain ends with dot." in body:
247 # Recipient address ends with a dot/period. This is in
248 ExceptionToRaise = ses_exceptions.SESDomainEndsWithDot
249 exc_reason = "Domain ends with dot."
250 elif "Local address contains control or whitespace" in bod
251 # I think this pertains to the recipient address.
252 ExceptionToRaise = ses_exceptions.SESLocalAddressChara
253 exc_reason = "Local address contains control or whites
254 elif "Illegal address" in body:
255 # A clearly mal-formed address.
256 ExceptionToRaise = ses_exceptions.SESIllegalAddressErr
257 exc_reason = "Illegal address"
258 # The re.search is to distinguish from the
259 # SESAddressNotVerifiedError error above.
260 elif re.search('Identity.*is not verified', body):
261 ExceptionToRaise = ses_exceptions.SESIdentityNotVerifi
262 exc_reason = "Identity is not verified."
263 elif "ownership not confirmed" in body:
264 ExceptionToRaise = ses_exceptions.SESDomainNotConfirme
265 exc_reason = "Domain ownership is not confirmed."
266 else:
267 # This is either a common AWS error, or one that we do
268 # its own exception to.
269 ExceptionToRaise = self.ResponseError
270 exc_reason = "unknown"
271
272 return ExceptionToRaise(response.code, exc_reason, body)

config.json

1
2 {
3 "database_host": "127.0.0.1",
4 "database_user": "sadlfkjasdlfkj",
5 "database_passwd": "asldkfjasdlkfj",
6 "database_db": "laskdjflaskdjf",
7
8 "rabbitmq_host": "127.0.0.1",
9 "rabbitmq_port": 5672,
10 "rabbitmq_vhost": "/",
11 "rabbitmq_username": "guest",
12 "rabbitmq_password": "guest",
13 "rabbitmq_queue": "queue",
14 "rabbitmq_exchange": "exchange",
15 "rabbitmq_routing_key": "routing_key",
16 "rabbitmq_rate_limit": 3
17 }

send_emails.py

1
2 from aconnection import ASESConnection
3 from pprint import pprint
4 from twisted.enterprise import adbapi
5 from twisted.internet import reactor, defer
6 from twisted.internet.defer import Deferred
7 from twisted.internet.ssl import ClientContextFactory
8 from twisted.web.client import Agent, HTTPConnectionPool
9 from twisted_queue import *
10 import functools
11 import json
12 import MySQLdb, MySQLdb.cursors
13 import re
14 import sys
15 import time
16 import traceback
17 import twisted.internet.task
18
19 # This class is used to enable SSL... Seems to just extend getCon
20 # accept extra arguments that HTTP might require.
21
22
23 class WebClientContextFactory(ClientContextFactory):
24 def getContext(self, hostname, port):
25 return ClientContextFactory.getContext(self)
26
27 class EmailSender:
28
29 def __init__(self, reactor, email_id):
30 contextFactory = WebClientContextFactory()
31
32 self.sent_count = 0
33 self.last_sent_count = 0
34 self.failed_count = 0
35 self.last_failed_count = 0
36
37 self.email_id = email_id
38 self.reactor = reactor
39
40 pool = HTTPConnectionPool(reactor, persistent=True
41 pool.maxPersistentPerHost = 50
42 pool.cachedConnectionTimeout = 10
43 https_agent = Agent(reactor, contextFactory, pool=
44
45 self.set_up_configuration()
46
47 self.db_pool = adbapi.ConnectionPool(
48 "MySQLdb",
49 host=self.configuration['database_host'],
50 user=self.configuration['database_user'],
51 passwd=self.configuration['database_passwd
52 db=self.configuration['database_db'],
53 cursorclass=MySQLdb.cursors.DictCursor,
54 )

55
56 # This probably should just get activated with a -
57 self.ses_connection = ASESConnection(
58 callback=self.sending_email_succeeded,
59 errback=self.boto_errback,
60 reactor=reactor,
61 agent=https_agent
62 )
63
64 if 'production' in self.configuration and self.con
65 print "WARNING: RUNNING PRODUCTION SEND.
66 time.sleep(10)
67 else:
68 self.ses_connection.host_override = 'ses.v
69
70 self.reactor.callWhenRunning(self.go)
71
72 def set_up_configuration(self):
73 try:
74 configuration = json.loads(open('config.js
75 except:
76 print "Failed to load manual configuration
77 configuration = {}
78
79 default_configuration = {
80 "database_host": "127.0.0.1",
81 "database_user": "asldkfjasldkfj",
82 "database_passwd": "asldkfjasldkfj",
83 "database_db": "asldkfjasldkfjasdlfkj",
84
85 "rabbitmq_host": "127.0.0.1",
86 "rabbitmq_port": 5672,
87 "rabbitmq_vhost": "/",
88 "rabbitmq_username": "guest",
89 "rabbitmq_password": "guest",
90 "rabbitmq_queue": "huge_email_address_queu
91 "rabbitmq_exchange": "huge_email_address_q
92 "rabbitmq_routing_key": "huge_email_addres
93 "rabbitmq_rate_limit": 1,
 94
95 "source_address": '"MOJO Themes" <noreply@
96
97 "reporting_interval": 1
98 }
99
100 self.configuration = default_configuration
101
102 for (k, v) in configuration.items():
103 self.configuration[k] = v
104
105 pprint({'Configuration': self.configuration})
106
107 def go(self):
108 self.load_email_content()
109 self.send_emails()
110
111 @defer.inlineCallbacks
112 def load_email_content(self):
113 templates = yield self.db_pool.runQuery(
114 "SELECT subject, body FROM email_content W
115 (self.email_id,)
116 )
117
118 self.email_templates = []
119
120 if not templates:
121 self.reactor.stop()
122 print "No emails loaded"
123 sys.exit(1)
124
125 try:
126 for template in templates:
127 self.load_template(template)
128 except Exception, e:
129 print e
130 self.reactor.stop()
131 sys.exit(1)
132
133 if not self.email_templates:
134 print "No templates loaded!"
135 self.reactor.stop()
136 sys.exit(1)
137

138 def load_template(self, template):

139 self.email_templates = [{
140 'subject': template['subject'],
141 'body': template['body']
142 }]
143
144 def get_template(self, **argv):
145 return self.email_templates[0]
146
147 def apply_template(self, template, receiver_info):
148 subject = template['subject']
149 body = template['body']
150
151 if 'host_account_id' in receiver_info:
152 host_account_id = receiver_info['host_acco
153 else:
154 host_account_id = "null"
155
156 user_id = receiver_info['user_id']
157
158 try:
159 # import pdb; pdb.set_trace()
160 body = body.replace('{{host_account_id}}',
161 body = body.replace('{{user_id}}', str(use
162 except Exception, e:
163 print "failed", sys.exc_info()[2]
164 traceback.print_tb(sys.exc_info()[2])
165 raise e
166
167 return {
168 'subject': subject,
169 'content': body,
170 'source': self.configuration['source_addre
171 }
172
173 def send_emails(self):
174 stats_loop = twisted.internet.task.LoopingCall(sel
175 stats_loop.start(self.configuration['reporting_int
176
177 self.message_queue = twisted_queue_receiver(
178 callback=self.send_email,

179 errback=self.queue_errback,
180 host=self.configuration['rabbitmq_host'],
181 port=self.configuration['rabbitmq_port'],
182 vhost=self.configuration['rabbitmq_vhost']
183 username=self.configuration['rabbitmq_user
184 password=self.configuration['rabbitmq_pass
185 queue=self.configuration['rabbitmq_queue']
186 exchange=self.configuration['rabbitmq_exch
187 routing_key=self.configuration['rabbitmq_r
188 rate_limit=self.configuration['rabbitmq_ra
189 no_ack=False,
190 durable=True
191 )
192
193 def queue_errback(self, err):
194 print err
195 print "I should probably be killed because I am a
196 reactor.stop()
197 sys.exit(1)
198
199 @defer.inlineCallbacks
200 def send_email(self, channel, raw_message):
201 if not (channel or raw_message):
202 defer.returnValue(None)
203
204 try:
205 receiver_info = json.loads(raw_message.con
206 except ValueError, e:
207 channel.basic_ack(delivery_tag=raw_message
208
209 print "failed parsing", raw_message.conten
210 self.sending_email_failed(e, raw_message.c
211
212 defer.returnValue(None)
213
214 try:
215 template = self.get_template(receiver_info
216 except Exception, e:
217 channel.basic_ack(delivery_tag=raw_message
218
219 print "Missing email template:", receiver_
220
221 self.sending_email_failed(e, raw_message.c
222 defer.returnValue(None)
223
224 try:
225 templated_email = self.apply_template(temp
226 except ValueError, e:
227 channel.basic_ack(delivery_tag=raw_message
228 print "Couldn't template email:", receiver
229
230 self.sending_email_failed(e, raw_message.c
231 defer.returnValue(None)
232
233 email_subject = templated_email['subject']
234 email_content = templated_email['content']
235 email_source = templated_email['source']
236
237 try:
238 self.ses_connection.send_email(
239 source=email_source, subject=email
240 format='html', to_addresses=[recei
241 except Exception, e:
242 channel.basic_ack(delivery_tag=raw_message
243 print "Send_email exception:", e
244
245 print receiver_info['email']
246
247 self.sending_email_failed(e, raw_message.c
248 defer.returnValue(None)
249
250 try:
251 yield channel.basic_ack(delivery_tag=raw_m
252 except Exception, e:
253 print "Couldn't ack message because ???",
254 defer.returnValue(None)
255
256 def print_stats(self):
257 diff_sent_count = self.sent_count - self.last_sent
258 diff_failed_count = self.failed_count - self.last_
259
260 self.last_sent_count = self.sent_count
261 self.last_failed_count = self.failed_count

262
263 print
264 print "sent_count:", self.sent_count, "period rate
265 print "failed_count:", self.failed_count, "period
266 sys.stdout.flush()
267
268 def sending_email_succeeded(self, e, data):
269 # sys.stdout.write('.')
270
271 # Could stuff into a database here if desired
272
273 self.sent_count += 1
274
275 def boto_errback(self, data, error):
276 print "failed sending:", error, data
277
278 self.failed_count += 1
279
280 def sending_email_failed(self, e, data):
281
282 # Could stuff into a database or something else if
283 print "failed sending to", data, "because", e
284
285 self.failed_count += 1
286
287 if __name__ == '__main__':
288 sender = EmailSender(reactor, sys.argv[1])
289
290 reactor.run()

twisted_queue.py

1 # This adds the cloudfront servers' ip addresses into Memcache.

2 # Memcache Key: cloudfront_server_addresses
3 # Format:
 4 # {
5 # 'subnet': ['server', 'server', 'server', 'server', ...],
6 # ...
7 # }
8
9 import sys
10 from pprint import pprint
11 import time
12
13 from twisted.internet.defer import inlineCallbacks
14 from twisted.internet import reactor
15 from twisted.internet.protocol import ClientCreator
16 from twisted.python import log
17
18 from txamqp.protocol import AMQClient
19 from txamqp.client import TwistedDelegate
20
21 import txamqp.spec
22
23 class twisted_queue_receiver:
24 def __init__(self, callback, errback, host, port, vhost, u
25 import sys
26
27 spec = "amqp0-8.stripped.rabbitmq.xml"
28
29 self.queue = queue
30 self.exchange = exchange
31 self.routing_key = routing_key
32 self.callback = callback
33 self.durable = durable
34 self.rate_limit = rate_limit
35
36 self.min_interval = 1.0 / float(self.rate_limit)
37 self.last_time_called = time.time()
38
39 self.prefetch_count = prefetch_count
40
41 self.no_ack = no_ack
42
43 spec = txamqp.spec.load(spec)
44
45 delegate = TwistedDelegate()

46
47 d = ClientCreator(reactor, AMQClient, delegate=del
48
49 d.addCallback(self.gotConnection, username, passwo
50
51 d.addErrback(errback)
52
53 @inlineCallbacks
54 def gotConnection(self, conn, username, password):
55 print "Connected to broker."
56 yield conn.authenticate(username, password)
57
58 print "Authenticated. Ready to receive messages"
59 self.channel = yield conn.channel(1)
60 yield self.channel.channel_open()
61
62 yield self.channel.basic_qos(prefetch_count=self.p
63
64 yield self.channel.queue_declare(queue=self.queue,
65 yield self.channel.exchange_declare(exchange=self.
66
67 yield self.channel.queue_bind(queue=self.queue, ex
68
69 yield self.channel.basic_consume(queue=self.queue,
70
71 self.queue = yield conn.queue(self.routing_key)
72
73 self.getMessages()
74
75 @inlineCallbacks
76 def getMessages(self):
77 """
78 def RateLimited(maxPerSecond):
79 minInterval = 1.0 / float(maxPerSecond)
80 def decorate(func):
81 lastTimeCalled = [0.0]
82 def rateLimitedFunction(*args,**ka
83 elapsed = time.time() - la
84 leftToWait = minInterval -
 85 if leftToWait>0:
86 time.sleep(leftToW
87 ret = func(*args,**kargs)
88 lastTimeCalled[0] = time.t
89 return ret
90 return rateLimitedFunction
91 return decorate
92 """
93
94 elapsed = time.time() - self.last_time_called
95 left_to_wait = self.min_interval - elapsed
96
97 if left_to_wait > 0:
98 yield reactor.callLater(left_to_wait, self
99 else:
100 self.last_time_called = time.time()
101
102 message = yield self.queue.get()
103 self.callback(self.channel, message)
104
105 elapsed = time.time() - self.last_time_cal
106 left_to_wait = self.min_interval - elapsed
107
108 if left_to_wait < 0:
109 left_to_wait = 0
110
111 #print "left_to_wait: ", left_to_wait
112
113 yield reactor.callLater(left_to_wait*1.01,
114
115 class twisted_queue_sender:
116
117 @inlineCallbacks
118 def gotConnection(self, conn, username, password):
119 print "Connected to broker."
120 yield conn.authenticate(username, password)
121
122 print "Authenticated. Ready to send messages"
123 self.channel = yield conn.channel(1)
124 yield self.channel.channel_open()
125
126 yield self.channel.queue_declare(queue=self.queue,
127 yield self.channel.exchange_declare(exchange=self.
128

129 yield self.channel.queue_bind(queue=self.queue, ex

130
131 yield self.callback(self.channel)
132
133 yield self.channel.channel_close()
134
135 #channel0 = yield conn.channel(0)
136 #yield channel0.channel_close()
137 #reactor.stop()
138
139
140 @inlineCallbacks
141 def put(self, msg):
142 yield self.queue.put(msg)
143
144 def __init__(self, callback, host, port, vhost, username,
145 import sys
146
147 spec = "amqp0-8.stripped.rabbitmq.xml"
148
149 self.exchange = exchange
150 self.routing_key = routing_key
151 self.durable = durable
152 self.callback = callback
153 self.queue = queue
154
155 spec = txamqp.spec.load(spec)
156
157 delegate = TwistedDelegate()
158
159 d = ClientCreator(reactor, AMQClient, delegate=del
160
161 d.addCallback(self.gotConnection, username, passwo
162
163 def whoops(err):
164 if reactor.running:
165 log.err(err)
166 reactor.stop()
167
168 d.addErrback(whoops)
169