python方式对接AWS的kinesis

1、准备

# 安装依赖
pip3 install boto3 awscli
pip3 install PyJWT
pip3 install redis
pip3 install PyMySQL

执行aws configure命令

wikoapp01@wiko-app-preload:~$ aws configure
AWS Access Key ID [****************EOFB]: accessKey
AWS Secret Access Key [****************r1Md]: secretKey
Default region name [us-west-1]: eu-west-1
Default output format [json]:

假如执行aws configure命令时报aws命令找不到，可以尝试使用pip install awscli，测试后发现也同样可以使用。

2、生产者

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @author 黑夜流星
# @date 2021/2/4.

import boto3
import json
from datetime import datetime
import calendar
import random
import time

my_stream_name = 'python_stream'

kinesis_client = boto3.client('kinesis', region_name='eu-west-1')


def put_to_stream(thing_id, property_value, property_timestamp):
    payload = {"calories": 33, "date": "2021-02-04", "distance": 412, "kilometres": 0.4, "miles": 0.3, "pace": 589, "stepLength": 70.0,
               "steps": "[0,0,0,0,0,0,18,116,347,108,0,0,0,0,0,0,0,0,0,0,0,0,0,0]",
               "token": "xxxxxxxxxxxxxxxxxxxx",
               "user": 25533}

    print(payload)

    put_response = kinesis_client.put_record(
        StreamName=my_stream_name,
        Data=json.dumps(payload),
        PartitionKey=thing_id)


while True:
    property_value = random.randint(40, 120)
    property_timestamp = calendar.timegm(datetime.utcnow().timetuple())
    thing_id = 'aa-bb'

    put_to_stream(thing_id, property_value, property_timestamp)

    # wait for 5 second
    # time.sleep(5)

2、消费者

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @author 黑夜流星
# @date 2021/2/4.
import hashlib
import json
import time

import boto3
import jwt
import pymysql
import redis

my_stream_name = 'python_stream'

kinesis_client = boto3.client('kinesis', region_name='eu-west-1')

response = kinesis_client.describe_stream(StreamName=my_stream_name)

my_shard_id = response['StreamDescription']['Shards'][0]['ShardId']

shard_iterator = kinesis_client.get_shard_iterator(StreamName=my_stream_name,
                                                   ShardId=my_shard_id,
                                                   ShardIteratorType='LATEST')

my_shard_iterator = shard_iterator['ShardIterator']

record_response = kinesis_client.get_records(ShardIterator=my_shard_iterator,
                                             Limit=100)

rds = redis.StrictRedis(host='127.0.0.1', port=6379, decode_responses=True)


def check_token(token):
    data = jwt.decode(token, "jwt秘钥", algorithms=['HS512'], options={"verify_signature": False})
    bean = json.loads(data['sub'])
    exp = int(data["exp"])
    now = int(time.time())
    if exp < now:
        return False, None

    return True, bean


if __name__ == "__main__":
    while 'NextShardIterator' in record_response:
        record_response = kinesis_client.get_records(ShardIterator=record_response['NextShardIterator'],
                                                     Limit=100)
        # print(record_response)
        if len(record_response['Records']) > 0:
            # 打开数据库连接
            db = pymysql.connect(host="127.0.0.1", user="root", password="root", database="test_db")

            # 使用cursor()方法获取操作游标
            cursor = db.cursor()
            try:
                for record in record_response['Records']:
                    decrypted_message = str(record['Data'], encoding='utf-8')
                    # print(decrypted_message)
                    record_json = json.loads(decrypted_message)
                    token = record_json['token']
                    flag, bean = check_token(token)
                    if flag:
                        email = bean['email']
                        if email:
                            md5_string = rds.get(bean['email'])
                            m = hashlib.md5(token.encode())
                            res = m.hexdigest()
                            if res in md5_string:
                                sql = """insert into xxx_table
                                        (user_id, for_date, step_counting, calorie, distance, create_time, update_time, steps, step_length)
                                    values
                                        (%s, '%s', %s,  %s,%s, NOW(),NOW(), '%s', %s)
                                    ON DUPLICATE KEY UPDATE
                                        step_counting = %s,calorie = %s,distance = %s,
                                        update_time = NOW(),steps = '%s', step_length = %s""" % (
                                    record_json['user'], record_json['date'].replace('-', ''), record_json['pace'], record_json['calories'], record_json['distance'], record_json['steps'],
                                    record_json['stepLength'],
                                    record_json['pace'], record_json['calories'], record_json['distance'],
                                    record_json['steps'],
                                    record_json['stepLength'])
                                # print(sql)

                                # 执行sql语句
                                cursor.execute(sql)
                                # 提交到数据库执行
                                db.commit()
            except Exception as e:
                # 如果发生错误则回滚
                db.rollback()

            # 关闭数据库连接
            db.close()
            # wait for 5 seconds
            # time.sleep(5)