欢迎来到代码驿站!

Python代码

当前位置:首页 > 软件编程 > Python代码

python hbase读取数据发送kafka的方法

时间:2020-11-08 12:25:21|栏目:Python代码|点击:

本例子实现从hbase获取数据,并发送kafka。

使用

#!/usr/bin/env python
#coding=utf-8
 
import sys
import time
import json
 
sys.path.append('/usr/local/lib/python3.5/site-packages')
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hbase1 import Hbase #调用hbase thrif1
from hbase1.ttypes import *
from kafka import KafkaConsumer
from kafka import KafkaProducer
from kafka.errors import KafkaError
import unittest
 
class HbaseOpreator:
 def __init__(self,host,port,table='test'):
  self.tableName=table
  self.transport=TTransport.TBufferedTransport(TSocket.TSocket(host,port))
  self.protocol=TBinaryProtocol.TBinaryProtocol(self.transport)
  self.client=Hbase.Client(self.protocol)
  self.transport.open()
 
 def __del__(self):
  self.transport.close()
 
 
 def scanTablefilter(self,table,*args):
  d=dict() 
  L=[]
  try:
   tableName=table
   # scan = Hbase.TScan(startRow, stopRow)
   scan=TScan()
   #主键首字母123
   # filter = "PrefixFilter('123_')"
   # filter = "RowFilter(=,'regexstring:.aaa')"
   #过滤条件,当前为 statis_date 字段,值为20170223
   # fitler = "SingleColumnValueFilter(tableName,'f','statis_date','20170223')"
   # filter="SingleColumnValueFilter('f','statis_date',=,'binary:20170223') AND SingleColumnValueFilter('f','name',=,'binary:LXS')"
   filter="SingleColumnValueFilter('info','name',=,'binary:lilei') OR SingleColumnValueFilter('info','name',=,'binary:lily')"
   scan.filterString=filter
   id=self.client.scannerOpenWithScan(tableName,scan,None)
   result=self.client.scannerGet(id)
   # result=self.client.scannerGetList(id,100)
   while result:
    for r in result:
     key=r.row
     name=r.columns.get('info:name').value
     age=r.columns.get('info:age').value
     phone=r.columns.get('info:phone').value
     d['key']=key
     d['name']=name
     d['age']=age
     d['phone']=phone
     # encode_result_json=json.dumps(d).encode(encoding="utf-8")
     # print(encode_result_json)
     L.append(d)         
    result=self.client.scannerGet(id)    
   return json.dumps(L).encode(encoding="utf-8")  
  finally:
   # self.client.scannerClose(scan)
   print("scan finish")
 
def sendKfafkaProduct(data):
 # self.host_port='localhost:9092'
 producer = KafkaProducer(bootstrap_servers=['localhost:9092'])
 for d in data:
  producer.send('test', key=b'lxs', value=d)
  time.sleep(5)
  print(d)
 
 while True:
  producer.send('test', key=b'lxs', value=data)
  time.sleep(5)
  print(data)
 
if __name__== '__main__':
 # unittest.main()
 
 B=HbaseOpreator('10.27.1.138',9090)
 value=B.scanTablefilter('ns_lbi:test_hbase_student')
 print(value)
 #sendKfafkaProduct(value)
 

上一篇:python实现提取str字符串/json中多级目录下的某个值

栏    目:Python代码

下一篇:Python模块的加载讲解

本文标题:python hbase读取数据发送kafka的方法

本文地址:http://www.codeinn.net/misctech/20338.html

推荐教程

广告投放 | 联系我们 | 版权申明

重要申明:本站所有的文章、图片、评论等,均由网友发表或上传并维护或收集自网络,属个人行为,与本站立场无关。

如果侵犯了您的权利,请与我们联系,我们将在24小时内进行处理、任何非本站因素导致的法律后果,本站均不负任何责任。

联系QQ:914707363 | 邮箱:codeinn#126.com(#换成@)

Copyright © 2020 代码驿站 版权所有