hdfs官网：http://hdfscli.readthedocs.io/en/latest/api.html
一个非常好的博客：http://blog.csdn.net/gamer_gyt/article/details/52446757
hdfs库中自带avro序列化与反序列化模块，不需要单独做
#!/usr/bin/env python
# encoding: utf-8 """Avro extension example.""" from hdfs import Config
from hdfs.ext.avro import AvroReader, AvroWriter # Get the default alias' client.
client = Config().get_client() # Some sample data.
records = [
{'name': 'Ann', 'age': 23},
{'name': 'Bob', 'age': 22},
] # Write an Avro File to HDFS (since our records' schema is very simple, we let
# the writer infer it automatically, otherwise we would pass it as argument).
with AvroWriter(client, 'names.avro', overwrite=True) as writer:
for record in records:
writer.write(record) # Read it back.
with AvroReader(client, 'names.avro') as reader:
schema = reader.schema # The inferred schema.
content = reader.content # The remote file's HDFS content object.
assert list(reader) == records # The records match!
遍历hdfs目录
from hdfs import *
import os
from hdfs.ext.avro import AvroReader, AvroWriter def main():
client=Client("http://127.0.0.1:50070")
path = "/test/tmp_data"
for root, dir, files in client.walk(path):
for file in files:
full_path = os.path.join(root, file)
print full_path
with AvroReader(client, full_path) as reader:
schema = reader.schema # The inferred schema.
content = reader.content # The remote file's HDFS content object.
#assert list(reader) == records
for user in list(reader):
print user main()