pyspark指定schema
通过StructType对象指定DataFrame的Schema
没有嵌套结构的jsonjsonString = [
"""{ "id" : "01001", "city" : "AGAWAM", "pop" : 15338, "state" : "MA" }""",
"""{ "id" : "01002", "city" : "CUSHMAN", "pop" : 36963, "state" : "MA" }"""
]
jsonRDD = sc.parallelize(jsonString)
from pyspark.sql.types import *
#定义结构类型
#StructType：schema的整体结构，表示JSON的对象结构
#XXXStype:指的是某一列的数据类型
jsonSchema = StructType() \
.add("id", StringType(),True) \
.add("city", StringType()) \
.add("pop" , LongType()) \
.add("state",StringType())
jsonSchema = StructType() \
.add("id", LongType(),True) \
.add("city", StringType()) \
.add("pop" , DoubleType()) \
.add("state",StringType())
reader = spark.read.schema(jsonSchema)
jsonDF = reader.json(jsonRDD)
jsonDF.printSchema()
jsonDF.show()
带有嵌套结构的json
from pyspark.sql.types import *
jsonSchema = StructType([
StructField("id", StringType(), True),
StructField("city", StringType(), True),
StructField("loc" , ArrayType(DoubleType())),
StructField("pop", LongType(), True),
StructField("state", StringType(), True)
])
reader = spark.read.schema(jsonSchema)
jsonDF = reader.json('data/nest.json')
jsonDF.printSchema()
jsonDF.show(2)
jsonDF.filter(jsonDF.pop>4000).show(10)
作者：Natal-lzz
