给定一个嵌套的模式，我想在保持其他模式结构的同时展平一个子结构。在本例中，我希望展平product.spec中包含的列，以便最终得到 product.sid & product.desc .

import org.apache.spark.sql.functions._
import spark.implicits._

case class Spec(sid: Int, desc: String)
case class Prod(pid: String, spec: Spec)

val df = Seq(
  (101, "jenn", Seq(1, 2), Seq(Spec(1, "A"), Spec(2, "B")), Prod("X11", Spec(11, "X")), 1100.0),
  (202, "mike", Seq(3), Seq(Spec(3, "C")), Prod("Y22", Spec(22, "Y")), 2200.0)
).toDF("uid", "user", "ids", "specs", "product", "amount")

df.printSchema
>>>
root
 |-- uid: integer (nullable = false)
 |-- user: string (nullable = true)
 |-- ids: array (nullable = true)
 |    |-- element: integer (containsNull = false)
 |-- specs: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- sid: integer (nullable = false)
 |    |    |-- desc: string (nullable = true)
 |-- product: struct (nullable = true)
 |    |-- pid: string (nullable = true)
 |    |-- spec: struct (nullable = true)
 |    |    |-- sid: integer (nullable = false)
 |    |    |-- desc: string (nullable = true)
 |-- amount: double (nullable = false)

所需的新架构：

root
 |-- uid: integer (nullable = false)
 |-- user: string (nullable = true)
 |-- ids: array (nullable = true)
 |    |-- element: integer (containsNull = false)
 |-- specs: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- sid: integer (nullable = false)
 |    |    |-- desc: string (nullable = true)
 |-- product: struct (nullable = true)
 |    |-- pid: string (nullable = true)
 |    |-- spec: struct (nullable = true)
 |    |-- sid: integer (nullable = false)
 |    |-- desc: string (nullable = true)
 |-- amount: double (nullable = false)

遗憾的是，默认的sql接口不允许轻松地完成这项工作，因为它只使用列名。

//incorrect result
df.withColumn("product.sid", col("product.spec.sid")).withColumn("product.desc", col("product.spec.desc")).drop("product.spec.desc").drop("product.spec.sid").printSchema
>>
root
 |-- uid: integer (nullable = false)
 |-- user: string (nullable = true)
 |-- ids: array (nullable = true)
 |    |-- element: integer (containsNull = false)
 |-- specs: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- sid: integer (nullable = false)
 |    |    |-- desc: string (nullable = true)
 |-- product: struct (nullable = true)
 |    |-- pid: string (nullable = true)
 |    |-- spec: struct (nullable = true)
 |    |    |-- sid: integer (nullable = false)
 |    |    |-- desc: string (nullable = true)
 |-- amount: double (nullable = false)
 |-- product.sid: integer (nullable = true)
 |-- product.desc: string (nullable = true)