over

Apply a window specification to the column.

Syntax

over(window)

Parameters

Returns

Column

Examples

from pyspark.sql import Window
from pyspark.sql.functions import rank, min, desc
window = (
    Window.partitionBy("name")
    .orderBy("age")
    .rowsBetween(Window.unboundedPreceding, Window.currentRow)
)
df = spark.createDataFrame(
     [(2, "Alice"), (5, "Bob")], ["age", "name"])
df.withColumn(
     "rank", rank().over(window)
).withColumn(
     "min", min('age').over(window)
).sort(desc("age")).show()

# +---+-----+----+---+
# |age| name|rank|min|
# +---+-----+----+---+
# |  5|  Bob|   1|  5|
# |  2|Alice|   1|  2|
# +---+-----+----+---+