Please enable JavaScript.

Coggle requires JavaScript to display documents.

PySpark - Coggle Diagram

- - - - SparkSession as spark
        
        catalog :attribute to list all data in cluster
        
        spark.catalog.listTables()
        
        spark.sql(query)
        
        .show()
        
        toPandas()
        
        temp = spark.createDataFrame(df)
        
        from DF to catalog: temp.createOrReplaceTempView('name')
        
        from catalog to DF: spark.table('col')
        
        df.withColumn('new_col', df.old_col +/-/* sth)
        
        df.filter()
        
        df.select()
        
        to name: (df.col / 60).alias('name')
        
        flights.select("origin", "dest", "tailnum", avg_speed)
        
        flights.selectExpr("origin", "dest", "tailnum", "distance/(air_time/60) as avg_speed")
        
        GroupedData
        
        df.groupBy()
        
        4 more items...
        
        pyspark.sql.functions as F
        
        round()
        
        df.withColumnRenamed('old_name', 'new_name')
        
        Joining
        
        df1.join(df2, on = 'col_key', how = 'leftouter')
        
        df.drop()
        
        df.dropna()
        
        df.distinct()
        
        spark.read.csv(file_path, header = True, schema = schema, inferSchema = T/F, nullValue='NA'))
        
        from pyspark.sql.types import StructType, StructField, IntegerType, StringType
        
        schema = StructType([StructField("id", IntegerType()),
        
        csv.printSchema()
      - SparkSession.builder.getOrCreate()
      - SparkSession.builder.master('local[*]').appName('name').getOrCreate()