Spark API example fails on Windows #1390
Description
[main] WARN org.apache.hadoop.util.Shell - Did not find winutils.exe: {}
java.io.FileNotFoundException: java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset. -see https://wiki.apache.org/hadoop/WindowsProblems
at org.apache.hadoop.util.Shell.fileNotFoundException(Shell.java:600)
at org.apache.hadoop.util.Shell.getHadoopHomeDir(Shell.java:621)
at org.apache.hadoop.util.Shell.getQualifiedBin(Shell.java:644)
at org.apache.hadoop.util.Shell.<clinit>(Shell.java:741)
at org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:79)
at org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1713)
at org.apache.hadoop.security.SecurityUtil.setConfigurationInternal(SecurityUtil.java:103)
at org.apache.hadoop.security.SecurityUtil.<clinit>(SecurityUtil.java:92)
at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:312)
at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:300)
at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:575)
at org.apache.spark.util.Utils$.$anonfun$getCurrentUserName$1(Utils.scala:2561)
at scala.Option.getOrElse(Option.scala:201)
at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2561)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:316)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2714)
at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:953)
at scala.Option.getOrElse(Option.scala:201)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:947)
at org.jetbrains.kotlinx.dataframe.examples.kotlinSpark.UntypedDatasetKt.main(untypedDataset.kt:102)
at org.jetbrains.kotlinx.dataframe.examples.kotlinSpark.UntypedDatasetKt.main(untypedDataset.kt)
Caused by: java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset.
at org.apache.hadoop.util.Shell.checkHadoopHomeInner(Shell.java:520)
at org.apache.hadoop.util.Shell.checkHadoopHome(Shell.java:491)
Caused by: java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset.
at org.apache.hadoop.util.Shell.<clinit>(Shell.java:568)
... 17 more
[main] WARN org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[main] INFO org.apache.spark.resource.ResourceUtils - ==============================================================
[main] INFO org.apache.spark.resource.ResourceUtils - No custom resources configured for spark.driver.
[main] INFO org.apache.spark.resource.ResourceUtils - ==============================================================
[main] INFO org.apache.spark.SparkContext - Submitted application: Kotlin Spark Sample
[main] INFO org.apache.spark.resource.ResourceProfile - Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)
[main] INFO org.apache.spark.resource.ResourceProfile - Limiting resource is cpu
[main] INFO org.apache.spark.resource.ResourceProfileManager - Added ResourceProfile id: 0
[main] INFO org.apache.spark.SecurityManager - Changing view acls to: Aleksey.Zinovev
[main] INFO org.apache.spark.SecurityManager - Changing modify acls to: Aleksey.Zinovev
[main] INFO org.apache.spark.SecurityManager - Changing view acls groups to:
[main] INFO org.apache.spark.SecurityManager - Changing modify acls groups to:
[main] INFO org.apache.spark.SecurityManager - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(Aleksey.Zinovev); groups with view permissions: Set(); users with modify permissions: Set(Aleksey.Zinovev); groups with modify permissions: Set()
[main] INFO org.apache.spark.util.Utils - Successfully started service 'sparkDriver' on port 51105.
[main] INFO org.apache.spark.SparkEnv - Registering MapOutputTracker
Exception in thread "main" java.lang.ExceptionInInitializerError
at org.apache.spark.unsafe.array.ByteArrayMethods.<clinit>(ByteArrayMethods.java:56)
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes$lzycompute(MemoryManager.scala:264)
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes(MemoryManager.scala:254)
at org.apache.spark.memory.MemoryManager.$anonfun$pageSizeBytes$1(MemoryManager.scala:273)
at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.scala:17)
at scala.Option.getOrElse(Option.scala:201)
at org.apache.spark.memory.MemoryManager.<init>(MemoryManager.scala:273)
at org.apache.spark.memory.UnifiedMemoryManager.<init>(UnifiedMemoryManager.scala:58)
at org.apache.spark.memory.UnifiedMemoryManager$.apply(UnifiedMemoryManager.scala:207)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:320)
at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194)
at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:279)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:464)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2714)
at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:953)
at scala.Option.getOrElse(Option.scala:201)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:947)
at org.jetbrains.kotlinx.dataframe.examples.kotlinSpark.UntypedDatasetKt.main(untypedDataset.kt:102)
at org.jetbrains.kotlinx.dataframe.examples.kotlinSpark.UntypedDatasetKt.main(untypedDataset.kt)
Caused by: java.lang.IllegalStateException: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
at org.apache.spark.unsafe.Platform.<clinit>(Platform.java:113)
Caused by: java.lang.IllegalStateException: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
... 19 more
Caused by: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
at java.base/java.lang.Class.getConstructor0(Class.java:3761)
at java.base/java.lang.Class.getDeclaredConstructor(Class.java:2930)
Caused by: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.<init>(long,int)
at org.apache.spark.unsafe.Platform.<clinit>(Platform.java:71)```