var root = "dbfs:/datasets/student-project-01"
try{
  dbutils.fs.ls(root)
} catch {
  case e: java.io.FileNotFoundException => dbutils.fs.mkdirs(root)
}

display(dbutils.fs.ls(root))

var fl_root = "dbfs:/datasets/student-project-01/familjeliv/"
try{
  dbutils.fs.ls(fl_root)
} catch {
  case e: java.io.FileNotFoundException => dbutils.fs.mkdirs(fl_root)
}

var fb_root = "dbfs:/datasets/student-project-01/flashback/"
try{
  dbutils.fs.ls(fb_root)
} catch {
  case e: java.io.FileNotFoundException => dbutils.fs.mkdirs(fb_root)
}

display(dbutils.fs.ls(root))

//[   ]	familjeliv-adoption.xml.bz2	            2017-07-23 17:39	195M	 
//[   ]	familjeliv-allmanna-ekonomi.xml.bz2	    2017-09-18 13:50	838M	 
//[   ]	familjeliv-allmanna-familjeliv.xml.bz2	2017-09-19 15:49	1.1G	 
//[   ]	familjeliv-allmanna-fritid.xml.bz2	    2017-09-19 17:30	588M	 
//[   ]	familjeliv-allmanna-husdjur.xml.bz2	    2017-09-20 15:31	846M	 
//[   ]	familjeliv-allmanna-hushem.xml.bz2	    2017-09-21 12:27	1.3G	 
//[   ]	familjeliv-allmanna-kropp.xml.bz2	    2017-09-21 18:41	2.3G	 
//[   ]	familjeliv-allmanna-noje.xml.bz2	    2017-09-21 19:57	1.6G	 
//[   ]	familjeliv-allmanna-samhalle.xml.bz2	2017-09-22 03:52	5.0G	 
//[   ]	familjeliv-allmanna-sandladan.xml.bz2	2017-09-22 10:43	778M	 
//[   ]	familjeliv-anglarum.xml.bz2	            2017-07-23 18:10	336M	 
//[   ]	familjeliv-expert.xml.bz2	            2017-07-20 11:32	142M	 
//[   ]	familjeliv-foralder.xml.bz2	            2017-08-04 19:16	10G	 
//[   ]	familjeliv-gravid.xml.bz2	            2017-07-15 04:50	7.5G	 
//[   ]	familjeliv-kansliga.xml.bz2	            2017-09-05 18:13	14G	 
//[   ]	familjeliv-medlem-allmanna.xml.bz2	    2017-07-24 03:47	4.4G	 
//[   ]	familjeliv-medlem-foraldrar.xml.bz2	    2017-07-20 22:11	4.5G	 
//[   ]	familjeliv-medlem-planerarbarn.xml.bz2	2017-07-18 15:08	1.9G	 
//[   ]	familjeliv-medlem-vantarbarn.xml.bz2	2017-07-17 08:04	4.5G	 
//[   ]	familjeliv-pappagrupp.xml.bz2	        2017-06-28 16:18	38M	 
//[   ]	familjeliv-planerarbarn.xml.bz2	        2017-08-28 20:55	2.8G	 
//[   ]	familjeliv-sexsamlevnad.xml.bz2	        2017-08-25 16:39	2.3G	 
//[   ]	familjeliv-svartattfabarn.xml.bz2	    2017-07-03 07:04	2.6G

//[   ]	flashback-dator.xml.bz2	                2017-04-06 09:08	4.5G	 
//[   ]	flashback-droger.xml.bz2	            2017-04-06 08:59	3.5G	 
//[   ]	flashback-ekonomi.xml.bz2	            2017-04-06 10:53	1.2G	 
//[   ]	flashback-flashback.xml.bz2	            2017-04-05 18:16	429M	 
//[   ]	flashback-fordon.xml.bz2	            2017-04-06 12:00	1.0G	 
//[   ]	flashback-hem.xml.bz2	                2017-04-07 03:10	4.6G	 
//[   ]	flashback-kultur.xml.bz2	            2017-04-06 22:51	5.5G	 
//[   ]	flashback-livsstil.xml.bz2	            2017-04-07 00:11	1.7G	 
//[   ]	flashback-mat.xml.bz2	                2017-04-07 08:52	1.0G	 
//[   ]	flashback-ovrigt.xml.bz2	            2017-04-07 18:54	1.9G	 
//[   ]	flashback-politik.xml.bz2	            2017-04-14 17:06	9.0G	 
//[   ]	flashback-resor.xml.bz2	                2017-04-09 15:52	566M	 
//[   ]	flashback-samhalle.xml.bz2	            2017-04-12 20:36	8.3G	 
//[   ]	flashback-sex.xml.bz2	                2017-04-11 20:32	1.3G	 
//[   ]	flashback-sport.xml.bz2	                2017-04-12 22:10	3.3G	 
//[   ]	flashback-vetenskap.xml.bz2	            2017-04-14 20:34	5.8G	 

import sys.process._

val fl_data = Array("familjeliv-allmanna-ekonomi.xml",
                    "familjeliv-sexsamlevnad.xml")

val fb_data = Array("flashback-ekonomi.xml",
                    "flashback-sex.xml")

val url = "http://spraakbanken.gu.se/lb/resurser/meningsmangder/"
val tmp_folder_fl = "/tmp/familjeliv/"
val tmp_folder_fb = "/tmp/flashback/"

s"rm -f -r ${tmp_folder_fl}" .!! // Remove tmp folder if exists
s"rm -f -r ${tmp_folder_fb}" .!!

for (name <- fl_data){
  try{
    dbutils.fs.ls(s"${fl_root}${name}")
    println(s"${name} already exists!")
  }
  catch{
    case e: java.io.FileNotFoundException => {
      println(s"Downloading ${name} ...")
      s"wget -P ${tmp_folder_fl} ${url}${name}.bz2" .!!
      println("Unzipping ...")
      s"bzip2 -d ${tmp_folder_fl}${name}.bz2" .!!
      println("Moving ... ")
      val localpath=s"file:${tmp_folder_fl}${name}"
      dbutils.fs.mv(localpath, fl_root)
      println(s"Done ${name}!")
    }
  }
}

s"rm -f -r ${tmp_folder_fl}" .!!

for (name <- fb_data){
  try{
    dbutils.fs.ls(s"${fb_root}${name}")
    println(s"${name} already exists!")
  }
  catch{
    case e: java.io.FileNotFoundException => {
      println(s"Downloading ${name} ...")
      s"wget -P ${tmp_folder_fb} ${url}${name}.bz2" .!!
      println("Unzipping ...")
      s"bzip2 -d ${tmp_folder_fb}${name}.bz2" .!!
      println("Moving ... ")
      val localpath=s"file:${tmp_folder_fb}${name}"
      dbutils.fs.mv(localpath, fb_root)
      println(s"Done ${name}!")
    }
  }
}
s"rm -f -r ${tmp_folder_fb}" .!!

familjeliv-allmanna-ekonomi.xml already exists!
familjeliv-sexsamlevnad.xml already exists!
flashback-ekonomi.xml already exists!
flashback-sex.xml already exists!
import sys.process._
fl_data: Array[String] = Array(familjeliv-allmanna-ekonomi.xml, familjeliv-sexsamlevnad.xml)
fb_data: Array[String] = Array(flashback-ekonomi.xml, flashback-sex.xml)
url: String = http://spraakbanken.gu.se/lb/resurser/meningsmangder/
tmp_folder_fl: String = /tmp/familjeliv/
tmp_folder_fb: String = /tmp/flashback/
res14: String = ""

// Data comes in XML-files with the following structure.
/*
<corpus id="familjeliv-adoption">
<forum id="13-242" title="Adoption &gt; Intresserad" url="http://www.familjeliv.
se/forum/13/242">
<thread id="34277335" title="Tips för att välja land" postcount="25" lastpost="2
008-07-08 17:55:14" url="http://www.familjeliv.se/forum/thread/34277335-tips-for
-att-valja-land">
<text datefrom="20080707" dateto="20080707" timefrom="220854" timeto="220854" lix="30.55" ovix="60.74" nk="0.51" id="34284994" username="Miss TN" date="2008-07-07 22:08:54" url="http://www.familjeliv.se/forum/thread/34277335-tips-for-att-valja-land/2#anchor-m16">
<sentence id="bacc55562-baca83a75" _geocontext="|">
<w pos="VB" msd="VB.PRS.AKT" lemma="|känna|" lex="|känna..vb.2|känna..vb.1|" sense="|känna..1:0.522|känna..2:0.313|känna..4:0.158|känna..3:0.006|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="01" dephead="07" deprel="AA">Känner</w>
<w pos="PN" msd="PN.UTR.SIN.DEF.SUB" lemma="|ni|" lex="|ni..pn.1|" sense="|ni..1:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="02" dephead="01" deprel="OO">ni</w>
<w pos="PP" msd="PP" lemma="|för|" lex="|för..pp.1|" sense="|för..1:-1.000|för..5:-1.000|för..6:-1.000|för..7:-1.000|för..9:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="03" dephead="01" deprel="OA">för</w>
<w pos="DT" msd="DT.NEU.SIN.IND" lemma="|en|" lex="|en..al.1|" sense="|den..1:-1.000|en..2:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="04" dephead="06" deprel="DT">ett</w>
<w pos="JJ" msd="JJ.POS.NEU.SIN.IND.NOM" lemma="|låg|" lex="|låg..av.1|" sense="|låg..1:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="05" dephead="06" deprel="AT">lågt</w>
<w pos="NN" msd="NN.NEU.SIN.IND.NOM" lemma="|medgivande|" lex="|medgivande..nn.1|" sense="|medgivande..1:0.595|medgivande..2:0.405|" prefix="|medge..vb.1|mede..nn.1|" suffix="|givande..nn.1|ande..nn.1|" compwf="|medgiv+ande|med+givande|med+giv+ande|" complemgram="|medge..vb.1+ande..nn.1:4.632e-17|mede..nn.1+givande..nn.1:6.075e-27|mede..nn.1+giv..nn.1+ande..nn.1:5.387e-27|mede..nn.1+ge..vb.1+ande..nn.1:1.257e-29|" ref="06" dephead="03" deprel="PA">medgivande</w>
<w pos="VB" msd="VB.PRS.AKT" lemma="|skola|" lex="|skola..vb.2|" sense="|skola..4:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="07" deprel="ROOT">ska</w>
<w pos="PN" msd="PN.UTR.PLU.DEF.SUB" lemma="|ni|" lex="|ni..pn.1|" sense="|ni..1:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="08" dephead="07" deprel="SS">ni</w>
<w pos="AB" msd="AB" lemma="|verkligen|" lex="|verkligen..ab.1|" sense="|verkligen..1:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="09" dephead="07" deprel="MA">verkligen</w>
<w pos="VB" msd="VB.INF.AKT" lemma="|sträva|" lex="|sträva..vb.1|" sense="|sträva..1:-1.000|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="10" dephead="07" deprel="VG">sträva</w>
.
.
.
</sentence>
</text>
</thread>
<thread id="42755312" title="Om vi planerar ett barn om sju år, när ska vi dra igång adoptionsprocessen?" postcount="3" lastpost="2009-04-01 18:39:55" url="http://www.familjeliv.se/forum/thread/42755312-om-vi-planerar-ett-barn-om-sju-ar-nar-ska-vi-dra-igang-adoptionsprocessen">
<text datefrom="20090331" dateto="20090331" timefrom="201724" timeto="201724" lix="27.48" ovix="50.60" nk="0.37" id="42755312" username="alvaereva" date="2009-03-31 20:17:24" url="http://www.familjeliv.se/forum/thread/42755312-om-vi-planerar-ett-barn-om-sju-ar-nar-ska-vi-dra-igang-adoptionsprocessen/1">
<sentence id="bac2ec05f-bace4e647" _geocontext="|">
<w pos="KN" msd="KN" lemma="|" lex="|" sense="|" prefix="|" suffix="|" compwf="|" complemgram="|" ref="01" deprel="ROOT">Som</w>
.
.
.
*/

import org.apache.spark.sql.functions.{col,concat_ws, udf, flatten, explode, collect_list, collect_set, lit}
import org.apache.spark.sql.types.{ ArrayType, StructType, StructField, StringType, IntegerType }
import com.databricks.spark.xml._ // Add the DataFrame.read.xml() method
import org.apache.spark.sql.functions._


def read_xml(file_name: String): org.apache.spark.sql.DataFrame = {
  val sentence_schema = StructType(Array(
    StructField("w", ArrayType(StringType, true), nullable=true)
  ))
  val text_schema = StructType(Array(
     StructField("sentence", ArrayType(sentence_schema), nullable=false)
  ))

  val thread_schema = StructType(Array(
    StructField("_id", StringType, nullable = false),
    StructField("_title", StringType, nullable = false),
    StructField("_url", StringType, nullable = false),
    StructField("text", text_schema, nullable=false)
  ))

  val forum_schema = StructType(Array(
    StructField("_id", StringType, nullable = false),
    StructField("_title", StringType, nullable = false),
    StructField("_url", StringType, nullable = false),
    StructField("thread", ArrayType(thread_schema), nullable=false)
  ))

  val corpus_schema = StructType(Array(
    StructField("_id", StringType, nullable = false),
    StructField("forum", forum_schema, nullable=false)
  ))

  spark.read
    .option("rowTag", "forum")
    .schema(forum_schema)
    .xml(file_name)
 }

def get_dataset(file_name: String) : org.apache.spark.sql.DataFrame = {
  val xml_df = read_xml(file_name)
  val ds = xml_df.withColumn("thread", explode($"thread"))
  val splitted_name = file_name.split("/")
  val forum = splitted_name(splitted_name.size-2)
  val corpus = splitted_name(splitted_name.size-1)
  val value = udf((arr: Seq[String]) => arr.mkString(","))
  ds.select(col("_id") as "forum_id",
                     col("_title") as "forum_title",
                     col("thread._id") as "thread_id",
                     col("thread._title") as "thread_title",
                     flatten(col("thread.text.sentence.w")) as "w")
                .withColumn("w", explode($"w"))
               .groupBy("thread_id")
               .agg(first("thread_title") as("thread_title"),
                    collect_list("w") as "w",
                    first("forum_id") as "forum_id",
                    first("forum_title") as "forum_title")
               .withColumn("w", concat_ws(",",col("w")))
               .withColumn("platform", lit(forum))
               .withColumn("corpus_id", lit(corpus))
}

import org.apache.spark.sql.functions.{col, concat_ws, udf, flatten, explode, collect_list, collect_set, lit}
import org.apache.spark.sql.types.{ArrayType, StructType, StructField, StringType, IntegerType}
import com.databricks.spark.xml._
import org.apache.spark.sql.functions._
read_xml: (file_name: String)org.apache.spark.sql.DataFrame
get_dataset: (file_name: String)org.apache.spark.sql.DataFrame

def save_df(df: org.apache.spark.sql.DataFrame, filePath:String){
  df.write.format("parquet").save(filePath)  
}

def load_df(filePath: String): org.apache.spark.sql.DataFrame = {
  spark.read.format("parquet").load(filePath)
 }

def no_forums(df: org.apache.spark.sql.DataFrame): Long = {
  val forums = df.select("forum_title").distinct()
  forums.show(false)
  forums.count()
}

save_df: (df: org.apache.spark.sql.DataFrame, filePath: String)Unit
load_df: (filePath: String)org.apache.spark.sql.DataFrame
no_forums: (df: org.apache.spark.sql.DataFrame)Long

var fl_root = "dbfs:/datasets/student-project-01/familjeliv/"
var fb_root = "dbfs:/datasets/student-project-01/flashback/"

val fl_data = Array("familjeliv-allmanna-ekonomi",
                    "familjeliv-sexsamlevnad")

val fb_data = Array("flashback-ekonomi",
                    "flashback-sex")

for (name <- fl_data){
  try{
    println(s"${fb_root}${name}_df")
    dbutils.fs.ls(s"${fl_root}${name}_df")
    println(s"${name}_df already exists!")
  }
  catch{
    case e: java.io.FileNotFoundException => {
      val file_name = s"${fl_root}${name}.xml"
      val df = get_dataset(file_name)
      val file_path = s"${fl_root}${name}_df"
      save_df(df, file_path)
    }
  }
}

for (name <- fb_data){
  try{
    println(s"${fb_root}${name}_df")
    dbutils.fs.ls(s"${fb_root}${name}_df")
    println(s"${name}_df already exists!")
  }
  catch{
    case e: java.io.FileNotFoundException => {
      val file_name = s"${fb_root}${name}.xml"
      val df = get_dataset(file_name)
      val file_path = s"${fb_root}${name}_df"
      save_df(df, file_path)
    }
  }
}

dbfs:/datasets/student-project-01/flashback/familjeliv-allmanna-ekonomi_df
familjeliv-allmanna-ekonomi_df already exists!
dbfs:/datasets/student-project-01/flashback/familjeliv-sexsamlevnad_df
familjeliv-sexsamlevnad_df already exists!
dbfs:/datasets/student-project-01/flashback/flashback-ekonomi_df
flashback-ekonomi_df already exists!
dbfs:/datasets/student-project-01/flashback/flashback-sex_df
flashback-sex_df already exists!
fl_root: String = dbfs:/datasets/student-project-01/familjeliv/
fb_root: String = dbfs:/datasets/student-project-01/flashback/
fl_data: Array[String] = Array(familjeliv-allmanna-ekonomi, familjeliv-sexsamlevnad)
fb_data: Array[String] = Array(flashback-ekonomi, flashback-sex)

/scalable-data-science/000_0-sds-3-x-projects/student-project-01_group-TheTwoCultures/01_load_data

var file_name = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-allmanna-ekonomi.xml"
var xml_df = read_xml(file_name).cache()
var df = get_dataset(file_name).cache()

file_name: String = dbfs:/datasets/student-project-01/familjeliv/familjeliv-allmanna-ekonomi.xml
xml_df: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [_id: string, _title: string ... 2 more fields]
df: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 5 more fields]

xml_df.printSchema()

root
 |-- _id: string (nullable = false)
 |-- _title: string (nullable = false)
 |-- _url: string (nullable = false)
 |-- thread: array (nullable = false)
 |    |-- element: struct (containsNull = true)
 |    |    |-- _id: string (nullable = false)
 |    |    |-- _title: string (nullable = false)
 |    |    |-- _url: string (nullable = false)
 |    |    |-- text: struct (nullable = false)
 |    |    |    |-- sentence: array (nullable = false)
 |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |-- w: array (nullable = true)
 |    |    |    |    |    |    |-- element: string (containsNull = true)

xml_df.show(10)

+------+--------------------+--------------------+--------------------+
|   _id|              _title|                _url|              thread|
+------+--------------------+--------------------+--------------------+
|19-290|Allmänna rubriker...|http://www.familj...|[[70148929, Frivi...|
|19-290|Allmänna rubriker...|http://www.familj...|[[58302374, Missh...|
|19-290|Allmänna rubriker...|http://www.familj...|[[36330819, Är så...|
|19-290|Allmänna rubriker...|http://www.familj...|[[75852809, Hur k...|
|19-290|Allmänna rubriker...|http://www.familj...|[[42304381, Hej a...|
|19-290|Allmänna rubriker...|http://www.familj...|[[41294375, när p...|
|19-295|Allmänna rubriker...|http://www.familj...|[[47653437, Har v...|
|19-295|Allmänna rubriker...|http://www.familj...|[[75266317, Anmäl...|
|19-295|Allmänna rubriker...|http://www.familj...|[[76559817, Fel a...|
|19-295|Allmänna rubriker...|http://www.familj...|[[62028128, Vad g...|
+------+--------------------+--------------------+--------------------+
only showing top 10 rows

df.printSchema()

root
 |-- thread_id: string (nullable = true)
 |-- thread_title: string (nullable = true)
 |-- w: string (nullable = false)
 |-- forum_id: string (nullable = true)
 |-- forum_title: string (nullable = true)
 |-- platform: string (nullable = false)
 |-- corpus_id: string (nullable = false)

display(df)

//Imports
import org.apache.spark.ml.feature.StopWordsRemover
import org.apache.spark.ml.feature.RegexTokenizer
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.feature.CountVectorizer
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator

import org.apache.spark.ml.feature.StopWordsRemover
import org.apache.spark.ml.feature.RegexTokenizer
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.feature.CountVectorizer
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator

/scalable-data-science/000_0-sds-3-x-projects/student-project-01_group-TheTwoCultures/01_load_data

//Load dataframes
val file_path_familjeliv = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad_df"
val file_path_flashback = "dbfs:/datasets/student-project-01/flashback/flashback-sex_df"
val df_familjeliv = load_df(file_path_familjeliv)
val df_flashback = load_df(file_path_flashback)

file_path_familjeliv: String = dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad_df
file_path_flashback: String = dbfs:/datasets/student-project-01/flashback/flashback-sex_df
df_familjeliv: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]
df_flashback: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]

//Extract the text
val df_text_flashback = df_flashback.select("w")
val df_text_familjeliv = df_familjeliv.select("w")

df_text_flashback: org.apache.spark.sql.DataFrame = [w: string]
df_text_familjeliv: org.apache.spark.sql.DataFrame = [w: string]

//Add label columns and make sure that we have exactly the same amount of data from both forums
val df_text_flashback_c = df_text_flashback.withColumn("c", lit(0))
val df_text_familjeliv_c = df_text_familjeliv.orderBy(rand()).limit(df_text_flashback_c.count().toInt).withColumn("c", lit(1))
val df_text_full = df_text_flashback_c.union(df_text_familjeliv_c)

//Check the counts
println(df_text_flashback_c.count())
println(df_text_familjeliv_c.count())
println(df_text_full.count())

56621
56621
113242
df_text_flashback_c: org.apache.spark.sql.DataFrame = [w: string, c: int]
df_text_familjeliv_c: org.apache.spark.sql.DataFrame = [w: string, c: int]
df_text_full: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [w: string, c: int]

val tokenizer = new RegexTokenizer()
  .setPattern("(?U),") // break by whitespace
  .setMinTokenLength(5) // Filter away tokens with length < 5
  .setInputCol("w") // name of the input column
  .setOutputCol("text") // name of the output column
val tokenized_df = tokenizer.transform(df_text_full).select("c", "text")
tokenized_df.show(3, false)

//Stopwordsremover (similar to lda notebook)
val stoppord = sc.textFile("dbfs:/datasets/student-project-01/stoppord.csv").collect()
val stopwordList = Array("bara","lite","finns","vill","samt","inga","även","finns","ganska","också","igen","just","that","with","http","jpg",  "kanske","tycker","gillar","bra","000","måste","tjej","tjejer","tjejen","tjejerna","kvinna","kvinnor","kille","killar","killen","män","rätt","män","com","and","html","många","aldrig","www","mpg","avi","wmv","riktigt","känner","väldigt","font","size","mms","2008","2009", "flashback", "familjeliv").union(stoppord).union(StopWordsRemover.loadDefaultStopWords("swedish"))

val remover = new StopWordsRemover()
  .setStopWords(stopwordList)
  .setInputCol("text")
  .setOutputCol("filtered")

//Run the stopwordsremover
val removed_df = remover.transform(tokenized_df).select("c", "filtered")

removed_df: org.apache.spark.sql.DataFrame = [c: int, filtered: array<string>]

//Unlimited size vocabulary just to see how much there is
val vectorizerall = new CountVectorizer()
   .setInputCol("filtered")
   .setOutputCol("features")
   .setMinDF(5) // Only count words that occur in at least 5 threadss
   .fit(removed_df) // returns CountVectorizerModel

//This is the one we use, limit size of vocabulary
val vectorizer = new CountVectorizer()
   .setInputCol("filtered")
   .setOutputCol("features")
   .setVocabSize(1000) // Size of dictonary
   .setMinDF(5) // Only count words that occur in at least 5 threadss
   .fit(removed_df) // returns CountVectorizerModel

vectorizerall: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVec_e9a01f8ad0fe, vocabularySize=129204
vectorizer: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVec_9d51ff227f19, vocabularySize=1000

// Let's take a look at the vocabulary
vectorizer.vocabulary

// Count the word frequencies
val tf = vectorizer.transform(removed_df.select("c", "filtered")).select("c", "features").cache()

//Print the feature vector to show what it looks like
tf.take(1).foreach(println)

[0,(1000,[0,4,5,6,12,16,33,34,48,53,56,60,64,66,68,73,83,84,91,100,101,105,107,119,123,125,127,141,143,163,171,201,205,210,212,261,273,302,325,338,341,348,361,367,383,414,424,453,454,491,571,621,632,635,667,684,693,701,829,849,933,981],[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,2.0,2.0,1.0,1.0,3.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0])]
tf: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [c: int, features: vector]

//Train test split
val random_order = tf.orderBy(rand())
val splits = random_order.randomSplit(Array(0.8, 0.2), seed = 1337)
val training = splits(0)
val test = splits(1)
println(training.count())
println(test.count())

90818
22424
random_order: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [c: int, features: vector]
splits: Array[org.apache.spark.sql.Dataset[org.apache.spark.sql.Row]] = Array([c: int, features: vector], [c: int, features: vector])
training: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [c: int, features: vector]
test: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [c: int, features: vector]

//Logistic regression
val lr = new LogisticRegression()
  .setLabelCol("c")
  .setMaxIter(100) //Run for 100 iterations (not necessary but let's stay on safe side)
  .setRegParam(0.0001) //Just a tiny bit of regularization to avoid overfitting
  .setElasticNetParam(0.5) // 50-50 between L1 and L2 loss

// Fit the model
val lrModel = lr.fit(training)

// Print the coefficients and intercept for logistic regression
println(s"Coefficients: ${lrModel.coefficients} Intercept: ${lrModel.intercept}")

// Check the training progress
lrModel.binarySummary.objectiveHistory.foreach(loss => println(loss))

//Ugly code to lookup maximum and minimum values
var maxarray = Array.ofDim[Double](5,2)
def findmax(idx: Int, value: Double) = {
  if (value > maxarray(4)(1)){
    maxarray(4)(0) = idx
    maxarray(4)(1) = value
    maxarray = maxarray.sortBy(- _(1))
  }
}
var minarray = Array.ofDim[Double](5,2)
def findmin(idx: Int, value: Double) = {
  if (value < minarray(4)(1)){
    minarray(4)(0) = idx
    minarray(4)(1) = value
    minarray = minarray.sortBy(_(1))
  }
}

maxarray: Array[Array[Double]] = Array(Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0))
findmax: (idx: Int, value: Double)Unit
minarray: Array[Array[Double]] = Array(Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0), Array(0.0, 0.0))
findmin: (idx: Int, value: Double)Unit

//Let's check which words are considered most important for classification
lrModel.coefficients.foreachActive((idx, value) => findmax(idx, value))

//First check familjeliv
println(maxarray.deep.foreach(println))
println(vectorizer.vocabulary(maxarray(0)(0).toInt))
println(vectorizer.vocabulary(maxarray(1)(0).toInt))
println(vectorizer.vocabulary(maxarray(2)(0).toInt))
println(vectorizer.vocabulary(maxarray(3)(0).toInt))
println(vectorizer.vocabulary(maxarray(4)(0).toInt))
lrModel.coefficients.foreachActive((idx, value) => findmin(idx, value))

//Check for flashback
println(minarray.deep.foreach(println))
println(vectorizer.vocabulary(minarray(0)(0).toInt))
println(vectorizer.vocabulary(minarray(1)(0).toInt))
println(vectorizer.vocabulary(minarray(2)(0).toInt))
println(vectorizer.vocabulary(minarray(3)(0).toInt))
println(vectorizer.vocabulary(minarray(4)(0).toInt))

Array(46.0, 1.8647655096872564)
Array(885.0, 1.275281418044729)
Array(950.0, 1.224376631679196)
Array(380.0, 0.9577079595234373)
Array(32.0, 0.8432880748567715)
()
anonym
förlossningen
maken
sambon
sambo
Array(990.0, -2.2314223680319945)
Array(664.0, -1.8291269454715258)
Array(857.0, -1.4232104863197035)
Array(275.0, -1.3427561053936439)
Array(173.0, -1.1857533047141897)
()
topic
bruden
vafan
brudar
jävligt

val predictions = lrModel.transform(test)
predictions.orderBy(rand()).select("c", "prediction", "probability").show(30, false)

+---+----------+-------------------------------------------+
|c  |prediction|probability                                |
+---+----------+-------------------------------------------+
|0  |0.0       |[0.9721401017870042,0.027859898212995764]  |
|0  |0.0       |[0.975623998737009,0.02437600126299096]    |
|1  |1.0       |[4.6730789993111417E-7,0.9999995326921002] |
|0  |0.0       |[0.933249707175278,0.066750292824722]      |
|0  |0.0       |[0.9902085789245901,0.009791421075409966]  |
|0  |0.0       |[0.5279677569376853,0.47203224306231467]   |
|0  |0.0       |[0.9932461412304279,0.00675385876957205]   |
|1  |1.0       |[2.43269453308815E-5,0.9999756730546691]   |
|1  |1.0       |[8.266454051870882E-10,0.9999999991733546] |
|0  |0.0       |[0.9997151003194746,2.8489968052548283E-4] |
|1  |0.0       |[0.5514931570249911,0.44850684297500887]   |
|0  |0.0       |[0.5858664716477586,0.41413352835224143]   |
|1  |1.0       |[0.002100566198113697,0.9978994338018863]  |
|1  |1.0       |[0.07917634407205193,0.920823655927948]    |
|0  |0.0       |[0.9970675008521647,0.0029324991478353007] |
|0  |0.0       |[0.9999595461915014,4.045380849869759E-5]  |
|0  |1.0       |[0.33337692071405434,0.6666230792859457]   |
|1  |1.0       |[0.36761800025826114,0.6323819997417389]   |
|1  |1.0       |[0.3245585295503879,0.6754414704496121]    |
|1  |1.0       |[0.2355899833856519,0.7644100166143482]    |
|0  |0.0       |[0.9999999999997755,2.2452150253864004E-13]|
|1  |1.0       |[0.18608690603389255,0.8139130939661074]   |
|0  |0.0       |[0.740890026139782,0.25910997386021795]    |
|0  |0.0       |[0.963586227883629,0.036413772116371014]   |
|1  |1.0       |[0.0021508873399861557,0.9978491126600139] |
|1  |1.0       |[0.3858439417926455,0.6141560582073544]    |
|1  |1.0       |[0.4517753939274335,0.5482246060725665]    |
|1  |1.0       |[0.02573645474447229,0.9742635452555276]   |
|0  |0.0       |[0.8022052550237544,0.19779474497624555]   |
|1  |1.0       |[0.042382658471976975,0.9576173415280229]  |
+---+----------+-------------------------------------------+
only showing top 30 rows

predictions: org.apache.spark.sql.DataFrame = [c: int, features: vector ... 3 more fields]

//Check auroc value
val evaluator = new BinaryClassificationEvaluator().setLabelCol("c")
evaluator.evaluate(predictions)

evaluator: org.apache.spark.ml.evaluation.BinaryClassificationEvaluator = BinaryClassificationEvaluator: uid=binEval_0812f13ed2be, metricName=areaUnderROC, numBins=1000
res19: Double = 0.928445521562674

// import required libraries
import org.apache.spark.ml.feature.{Word2Vec,Word2VecModel}
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.sql.Row
import org.apache.spark.ml.feature.RegexTokenizer
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator

import org.apache.spark.ml.feature.{Word2Vec, Word2VecModel}
import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.sql.Row
import org.apache.spark.ml.feature.RegexTokenizer
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator

/scalable-data-science/000_0-sds-3-x-projects/student-project-01_group-TheTwoCultures/01_load_data

// process .xml-files
val file_name = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad.xml"
val df = get_dataset(file_name)
val file_name2 = "dbfs:/datasets/student-project-01/flashback/flashback-sex.xml"
val df2 = get_dataset(file_name2)

// paths to saved dataframes
val file_path_familjeliv = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad_df" 
val file_path_flashback = "dbfs:/datasets/student-project-01/flashback/flashback-sex_df"

// load saved data frame
val df_familjeliv = load_df(file_path_familjeliv)
val df_flashback = load_df(file_path_flashback)

file_path_familjeliv: String = dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad_df
file_path_flashback: String = dbfs:/datasets/student-project-01/flashback/flashback-sex_df
df_familjeliv: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]
df_flashback: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]

display(df_familjeliv)

val df = df_flashback.withColumn("c", lit(0.0)).union(df_familjeliv.withColumn("c", lit(1.0)))

df: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 6 more fields]

// define the tokenizer
val tokenizer = new RegexTokenizer()
  .setPattern("(?U),") // break by commas
  .setMinTokenLength(5) // Filter away tokens with length < 5
  .setInputCol("w") // name of the input column
  .setOutputCol("text") // name of the output column

tokenizer: org.apache.spark.ml.feature.RegexTokenizer = regexTok_f0701eaf4f60

// define the thread title tokenizer

val df_tokenized = tokenizer.transform(df)
display(df_tokenized.select("w","text"))

// define the model
val word2Vec = new Word2Vec()
  .setInputCol("text")
  .setOutputCol("result")
  .setVectorSize(200)
  .setMinCount(0)

word2Vec: org.apache.spark.ml.feature.Word2Vec = w2v_945abe6fab57

// train it
val word2Vec_model = word2Vec.fit(df_tokenized)

// save it
word2Vec_model.save("dbfs:/datasets/student-project-01/word2vec_model_sex")

// load a saved model

val model = Word2VecModel.load("dbfs:/datasets/student-project-01/word2vec_model_sex")

model: org.apache.spark.ml.feature.Word2VecModel = w2v_854b46dceacc

// transform the text using the model

val embedded_text = model.transform(df_tokenized)

embedded_text: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 8 more fields]

display(embedded_text.select("c","result","text"))

// Logistic regression
val logreg = new LogisticRegression()
  .setLabelCol("c")
  .setFeaturesCol("result")
  .setMaxIter(100)
  .setRegParam(0.0001)
  .setElasticNetParam(0.5)

logreg: org.apache.spark.ml.classification.LogisticRegression = logreg_55ef614f783e

val pipeline = new Pipeline().setStages(Array(tokenizer, word2Vec, logreg))

pipeline: org.apache.spark.ml.Pipeline = pipeline_2254409a91a2

val random_order = df.orderBy(rand())
val splits = random_order.randomSplit(Array(0.8, 0.2))
val training = splits(0)
val test = splits(1)

random_order: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 6 more fields]
splits: Array[org.apache.spark.sql.Dataset[org.apache.spark.sql.Row]] = Array([thread_id: string, thread_title: string ... 6 more fields], [thread_id: string, thread_title: string ... 6 more fields])
training: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 6 more fields]
test: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 6 more fields]

// fit the model to the training data
val logreg_model = pipeline.fit(training)

// save the model to filesystem
logreg_model.save("dbfs:/datasets/student-project-01/word2vec_logreg_model")

// load saved model

val loaded_model = PipelineModel.load("dbfs:/datasets/student-project-01/word2vec_logreg_model")

loaded_model: org.apache.spark.ml.PipelineModel = pipeline_25839437fccb

val predictions = loaded_model.transform(test).orderBy(rand())

predictions: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [thread_id: string, thread_title: string ... 11 more fields]

predictions.select("c","prediction","probability").show(30,false)

+---+----------+------------------------------------------+
|c  |prediction|probability                               |
+---+----------+------------------------------------------+
|1.0|1.0       |[0.005855361335036372,0.9941446386649635] |
|1.0|1.0       |[0.2712120894396273,0.7287879105603726]   |
|1.0|1.0       |[0.0017886928649958375,0.9982113071350042]|
|1.0|1.0       |[2.263165652125581E-4,0.9997736834347875] |
|0.0|1.0       |[0.42059601285820825,0.5794039871417918]  |
|1.0|1.0       |[6.566687042616189E-4,0.9993433312957383] |
|0.0|0.0       |[0.7054463412596114,0.29455365874038864]  |
|1.0|1.0       |[0.03103196407369915,0.9689680359263009]  |
|0.0|0.0       |[0.9294663779954874,0.07053362200451263]  |
|1.0|1.0       |[0.13974006800394764,0.8602599319960523]  |
|1.0|1.0       |[0.08228914085494436,0.9177108591450557]  |
|0.0|0.0       |[0.9788989176701534,0.021101082329846567] |
|0.0|0.0       |[0.9975070728891363,0.0024929271108637065]|
|1.0|1.0       |[0.0010781075297480556,0.998921892470252] |
|0.0|0.0       |[0.9253825302681451,0.07461746973185476]  |
|1.0|1.0       |[0.01751495449884683,0.9824850455011531]  |
|1.0|0.0       |[0.9864736560167631,0.013526343983237045] |
|1.0|1.0       |[0.002472519507918196,0.9975274804920817] |
|0.0|0.0       |[0.6174112612306129,0.3825887387693872]   |
|0.0|0.0       |[0.7130899106721519,0.2869100893278482]   |
|0.0|0.0       |[0.9263664682801233,0.07363353171987672]  |
|0.0|0.0       |[0.9561455191484204,0.04385448085157954]  |
|0.0|0.0       |[0.5835745861693306,0.41642541383066944]  |
|1.0|1.0       |[0.4296249407516458,0.5703750592483542]   |
|1.0|1.0       |[0.0032969395487662213,0.9967030604512337]|
|1.0|1.0       |[0.008645133666934816,0.9913548663330651] |
|1.0|1.0       |[4.1492836709996625E-5,0.9999585071632902]|
|0.0|1.0       |[0.43037903982909986,0.5696209601709002]  |
|0.0|1.0       |[0.43707897641990706,0.562921023580093]   |
|1.0|1.0       |[0.29846393214228517,0.7015360678577148]  |
+---+----------+------------------------------------------+
only showing top 30 rows

val evaluator = new BinaryClassificationEvaluator().setLabelCol("c")
evaluator.evaluate(predictions)

evaluator: org.apache.spark.ml.evaluation.BinaryClassificationEvaluator = binEval_0d518432d17a
res13: Double = 0.9499399325125091

val df_threads = df.select("c","thread_title").withColumnRenamed("thread_title","w")
val evaluation = loaded_model.transform(df_threads).orderBy(rand())
evaluator.evaluate(evaluation)

df_threads: org.apache.spark.sql.DataFrame = [c: double, w: string]
evaluation: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [c: double, w: string ... 5 more fields]
res14: Double = 0.5261045467524708

/scalable-data-science/000_0-sds-3-x-projects/student-project-01_group-TheTwoCultures/00_download_data

// Here we just import methods from notebook 01_load_data, so we can load our data

%run /scalable-data-science/000_0-sds-3-x-projects/student-project-01_group-TheTwoCultures/01_load_data

import org.apache.spark.ml.feature.{RegexTokenizer, StopWordsRemover, CountVectorizer}
import org.apache.spark.mllib.clustering.{LDA, OnlineLDAOptimizer}
import org.apache.spark.ml.{Pipeline, PipelineModel, PipelineStage}
import org.apache.spark.sql.types.{ LongType }
import org.apache.spark.ml.linalg.Vector

def get_lda(df: org.apache.spark.sql.DataFrame,  // Main function to do LDA
            min_token: Int,
            vocabSize: Int,
            minDF: Int,            
            numTopics: Int,
            maxIter: Int,
            stopWords: Array[String]) = {

  val corpus_df = df.select("w", "thread_id").withColumn("thread_id",col("thread_id").cast(LongType)) // From the whole dataframe we take out each thread

  val tokenizer = new RegexTokenizer()
  .setPattern("(?U)[\\W_]+") // break by white space character(s)  - try to remove emails and other patterns
  .setMinTokenLength(min_token) // Filter away tokens with length < min_token
  .setInputCol("w") // name of the input column
  .setOutputCol("tokens") // name of the output column

  val remover = new StopWordsRemover()
  .setStopWords(stopWords)
  .setInputCol(tokenizer.getOutputCol)
  .setOutputCol("filtered")

  val tokenized_df = tokenizer.transform(corpus_df) // Removes uninteresting words from each thread
  val filtered_df = remover.transform(tokenized_df).select("thread_id","filtered") // Chosing only the filtered threads

  val vectorizer = new CountVectorizer() // Creates dictionary and counts the occurences of different words
   .setInputCol(remover.getOutputCol)
   .setOutputCol("features")
   .setVocabSize(vocabSize) // Size of dictonary
   .setMinDF(minDF)
   .fit(filtered_df) // returns CountVectorizerModel

  val lda = new LDA() // Creates LDA Model with some user defined choice of parameters
  .setOptimizer(new OnlineLDAOptimizer().setMiniBatchFraction(0.8))
  .setK(numTopics)
  .setMaxIterations(maxIter)
  .setDocConcentration(-1) // use default values
  .setTopicConcentration(-1)

  val countVectors = vectorizer.transform(filtered_df).select("thread_id", "features")

  val lda_countVector = countVectors.map { case Row(id: Long, countVector: Vector) => (id, countVector) }
  val lda_countVector_mllib = lda_countVector.map { case (id, vector) => (id, org.apache.spark.mllib.linalg.Vectors.fromML(vector)) }.rdd

  val lda_model = lda.run(lda_countVector_mllib)
  (lda_model, vectorizer)
}

import org.apache.spark.ml.feature.{RegexTokenizer, StopWordsRemover, CountVectorizer}
import org.apache.spark.mllib.clustering.{LDA, OnlineLDAOptimizer}
import org.apache.spark.ml.{Pipeline, PipelineModel, PipelineStage}
import org.apache.spark.sql.types.LongType
import org.apache.spark.ml.linalg.Vector
get_lda: (df: org.apache.spark.sql.DataFrame, min_token: Int, vocabSize: Int, minDF: Int, numTopics: Int, maxIter: Int, stopWords: Array[String])(org.apache.spark.mllib.clustering.LDAModel, org.apache.spark.ml.feature.CountVectorizerModel)

wget https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv -O /tmp/stopwords.csv

--2021-01-11 14:30:09--  https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.52.133
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.52.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1936 (1.9K) [text/plain]
Saving to: ‘/tmp/stopwords.csv’

     0K .                                                     100% 11.5M=0s

2021-01-11 14:30:09 (11.5 MB/s) - ‘/tmp/stopwords.csv’ saved [1936/1936]

cp file:/tmp/stopwords.csv dbfs:/datasets/student-project-01/stopwords.csv

res24: Boolean = true

//Creating a list with stopwords

import org.apache.spark.ml.feature.StopWordsRemover
val stoppord = sc.textFile("dbfs:/datasets/student-project-01/stopwords.csv").collect()
val stopwordList = Array("bara","lite","finns","vill","samt","inga","även","finns","ganska","också","igen","just","that","with","http","jpg","kanske","tycker","gillar", "bra","the","000","måste","tjej","tjejer","tjejen","tjejerna","kvinna","kvinnor","kille","killar","killen","män","rätt","män","com","and","html","många","aldrig","www","mpg","avi","wmv","fan","förhelvetet","riktigt","känner","väldigt","font","size","mms","2008","2009","95kr","dom","får","ska","kommer","två","vet","mer","pengar","pengarna","göra","fick","tror","andra","helt","kunna","behöver","betala","inget","dock","inget","tack"
).union(stoppord).union(StopWordsRemover.loadDefaultStopWords("swedish")) // In this step we add custom stop words and another premade list of stopwords.

// Loading the forums we will do LDA on.

val file_path_FL = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-allmanna-ekonomi_df"
val file_path_FB = "dbfs:/datasets/student-project-01/flashback/flashback-ekonomi_df"
//val file_path_FL = "dbfs:/datasets/student-project-01/familjeliv/familjeliv-sexsamlevnad_df"
//val file_path_FB = "dbfs:/datasets/student-project-01/flashback/flashback-sex_df"

val df_FL = load_df(file_path_FL)
val df_FB = load_df(file_path_FB)

file_path_FL: String = dbfs:/datasets/student-project-01/familjeliv/familjeliv-allmanna-ekonomi_df
file_path_FB: String = dbfs:/datasets/student-project-01/flashback/flashback-ekonomi_df
df_FL: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]
df_FB: org.apache.spark.sql.DataFrame = [thread_id: string, thread_title: string ... 5 more fields]

// Dataframes containg a forum from FB.

df_FB.show

// Overview of number of threads in different subforums

df_FL.groupBy($"forum_title").agg(count($"forum_title").as("count")).show(false)

+----------------------------------------------------------+-----+
|forum_title                                               |count|
+----------------------------------------------------------+-----+
|Allmänna rubriker > Ekonomi & juridik > Konsument/Inhandla|26307|
|Allmänna rubriker > Ekonomi & juridik > Familjerätt       |1623 |
|Allmänna rubriker > Ekonomi & juridik > Övrigt            |1638 |
|Allmänna rubriker > Ekonomi & juridik > Ekonomi           |24284|
|Allmänna rubriker > Ekonomi & juridik > Spartips          |217  |
|Allmänna rubriker > Ekonomi & juridik > Juridik           |3246 |
|Allmänna rubriker > Ekonomi & juridik > Företagande       |957  |
|Allmänna rubriker > Ekonomi & juridik > Lån & skulder     |1168 |
+----------------------------------------------------------+-----+

df_FB.groupBy($"forum_title").agg(count($"forum_title").as("count")).show(false)

+---------------------------------------------------+-----+
|forum_title                                        |count|
+---------------------------------------------------+-----+
|Ekonomi > Privatekonomi                            |15154|
|Ekonomi > Bitcoin och andra virtuella valutor      |524  |
|Ekonomi > Värdepapper, valutor och råvaror: allmänt|8300 |
|Ekonomi > Fonder                                   |645  |
|Ekonomi > Företagande och företagsekonomi          |12139|
|Ekonomi > Nationalekonomi                          |3114 |
|Ekonomi > Aktier                                   |508  |
|Ekonomi > Ekonomi: övrigt                          |19308|
|Ekonomi > Offshore och skatteplanering             |1567 |
+---------------------------------------------------+-----+

// Set parameters, and perform two different LDAs, one on FB and one on FL

val min_token = 4 // only accept tokens larger or equal to
val vocabSize = 10000 // Size of the vocab dictionary
val minDF = 5 // The minimum number of different documents a term must appear in to be included in the vocabulary
val minTF = 2 // The minimum number of times a term has to appear in a document to be included in the vocabulary

val numTopics_FL = no_forums(df_FL).toInt //4 // Number of topics in LDA model
val numTopics_FB = no_forums(df_FB).toInt
val maxIter = 8  // Maximum number of iterations for LDA model
val stopwords = stopwordList
val (ldaModel_FL, vectorizer_FL) = get_lda(df_FL, min_token, vocabSize, minDF, numTopics_FL, maxIter, stopwords)
val (ldaModel_FB, vectorizer_FB) = get_lda(df_FB, min_token, vocabSize, minDF, numTopics_FB, maxIter, stopwords)

// Here we pick out the most important terms in each topic for FL.

val topicIndices = ldaModel_FL.describeTopics(maxTermsPerTopic = 5)
val vocabList = vectorizer_FL.vocabulary
val topics = topicIndices.map { case (terms, termWeights) =>
  terms.map(vocabList(_)).zip(termWeights)
}
println(s"$numTopics_FL topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
  println(s"TOPIC $i")
  topic.foreach { case (term, weight) => println(s"$term\t$weight") }
  println(s"==========")
}

// Here we pick out the most important terms in each topic for FB.

val topicIndices = ldaModel_FB.describeTopics(maxTermsPerTopic = 5)
val vocabList = vectorizer_FB.vocabulary
val topics = topicIndices.map { case (terms, termWeights) =>
  terms.map(vocabList(_)).zip(termWeights)
}
println(s"$numTopics_FB topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
  println(s"TOPIC $i")
  topic.foreach { case (term, weight) => println(s"$term\t$weight") }
  println(s"==========")
}

// Zip topic terms with topic IDs
val termArray = topics.zipWithIndex

// Transform data into the form (term, probability, topicId)
val termRDD = sc.parallelize(termArray)
val termRDD2 =termRDD.flatMap( (x: (Array[(String, Double)], Int)) => {
  val arrayOfTuple = x._1
  val topicId = x._2
  arrayOfTuple.map(el => (el._1, el._2, topicId))
})

// Create DF with proper column names
val termDF = termRDD2.toDF.withColumnRenamed("_1", "term").withColumnRenamed("_2", "probability").withColumnRenamed("_3", "topicId")

display(termDF)

// Creates JSON data to display topic distribution of forum in FB
val rawJson = termDF.toJSON.collect().mkString(",\n")

displayHTML(s"""
<!DOCTYPE html>
<meta charset="utf-8">
<style>

circle {
  fill: rgb(31, 119, 180);
  fill-opacity: 0.5;
  stroke: rgb(31, 119, 180);
  stroke-width: 1px;
}

.leaf circle {
  fill: #ff7f0e;
  fill-opacity: 1;
}

text {
  font: 14px sans-serif;
}

</style>
<body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script>

var json = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson}
     ]
    }
   ]
};

var r = 1000,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble = d3.layout.pack()
    .sort(null)
    .size([r, r])
    .padding(1.5);

var vis = d3.select("body").append("svg")
    .attr("width", r)
    .attr("height", r)
    .attr("class", "bubble");


var node = vis.selectAll("g.node")
    .data(bubble.nodes(classes(json))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });

// Returns a flattened hierarchy containing all leaf nodes under the root.
function classes(root) {
  var classes = [];

  function recurse(term, node) {
    if (node.children) node.children.forEach(function(child) { recurse(node.term, child); });
    else classes.push({topicName: node.topicId, className: node.term, value: node.probability});
  }

  recurse(null, root);
  return {children: classes};
}
</script>
""")

// Similar as above but for FL

val topicIndices = ldaModel_FL.describeTopics(maxTermsPerTopic = 6)
val vocabList = vectorizer_FL.vocabulary
val topics = topicIndices.map { case (terms, termWeights) =>
  terms.map(vocabList(_)).zip(termWeights)
}
println(s"$numTopics_FL topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
  println(s"TOPIC $i")
  topic.foreach { case (term, weight) => println(s"$term\t$weight") }
  println(s"==========")
}

// Zip topic terms with topic IDs
val termArray = topics.zipWithIndex

// Transform data into the form (term, probability, topicId)
val termRDD = sc.parallelize(termArray)
val termRDD2 =termRDD.flatMap( (x: (Array[(String, Double)], Int)) => {
  val arrayOfTuple = x._1
  val topicId = x._2
  arrayOfTuple.map(el => (el._1, el._2, topicId))
})

// Create DF with proper column names
val termDF = termRDD2.toDF.withColumnRenamed("_1", "term").withColumnRenamed("_2", "probability").withColumnRenamed("_3", "topicId")

// Create JSON data to display topic distribution of forum in FB
val rawJson = termDF.toJSON.collect().mkString(",\n")

displayHTML(s"""
<!DOCTYPE html>
<meta charset="utf-8">
<style>

circle {
  fill: rgb(31, 119, 180);
  fill-opacity: 0.5;
  stroke: rgb(31, 119, 180);
  stroke-width: 1px;
}

.leaf circle {
  fill: #ff7f0e;
  fill-opacity: 1;
}

text {
  font: 14px sans-serif;
}

</style>
<body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script>

var json = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson}
     ]
    }
   ]
};

var r = 1000,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble = d3.layout.pack()
    .sort(null)
    .size([r, r])
    .padding(1.5);

var vis = d3.select("body").append("svg")
    .attr("width", r)
    .attr("height", r)
    .attr("class", "bubble");


var node = vis.selectAll("g.node")
    .data(bubble.nodes(classes(json))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });

// Returns a flattened hierarchy containing all leaf nodes under the root.
function classes(root) {
  var classes = [];

  function recurse(term, node) {
    if (node.children) node.children.forEach(function(child) { recurse(node.term, child); });
    else classes.push({topicName: node.topicId, className: node.term, value: node.probability});
  }

  recurse(null, root);
  return {children: classes};
}
</script>
""")

def rawJson(lda_model: org.apache.spark.mllib.clustering.LDAModel, vectorizer: org.apache.spark.ml.feature.CountVectorizerModel): String = {
  val topicIndices = lda_model.describeTopics(maxTermsPerTopic = 6)
  val vocabList = vectorizer.vocabulary
  val topics = topicIndices.map { case (terms, termWeights) =>
    terms.map(vocabList(_)).zip(termWeights)
  }
  println(s"$numTopics_FL topics:")
  topics.zipWithIndex.foreach { case (topic, i) =>
    println(s"TOPIC $i")
    topic.foreach { case (term, weight) => println(s"$term\t$weight") }
    println(s"==========")
  }

  // Zip topic terms with topic IDs
  val termArray = topics.zipWithIndex

  // Transform data into the form (term, probability, topicId)
  val termRDD = sc.parallelize(termArray)
  val termRDD2 =termRDD.flatMap( (x: (Array[(String, Double)], Int)) => {
    val arrayOfTuple = x._1
    val topicId = x._2
    arrayOfTuple.map(el => (el._1, el._2, topicId))
  })

  // Create DF with proper column names
  val termDF = termRDD2.toDF.withColumnRenamed("_1", "term").withColumnRenamed("_2", "probability").withColumnRenamed("_3", "topicId")

  // Create JSON data to display topic distribution
  termDF.toJSON.collect().mkString(",\n")
}
// Create JSON data to display topic distribution
val rawJson_FL = rawJson(ldaModel_FL, vectorizer_FL)
val rawJson_FB = rawJson(ldaModel_FB, vectorizer_FB)

displayHTML(s"""
<!DOCTYPE html>
<meta charset="utf-8">
<style>

circle {
  fill: rgb(31, 119, 180);
  fill-opacity: 0.5;
  stroke: rgb(31, 119, 180);
  stroke-width: 1px;
}

.leaf circle {
  fill: #ff7f0e;
  fill-opacity: 1;
}

text {
  font: 12px sans-serif;
}

</style>
<body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script>

/////////////////////////////////////////////////////////////////////////////   
var json_FL = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson_FL}
     ]
    }
   ]
};

var r1 = 500,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble1 = d3.layout.pack()
    .sort(null)
    .size([r1, r1])
    .padding(1.5);

var vis1 = d3.select("body").append("svg")
    .attr("width", r1)
    .attr("height", r1)
    .attr("class", "bubble1");


var node = vis1.selectAll("g.node")
    .data(bubble1.nodes(classes(json_FL))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });
/////////////////////////////////////////////////////////////////////////////   
var json_FB = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson_FB}
     ]
    }
   ]
};

var r2 = 500,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble2 = d3.layout.pack()
    .sort(null)
    .size([r2, r2])
    .padding(1.5);

var vis2 = d3.select("body").append("svg")
    .attr("width", r2)
    .attr("height", r2)
    .attr("class", "bubble1");


var node = vis2.selectAll("g.node")
    .data(bubble1.nodes(classes(json_FB))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });

/////////////////////////////////////////////////////////////////////////////   

// Returns a flattened hierarchy containing all leaf nodes under the root.
function classes(root) {
  var classes = [];

  function recurse(term, node) {
    if (node.children) node.children.forEach(function(child) { recurse(node.term, child); });
    else classes.push({topicName: node.topicId, className: node.term, value: node.probability});
  }

  recurse(null, root);
  return {children: classes};
}
</script>

""")

sc = spark.sparkContext
# Had to change weather 'none' to '"none"' for the string to parse
json_example_str = '{"2407890": {"width": 640,"height": 480,"location": "living room","weather": "none","objects": {"271881": {"name": "chair","x": 220,"y": 310,"w": 50,"h": 80,"attributes": ["brown", "wooden", "small"],"relations": {"32452": {"name": "on","object": "275312"},"32452": {"name": "near","object": "279472"}}}}}}'
json_rdd = sc.parallelize([json_example_str])
example_json_df = spark.read.json(json_rdd, multiLine=True)
example_json_df.show()

+--------------------+
|             2407890|
+--------------------+
|[480, living room...|
+--------------------+

example_json_df.first()

from graphframes import *
import json

# load train and validation graph data:
f_train = open("/dbfs/FileStore/shared_uploads/scenegraph_motifs/train_sceneGraphs.json")
train_scene_data = json.load(f_train)

f_val = open("/dbfs/FileStore/shared_uploads/scenegraph_motifs/val_sceneGraphs.json")
val_scene_data = json.load(f_val)

# Pythonic way of doing it, parsing a JSON graph representation.
# Creates vertices with the graph id, object name and id, optionally includes the attibutes
def json_to_vertices_edges(graph_json, scene_graph_id, include_object_attributes=False):
  vertices = []
  edges = []
  obj_id_to_name = {}

  vertex_ids = graph_json['objects']

  for vertex_id in vertex_ids:   
    vertex_obj = graph_json['objects'][vertex_id]
    name = vertex_obj['name']
    vertices_data = [scene_graph_id, vertex_id, name]

    if vertex_id not in obj_id_to_name:
      obj_id_to_name[vertex_id] = name

    if include_object_attributes:
      attributes = vertex_obj['attributes']  
      vertices_data.append(attributes)

    vertices.append(tuple(vertices_data))

    for relation in vertex_obj['relations']:
        src = vertex_id
        dst = relation['object']
        name = relation['name']
        edges.append([src, dst, name])

  for i in range(len(edges)):
    src_type = obj_id_to_name[edges[i][0]]
    dst_type = obj_id_to_name[edges[i][1]]
    edges[i].append(src_type)
    edges[i].append(dst_type)

  return (vertices, edges)

def parse_scene_graphs(scene_graphs_json, vertex_schema, edge_schema):  
  vertices = []
  edges = []

  # if vertice_schema has a field for attributes:
  include_object_attributes = len(vertex_schema) == 4

  for scene_graph_id in scene_graphs_json:
    vs, es = json_to_vertices_edges(scene_graphs_json[scene_graph_id], scene_graph_id, include_object_attributes)
    vertices += vs
    edges += es

  vertices = spark.createDataFrame(vertices, vertex_schema)
  edges = spark.createDataFrame(edges, edge_schema)

  return GraphFrame(vertices, edges)

from pyspark.sql.types import StructType, StructField, ArrayType, IntegerType, StringType

# create schemas for scene graphs:
vertex_schema = StructType([
  StructField("graph_id", StringType(), False), StructField("id", StringType(), False), StructField("object_name", StringType(), False)
])

vertex_schema_with_attr  = StructType([
  StructField("graph_id", StringType(), False),
  StructField("id", StringType(), False),
  StructField("object_name", StringType(), False),
  StructField("attributes", ArrayType(StringType()), True)
])

edge_schema = StructType([
  StructField("src", StringType(), False), StructField("dst", StringType(), False), StructField("relation_name", StringType(), False),
  StructField("src_type", StringType(), False), StructField("dst_type", StringType(), False)
])

# we will use the length of the vertice schemas to parse the graph from the json files appropriately:
len(vertex_schema), len(vertex_schema_with_attr)

scene_graphs_train = parse_scene_graphs(train_scene_data, vertex_schema_with_attr, edge_schema)

scene_graphs_train_without_attributes = GraphFrame(scene_graphs_train.vertices.select('graph_id', 'id', 'object_name'), scene_graphs_train.edges)

scene_graphs_val = parse_scene_graphs(val_scene_data, vertex_schema_with_attr, edge_schema)

display(scene_graphs_train.vertices)

display(scene_graphs_val.vertices)

display(scene_graphs_train.edges)

display(scene_graphs_val.edges)

# person next-to banana (yellow, small) vs person next-to banana (green)
display(scene_graphs_train.find('(a)-[ab]->(b)').filter("(a.object_name = 'person') and (b.object_name = 'banana')"))

# grouped_graphs = scene_graphs_train.vertices.groupBy('graph_id')
# display(grouped_graphs.count().sort('count', ascending=False))

print("Graphs/Scenes/Images: {}".format(scene_graphs_train.vertices.select('graph_id').distinct().count()))
print("Objects: {}".format(scene_graphs_train.vertices.count()))
print("Relations: {}".format(scene_graphs_train.edges.count()))

Graphs/Scenes/Images: 74289
Objects: 1231134
Relations: 3795907

display(scene_graphs_train.degrees.sort(["degree"],ascending=[0]).limit(20))

from pyspark.sql.functions import explode
# the attributes are sequences: we need to split them;
# explode the attributes in the vertices graph:
explodedAttributes = scene_graphs_train.vertices.select("id", "object_name", explode(scene_graphs_train.vertices.attributes).alias("attribute"))
explodedAttributes.printSchema()
display(explodedAttributes)

topAttributes = explodedAttributes.groupBy("attribute")
display(topAttributes.count().sort("count", ascending=False))

topAttributes = explodedAttributes.groupBy("attribute")
display(topAttributes.count().sort("count", ascending=False))

topObjects = scene_graphs_train.vertices.groupBy("object_name")
topObjects = topObjects.count()
display(topObjects.sort("count", ascending=False))

display(topObjects.sort("count", ascending=True))

topPairs = scene_graphs_train.edges.groupBy("src_type", "dst_type")
display(topPairs.count().sort("count", ascending=False))

topPairs = scene_graphs_train.edges.groupBy("src_type", "relation_name", "dst_type")
display(topPairs.count().sort("count", ascending=False))

topPairs = scene_graphs_train.edges.groupBy("relation_name")
display(topPairs.count().sort("count", ascending=False))

# Also possible to do:
# from pyspark.sql.functions import udf
#from pyspark.sql.types import BooleanType

#filtered_df = spark_df.filter(udf(lambda target: target.startswith('good'),
#                                  BooleanType())(spark_df.target))

topPairs = scene_graphs_train.edges.filter("(relation_name NOT LIKE 'to the%') and (relation_name NOT LIKE '%on') and (relation_name NOT LIKE '%in') and (relation_name NOT LIKE '% of')").groupBy("src_type", "relation_name", "dst_type")
display(topPairs.count().sort("count", ascending=False))

scene_graphs_train_without_attributes_graphid = GraphFrame(scene_graphs_train_without_attributes.vertices.select('id', 'object_name'), scene_graphs_train_without_attributes.edges)

motifs = scene_graphs_train_without_attributes_graphid.find("(a)-[ab]->(b); (b)-[bc]->(c)").filter("(a.object_name NOT LIKE b.object_name) and (a.object_name NOT LIKE c.object_name)")

display(motifs)

motifs_sorted = motifs.distinct()
display(motifs_sorted)

motifs_sorted.count()

circular_motifs = scene_graphs_train.find("(a)-[ab]->(b); (b)-[bc]->(c); (c)-[ca]->(a)")

display(circular_motifs)

circular_motifs.count()

circular_motifs = scene_graphs_train.find("(a)-[ab]->(b); (b)-[bc]->(c); (c)-[ca]->(a)").filter("(ab.relation_name NOT LIKE 'to the%') and (bc.relation_name NOT LIKE 'to the%') and (ca.relation_name NOT LIKE 'to the%') and (ab.relation_name NOT LIKE '% of') and (bc.relation_name NOT LIKE '% of') and (ca.relation_name NOT LIKE '% of')")

display(circular_motifs.select('ab', 'bc', 'ca'))

circular_motifs.count()

symmetric_motifs = scene_graphs_train.find("(a)-[ab]->(b); (b)-[ba]->(a)").filter("ab.relation_name LIKE ba.relation_name")

display(symmetric_motifs)

symmetric_motifs.count()

symmetric_motifs = scene_graphs_train.find("(a)-[ab]->(b); (b)-[ba]->(a)").filter("ab.relation_name LIKE ba.relation_name").filter("(ab.relation_name NOT LIKE 'near') and (ab.relation_name NOT LIKE '% of')")

display(symmetric_motifs.select('ab', 'ba'))

symmetric_motifs.count()

temp = GraphFrame(scene_graphs_train.vertices.select('graph_id', 'id', 'object_name'), scene_graphs_train.edges)
temp.vertices.write.parquet("/tmp/motif-vertices")
temp.edges.write.parquet("/tmp/motif-edges")

temp = GraphFrame(scene_graphs_val.vertices.select('graph_id', 'id', 'object_name'), scene_graphs_val.edges)#
temp.vertices.write.parquet("/tmp/motif-vertices-val")
temp.edges.write.parquet("/tmp/motif-edges-val")

import org.graphframes.{examples,GraphFrame}

val vertices = sqlContext.read.parquet("/tmp/motif-vertices")
val edges = sqlContext.read.parquet("/tmp/motif-edges")
val rank_graph = GraphFrame(vertices, edges)

import org.graphframes.{examples, GraphFrame}
vertices: org.apache.spark.sql.DataFrame = [graph_id: string, id: string ... 1 more field]
edges: org.apache.spark.sql.DataFrame = [src: string, dst: string ... 3 more fields]
rank_graph: org.graphframes.GraphFrame = GraphFrame(v:[id: string, graph_id: string ... 1 more field], e:[src: string, dst: string ... 3 more fields])

val vertices_val = sqlContext.read.parquet("/tmp/motif-vertices-val")
val edges_val = sqlContext.read.parquet("/tmp/motif-edges-val")
val rank_graph_val = GraphFrame(vertices_val, edges_val)

vertices_val: org.apache.spark.sql.DataFrame = [graph_id: string, id: string ... 1 more field]
edges_val: org.apache.spark.sql.DataFrame = [src: string, dst: string ... 3 more fields]
rank_graph_val: org.graphframes.GraphFrame = GraphFrame(v:[id: string, graph_id: string ... 1 more field], e:[src: string, dst: string ... 3 more fields])

val ranks = rank_graph.pageRank.resetProbability(0.15).tol(0.01).run()
display(ranks.vertices)

#temp = GraphFrame(scene_graphs_train.vertices.select('graph_id', 'id', 'object_name'), scene_graphs_train.edges)

#ranks = temp.pageRank(resetProbability=0.15, tol=0.01)
#display(ranks.vertices)

import org.apache.spark.sql.functions._

val sorted_ranks = ranks.vertices.sort(col("pagerank").desc)
display(sorted_ranks)

val val_graphs_without_attributes = GraphFrame(rank_graph_val.vertices.select("graph_id", "id", "object_name"), rank_graph_val.edges)
val val_ranks = val_graphs_without_attributes.pageRank.resetProbability(0.15).tol(0.01).run()
display(val_ranks.vertices)

val val_sorted_ranks = val_ranks.vertices.sort(col("pagerank").desc)
display(val_sorted_ranks)

val graph_pagerank_sums_objects = ranks.vertices.groupBy("object_name").sum("pagerank")
graph_pagerank_sums_objects.show()

val graph_pagerank_sums_objects_sorted = graph_pagerank_sums_objects.sort(col("sum(pagerank)").desc)

graph_pagerank_sums_objects_sorted: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [object_name: string, sum(pagerank): double]

display(graph_pagerank_sums_objects_sorted)

#%scala
val object_count = rank_graph.vertices.groupBy("object_name").count()
display(object_count)

val topObjects = object_count.sort(col("object_name").desc)
val graph_pagerank_sums_objects_sorted = graph_pagerank_sums_objects.sort(col("object_name").desc)
display(graph_pagerank_sums_objects_sorted)

val graph_pagerank_joined = graph_pagerank_sums_objects_sorted.join(topObjects, "object_name").withColumn("normalize(pagerank)", col("sum(pagerank)") / col("count"))
display(graph_pagerank_joined.sort(col("normalize(pagerank)").desc))

display(graph_pagerank_joined.sort(col("sum(pagerank)").desc).limit(30))

val graph_pagerank_sums = ranks.vertices.groupBy("graph_id").sum("pagerank")
display(graph_pagerank_sums)

val graph_val_pagerank_sums = val_ranks.vertices.groupBy("graph_id").sum("pagerank")
display(graph_val_pagerank_sums)

display(graph_val_pagerank_sums.sort(col("sum(pagerank)").desc))

merged_vertices = scene_graphs_val.vertices.selectExpr('object_name as id', 'attributes as attributes')
display(merged_vertices)

merged_vertices.count()

merged_vertices = merged_vertices.distinct()
display(merged_vertices)

merged_vertices.count()

merged_vertices_without_attributes = merged_vertices.select('id').distinct()
display(merged_vertices_without_attributes)

merged_vertices_without_attributes.count()

merged_edges = scene_graphs_val.edges.selectExpr('src_type as src', 'dst_type as dst', 'relation_name as relation_name')
display(merged_edges)

merged_edges.count()

scene_graphs_merged = GraphFrame(merged_vertices, merged_edges)

display(scene_graphs_merged.edges)

scene_graphs_merged_without_attributes = GraphFrame(merged_vertices_without_attributes, merged_edges)

display(scene_graphs_merged_without_attributes.vertices)

display(scene_graphs_merged_without_attributes.edges)

sc.setCheckpointDir("/tmp/scene-graph-motifs-connected-components")
connected_components = scene_graphs_merged.connectedComponents()
display(connected_components)

# displays the index of a component for a given object category

components_count = connected_components.groupBy('component')
display(components_count.count().sort("count", ascending=False))

connected_components_without_attributes = scene_graphs_merged_without_attributes.connectedComponents()
display(connected_components_without_attributes)

components_count = connected_components_without_attributes.groupBy('component')
display(components_count.count().sort("count", ascending=False))

pwd

/databricks/driver

wget http://snap.stanford.edu/data/soc-sign-epinions.txt.gz

ls -l

total 2924
drwxr-xr-x  2 root root    4096 Jan  1  1970 conf
-rw-r--r--  1 root root     733 Nov 24 15:24 derby.log
drwxr-xr-x 10 root root    4096 Nov 24 15:24 eventlogs
drwxr-xr-x  2 root root    4096 Nov 24 16:15 ganglia
drwxr-xr-x  2 root root    4096 Nov 24 16:04 logs
-rw-r--r--  1 root root 2972840 Dec  3  2009 soc-sign-epinions.txt.gz

gunzip soc-sign-epinions.txt.gz

ls -l

total 11000
drwxr-xr-x  2 root root     4096 Jan  1  1970 conf
-rw-r--r--  1 root root      733 Nov 24 15:24 derby.log
drwxr-xr-x 10 root root     4096 Nov 24 15:24 eventlogs
drwxr-xr-x  2 root root     4096 Nov 24 16:15 ganglia
drwxr-xr-x  2 root root     4096 Nov 24 16:04 logs
-rw-r--r--  1 root root 11243141 Dec  3  2009 soc-sign-epinions.txt

head soc-sign-epinions.txt

# Directed graph: soc-sign-epinions
# Epinions signed social network
# Nodes: 131828 Edges: 841372
# FromNodeId	ToNodeId	Sign
0	1	-1
1	128552	-1
2	3	1
4	5	-1
4	155	-1
4	558	1

mkdir -p epinions
mv soc-sign-epinions.txt epinions/

ls -l /dbfs/FileStore
mv epinions /dbfs/FileStore/

total 33
drwxrwxrwx 2 root root   24 May  1  2018 datasets_magellan
drwxrwxrwx 2 root root 4096 Nov 24 11:14 DIGSUM-files
drwxrwxrwx 2 root root 4096 Nov 24 11:14 import-stage
drwxrwxrwx 2 root root 4096 Nov 24 11:14 jars
drwxrwxrwx 2 root root 4096 Nov 24 11:14 plots
drwxrwxrwx 2 root root 4096 Nov 24 11:14 shared_uploads
drwxrwxrwx 2 root root 4096 Nov 24 11:14 simon_temp_files_feel_free_to_delete_any_time
drwxrwxrwx 2 root root 4096 Nov 24 11:14 tables
drwxrwxrwx 2 root root 4096 Nov 24 11:14 timelinesOfInterest
mv: preserving permissions for ‘/dbfs/FileStore/epinions/soc-sign-epinions.txt’: Operation not permitted
mv: preserving permissions for ‘/dbfs/FileStore/epinions’: Operation not permitted

ls /

ls /FileStore

ls file:/databricks/driver

//%fs mv file:///databricks/driver/epinions /FileStore/
%fs ls /FileStore/epinions/

import org.apache.spark.sql._
import org.apache.spark.sql.functions._

import org.graphframes._

// This import is needed to use the $-notation
import spark.implicits._

import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.graphframes._
import spark.implicits._

var df = spark.read.format("csv")
//   .option("header", "true")
  .option("inferSchema", "true")
  .option("comment", "#")
  .option("sep", "\t")
  .load("/FileStore/epinions")

df: org.apache.spark.sql.DataFrame = [_c0: int, _c1: int ... 1 more field]

df.count()

res1: Long = 841372

df.rdd.getNumPartitions 

res36: Int = 3

df.head(3)

res37: Array[org.apache.spark.sql.Row] = Array([0,1,-1], [1,128552,-1], [2,3,1])

df.printSchema()

root
 |-- _c0: integer (nullable = true)
 |-- _c1: integer (nullable = true)
 |-- _c2: integer (nullable = true)

val newNames = Seq("src", "dst", "rela")
val e = df.toDF(newNames: _*)

newNames: Seq[String] = List(src, dst, rela)
e: org.apache.spark.sql.DataFrame = [src: int, dst: int ... 1 more field]

e.printSchema()

root
 |-- src: integer (nullable = true)
 |-- dst: integer (nullable = true)
 |-- rela: integer (nullable = true)

// Vertex DataFrame
val v = spark.range(1, 131827).toDF("id")

v: org.apache.spark.sql.DataFrame = [id: bigint]

val g = GraphFrame(v, e)

g: org.graphframes.GraphFrame = GraphFrame(v:[id: bigint], e:[src: int, dst: int ... 1 more field])

g.edges.take(3)

res15: Array[org.apache.spark.sql.Row] = Array([0,1,-1], [1,128552,-1], [2,3,1])

// val results = g.triangleCount.run()

// Search for pairs of vertices with edges in both directions between them, i.e., find undirected or bidirected edges.
val pair = g.find("(a)-[e1]->(b); (b)-[e2]->(a)")
println(pair.count())
val filtered = pair.filter("e1.rela == e2.rela")
println(filtered.count())

259751
254345
pair: org.apache.spark.sql.DataFrame = [a: struct<id: bigint>, e1: struct<src: int, dst: int ... 1 more field> ... 2 more fields]
filtered: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [a: struct<id: bigint>, e1: struct<src: int, dst: int ... 1 more field> ... 2 more fields]

val triad = g.find("(a)-[eab]->(b); (b)-[eba]->(a); (b)-[ebc]->(c); (c)-[ecb]->(b); (c)-[eca]->(a); (a)-[eac]->(c)")
println(triad.count())

3314925
triad: org.apache.spark.sql.DataFrame = [a: struct<id: bigint>, eab: struct<src: int, dst: int ... 1 more field> ... 7 more fields]

val t111 = triad.filter("eab.rela = 1 AND eab.rela = ebc.rela AND ebc.rela = eca.rela")
println(t111.count())

3232357
t111: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [a: struct<id: bigint>, eab: struct<src: int, dst: int ... 1 more field> ... 7 more fields]

val t000 = triad.filter("eab.rela = -1 AND eab.rela = ebc.rela AND ebc.rela = eca.rela")
println(t000.count())

1610
t000: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [a: struct<id: bigint>, eab: struct<src: int, dst: int ... 1 more field> ... 7 more fields]

val t110 = triad.filter("eab.rela + ebc.rela + eca.rela = 1")
println(t110.count())

62634
t110: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [a: struct<id: bigint>, eab: struct<src: int, dst: int ... 1 more field> ... 7 more fields]

val t001 = triad.filter("eab.rela + ebc.rela + eca.rela = -1")
println(t001.count())

18324
t001: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [a: struct<id: bigint>, eab: struct<src: int, dst: int ... 1 more field> ... 7 more fields]

val n111 = t111.count()
val n001 = t001.count()
val n000 = t000.count() 
val n110 = t110.count()
val imbalanced = n000 + n110
val balanced = n111 + n001

n111: Long = 3232357
n001: Long = 18324
n000: Long = 1610
n110: Long = 62634
imbalanced: Long = 64244
balanced: Long = 3250681

val g: GraphFrame = examples.Graphs.friends

g: org.graphframes.GraphFrame = GraphFrame(v:[id: string, name: string ... 1 more field], e:[src: string, dst: string ... 1 more field])

display(g.edges)

val motifs = g.find("(a)-[e]->(b); (b)-[e2]->(a)")
motifs.show()

+----------------+--------------+----------------+--------------+
|               a|             e|               b|            e2|
+----------------+--------------+----------------+--------------+
|    [b, Bob, 36]|[b, c, follow]|[c, Charlie, 30]|[c, b, follow]|
|[c, Charlie, 30]|[c, b, follow]|    [b, Bob, 36]|[b, c, follow]|
+----------------+--------------+----------------+--------------+

motifs: org.apache.spark.sql.DataFrame = [a: struct<id: string, name: string ... 1 more field>, e: struct<src: string, dst: string ... 1 more field> ... 2 more fields]

//Import the necessary objects:
import org.apache.spark.mllib.linalg.distributed.IndexedRow
import org.apache.spark.mllib.linalg.Matrices



//Create the matrix A above as a dense matrix:
val Amatrix = Matrices.dense(4,2,Array(1,3,0,0,2,4,0,0))

//Zip each row of a with its index and map it to an indexed row object (x._2 is the index and x._1 the array).
//Once we have an IndexedRow r we can get the index and vector by calling r.index and r.vector 
val A = sc.parallelize(Amatrix.rowIter.toArray.zipWithIndex.map(x=>new IndexedRow(x._2,x._1)))

 A.take(2)

import org.apache.spark.mllib.linalg.distributed.IndexedRow
import org.apache.spark.mllib.linalg.Matrices
Amatrix: org.apache.spark.mllib.linalg.Matrix =
1.0  2.0
3.0  4.0
0.0  0.0
0.0  0.0
A: org.apache.spark.rdd.RDD[org.apache.spark.mllib.linalg.distributed.IndexedRow] = ParallelCollectionRDD[689] at parallelize at command-1767923094594942:13
res0: Array[org.apache.spark.mllib.linalg.distributed.IndexedRow] = Array(IndexedRow(0,[1.0,2.0]), IndexedRow(1,[3.0,4.0]))

import scala.collection.mutable.ArrayBuffer


//Function that maps an indexed row (index,(a_1,...,a_n)) to   ((j,k),a_j*a_k), j=1,..,n and k=1,...,n
def f(v: IndexedRow): Array[((Int,Int),Double)]={
var keyvaluepairs = ArrayBuffer[((Int,Int),Double)]()
for(j<-0 to v.vector.size-1){
  for(k<-0 to v.vector.size-1){
  keyvaluepairs.append(((j,k),v.vector(j)*v.vector(k)))
  }
}
keyvaluepairs.toArray
}

//map M to key-value rdd where key =(j,k) and value = a_ij*a_ik.
//We use flatmap since we don't need to keep the row structure.
val keyvalRDD = A.flatMap(row =>f(row))

keyvalRDD.take(5)

//Sum up all key-value pairs that have the same key (j,k) (corresponts to getting the element of A.T*A on the j:th row and k:th column).
val keyvalSum = keyvalRDD.reduceByKey((x,y)=>x+y)

keyvalSum.take(2)

import org.apache.spark.mllib.linalg.distributed.{CoordinateMatrix, MatrixEntry}
import breeze.linalg.DenseMatrix

//map to matrix entries
val matrix = keyvalSum.map(el => MatrixEntry(el._1._1, el._1._2, el._2))   

//Create a CoordinateMatrix
val mat = new CoordinateMatrix(matrix)


//Transform to RowMatrix and collect.
val ATArowmatrix = mat.toRowMatrix().rows.collect()

val m = mat.numRows()
val n = mat.numCols()

//Create an empty DenseMatrix (in which we will store the product A.T*A).
val ATA = DenseMatrix.zeros[Double](m.toInt,n.toInt)

//Each row will be a sparse vector. For each row we iterate over the non-zeros indices (foreachActive) and fill the i:th row of the ATA-matrix.
var i = 0
ATArowmatrix.foreach { vec =>
  vec.foreachActive { case (index, value) =>
    ATA(i, index) = value
  }
  i += 1
}

import scala.collection.mutable.ArrayBuffer
import breeze.linalg._, eigSym.EigSym

//lambda is a vector with the eigenvalues of A.T*A and evs the eigenvector matrix.
val EigSym(lambda, evs) = eigSym(ATA) 
//det(evs)

val w=lambda.map(x=>if(x >0) Math.sqrt(x) else 0) //square root of eigen values to compute the S matrix.
val S =diag(w)  
val V =evs

//Compute the inverse of S.
val Sinv = S.map(x=>if(x==0) 0 else 1/x)   //invert the diagonal matrix.

//Compute the product of V and inverse of S.
val M = V*Sinv

//Broadcast to the spark context.
sc.broadcast(M)

import org.apache.spark.mllib.linalg.distributed.{IndexedRowMatrix}

//Function that multiplies an indexedRow object with a DenseMatrix (from breeze.linalg.DenseMatrix) on the left and returns an Array.
def prod(u: IndexedRow, m: DenseMatrix[Double]): Array[Double]={
var w = ArrayBuffer[Double]()
for(i<-0 to m.cols-1){
  var x: Double =0
  for(j<-0 to m.rows-1){
      x=x+m(j,i)*u.vector(j)
  }
  w.append(x)
}
w.toArray
}


//COmpute the matrix product by multiplying each indexed row with the Matrix M (and then collect the result)
val Urows =A.map(row => prod(row,M)).collect() 

//Create a dense matrix U with the rows.
val U = DenseMatrix(Urows:_*)   

//Print the product USV.t to check that it equals A:
println(U*S*V.t)

import spark.implicits._
import org.apache.spark.sql.functions._

import spark.implicits._
import org.apache.spark.sql.functions._

// Load the (userID, artistID, play_count) triplets.
val fileName_data="dbfs:/FileStore/tables/project4/hetrec2011-lastfm-2k/user_artists.dat"
val df_raw = spark.read.format("csv").option("header", "true").option("delimiter", "\t").option("inferSchema","true").load(fileName_data).withColumnRenamed("weight","play_count")
df_raw.cache()
df_raw.orderBy(rand()).show(5)

// Load the (artistID, name) tuples.
val fileName_names="dbfs:/FileStore/tables/project4/hetrec2011-lastfm-2k/artists.dat"
val artist_names = spark.read.format("csv").option("header", "true").option("delimiter", "\t").option("inferSchema","true").load(fileName_names).withColumnRenamed("id","artistID").select("artistID","name")
artist_names.cache()
artist_names.show(5)

+------+--------+----------+
|userID|artistID|play_count|
+------+--------+----------+
|  1553|    2478|       171|
|    65|    1858|       254|
|   496|       9|       161|
|   304|     163|        19|
|   747|     507|       626|
+------+--------+----------+
only showing top 5 rows

+--------+-----------------+
|artistID|             name|
+--------+-----------------+
|       1|     MALICE MIZER|
|       2|  Diary of Dreams|
|       3|Carpathian Forest|
|       4|     Moi dix Mois|
|       5|      Bella Morte|
+--------+-----------------+
only showing top 5 rows

fileName_data: String = dbfs:/FileStore/tables/project4/hetrec2011-lastfm-2k/user_artists.dat
df_raw: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 1 more field]
fileName_names: String = dbfs:/FileStore/tables/project4/hetrec2011-lastfm-2k/artists.dat
artist_names: org.apache.spark.sql.DataFrame = [artistID: int, name: string]

val n_data = df_raw.count().asInstanceOf[Long].floatValue(); // Number of observations
val n_users = df_raw.agg(countDistinct("userID")).collect()(0)(0).asInstanceOf[Long].floatValue(); // Number of unique users
val n_artists = df_raw.agg(countDistinct("artistID")).collect()(0)(0).asInstanceOf[Long].floatValue(); // Number of unique artists
val sparsity = 1-n_data/(n_users*n_artists) //Sparsity of the data

println("Number of data points: " + n_data)
println("Number of users: " + n_users)
println("Number of artists: " + n_artists)
print("Sparsity:" + sparsity.toString + "\n")

Number of data points: 92834.0
Number of users: 1892.0
Number of artists: 17632.0
Sparsity:0.9972172
n_data: Float = 92834.0
n_users: Float = 1892.0
n_artists: Float = 17632.0
sparsity: Float = 0.9972172

display(df_raw.select("play_count"))

display(df_raw.select("play_count").filter($"play_count"<1000))

// Compute some statistics for the artists.
val artist_data_raw = df_raw.groupBy("artistID").agg(count("artistID") as "unique_users",
                                                      sum("play_count") as "total_plays_artist")
artist_data_raw.sort(desc("total_plays_artist")).join(artist_names,"artistID").show(5) // Top artists based on total plays
artist_data_raw.sort(desc("unique_users")).join(artist_names,"artistID").show(5) // Top artists based on number of unique listener

+--------+------------+------------------+------------------+
|artistID|unique_users|total_plays_artist|              name|
+--------+------------+------------------+------------------+
|     289|         522|           2393140|    Britney Spears|
|      72|         282|           1301308|      Depeche Mode|
|      89|         611|           1291387|         Lady Gaga|
|     292|         407|           1058405|Christina Aguilera|
|     498|         399|            963449|          Paramore|
+--------+------------+------------------+------------------+
only showing top 5 rows

+--------+------------+------------------+--------------+
|artistID|unique_users|total_plays_artist|          name|
+--------+------------+------------------+--------------+
|      89|         611|           1291387|     Lady Gaga|
|     289|         522|           2393140|Britney Spears|
|     288|         484|            905423|       Rihanna|
|     227|         480|            662116|   The Beatles|
|     300|         473|            532545|    Katy Perry|
+--------+------------+------------------+--------------+
only showing top 5 rows

artist_data_raw: org.apache.spark.sql.DataFrame = [artistID: int, unique_users: bigint ... 1 more field]

display(artist_data_raw.select("total_plays_artist"))

display(artist_data_raw.select("total_plays_artist").filter($"total_plays_artist"<10000))

display(artist_data_raw.select("unique_users"))

// Compute statistics for each user.
val user_data_raw = df_raw.groupBy("userID").agg(count("userID") as "unique_artists",
                                                  sum("play_count") as "total_plays_user")
user_data_raw.sort(desc("total_plays_user")).show(5) // Show users with most total plays

+------+--------------+----------------+
|userID|unique_artists|total_plays_user|
+------+--------------+----------------+
|   757|            50|          480039|
|  2000|            50|          468409|
|  1418|            50|          416349|
|  1642|            50|          388251|
|  1094|            50|          379125|
+------+--------------+----------------+
only showing top 5 rows

user_data_raw: org.apache.spark.sql.DataFrame = [userID: int, unique_artists: bigint ... 1 more field]

display(user_data_raw.select("total_plays_user"))

// Merge all statistics and data into a single dataframe.
val df_joined = df_raw.join(artist_data_raw, "artistID").join(user_data_raw, "userID").join(artist_names,"artistID").select("userID", "artistID","play_count", "name", "unique_artists","unique_users", "total_plays_user","total_plays_artist")
df_joined.show(5)

+------+--------+----------+-------------+--------------+------------+----------------+------------------+
|userID|artistID|play_count|         name|unique_artists|unique_users|total_plays_user|total_plays_artist|
+------+--------+----------+-------------+--------------+------------+----------------+------------------+
|     2|      51|     13883|  Duran Duran|            50|         111|          168737|            348919|
|     2|      52|     11690|    Morcheeba|            50|          23|          168737|             18787|
|     2|      53|     11351|          Air|            50|          75|          168737|             44230|
|     2|      54|     10300| Hooverphonic|            50|          18|          168737|             15927|
|     2|      55|      8983|Kylie Minogue|            50|         298|          168737|            449292|
+------+--------+----------+-------------+--------------+------------+----------------+------------------+
only showing top 5 rows

df_joined: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 6 more fields]

// Remove artists with less than 20 unique users, and recompute the statistics.
val df_filtered_1 = df_joined.filter($"unique_users">=20).select(df_joined("userID"),df_joined("artistID"),df_joined("play_count"))
val artist_data_1 = df_filtered_1.groupBy("artistID").agg(count("artistID") as "unique_users",
                                                          sum("play_count") as "total_plays_artist")
                                                     .withColumnRenamed("artistID","artistID_1")

val user_data_1 = df_filtered_1.groupBy("userID").agg(count("userID") as "unique_artists",
                                                          sum("play_count") as "total_plays_user")
                                                     .withColumnRenamed("userID","userID_1")

val df_joined_filtered_1 = df_filtered_1.join(artist_data_1, artist_data_1("artistID_1")===df_filtered_1("artistID"))
                                        .join(user_data_1, user_data_1("userID_1")===df_filtered_1("userID"))
                                        .select(df_filtered_1("userID"),df_filtered_1("artistID"),df_filtered_1("play_count"),
                                                 artist_data_1("unique_users"),artist_data_1("total_plays_artist"),
                                                 user_data_1("unique_artists"), user_data_1("total_plays_user"))

// Remove users with less than 5 unique users, and recompute the statistics.
val df_filtered_2 = df_joined_filtered_1.filter($"unique_artists">=5).select(df_filtered_1("userID"),df_filtered_1("artistID"),
                                                                             df_filtered_1("play_count"))

val artist_data = df_filtered_2.groupBy("artistID").agg(count("artistID") as "unique_users",
                                                        sum("play_count") as "total_plays_artist")
                                                   .withColumnRenamed("artistID","artistID_2")

val user_data = df_filtered_2.groupBy("userID").agg(count("userID") as "unique_artists",
                                                         sum("play_count") as "total_plays_user")
                                                   .withColumnRenamed("userID","userID_2")

// Now we collect our new filtered data.
val user_artist_data = df_filtered_2.join(artist_data, artist_data("artistID_2")===df_filtered_2("artistID"))
                                    .join(user_data, user_data("userID_2")===df_filtered_2("userID"))
                                    .select("userID","artistID","play_count","unique_users","total_plays_artist","unique_artists","total_plays_user")

user_artist_data.show(5)

+------+--------+----------+------------+------------------+--------------+----------------+
|userID|artistID|play_count|unique_users|total_plays_artist|unique_artists|total_plays_user|
+------+--------+----------+------------+------------------+--------------+----------------+
|   148|    1118|       214|          62|             53915|            12|            3026|
|   148|    1206|       245|          50|             32827|            12|            3026|
|   148|     206|       214|          83|             36944|            12|            3026|
|   148|     233|       170|         138|            160317|            12|            3026|
|   148|     429|       430|         162|             91740|            12|            3026|
+------+--------+----------+------------+------------------+--------------+----------------+
only showing top 5 rows

df_filtered_1: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 1 more field]
artist_data_1: org.apache.spark.sql.DataFrame = [artistID_1: int, unique_users: bigint ... 1 more field]
user_data_1: org.apache.spark.sql.DataFrame = [userID_1: int, unique_artists: bigint ... 1 more field]
df_joined_filtered_1: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 5 more fields]
df_filtered_2: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 1 more field]
artist_data: org.apache.spark.sql.DataFrame = [artistID_2: int, unique_users: bigint ... 1 more field]
user_data: org.apache.spark.sql.DataFrame = [userID_2: int, unique_artists: bigint ... 1 more field]
user_artist_data: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 5 more fields]

val n_data_new = user_artist_data.count().asInstanceOf[Long].floatValue(); // Number of observations
val n_users_new = user_artist_data.agg(countDistinct("userID")).collect()(0)(0).asInstanceOf[Long].floatValue(); // Number of unique users
val n_artists_new = user_artist_data.agg(countDistinct("artistID")).collect()(0)(0).asInstanceOf[Long].floatValue(); // Number of unique artists
val sparsity_new = 1-n_data/(n_users*n_artists) // Compute the sparsity

println("Number of data points: " + n_data_new)
println("Number of users: " + n_users_new)
println("Number of artists: " + n_artists_new)
print("Sparsity:" + sparsity.toString + "\n")

Number of data points: 53114.0
Number of users: 1819.0
Number of artists: 804.0
Sparsity:0.9972172
n_data_new: Float = 53114.0
n_users_new: Float = 1819.0
n_artists_new: Float = 804.0
sparsity_new: Float = 0.9972172

display(user_artist_data.select("play_count"))

display(artist_data.select("total_plays_artist"))

display(artist_data.select("unique_users"))

display(artist_data)

//Scaling the play_counts
val user_artist_data_scaled = user_artist_data
.withColumn("scaled_value", sqrt(col("play_count"))).drop("play_count").withColumnRenamed("scaled_value","play_count")
user_artist_data_scaled.show(5)

+------+--------+------------+------------------+--------------+----------------+------------------+
|userID|artistID|unique_users|total_plays_artist|unique_artists|total_plays_user|        play_count|
+------+--------+------------+------------------+--------------+----------------+------------------+
|   148|     436|         124|             88270|            12|            3026| 18.33030277982336|
|   148|    1206|          50|             32827|            12|            3026|15.652475842498529|
|   148|     512|          67|             62933|            12|            3026|15.620499351813308|
|   148|     429|         162|             91740|            12|            3026| 20.73644135332772|
|   148|    1943|          25|             13035|            12|            3026| 17.26267650163207|
+------+--------+------------+------------------+--------------+----------------+------------------+
only showing top 5 rows

user_artist_data_scaled: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 5 more fields]

display(user_artist_data_scaled.select("play_count"))

display(user_artist_data_scaled.select("play_count"))

// Split data into training, validation and test sets.
val Array(training_set, validation_set, test_set) = user_artist_data_scaled.select("userID","artistID","play_count").randomSplit(Array(0.6, 0.2, 0.2))
training_set.cache()
validation_set.cache()
test_set.cache()

training_set: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [userID: int, artistID: int ... 1 more field]
validation_set: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [userID: int, artistID: int ... 1 more field]
test_set: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [userID: int, artistID: int ... 1 more field]
res109: test_set.type = [userID: int, artistID: int ... 1 more field]

import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.Dataset
import org.apache.spark.sql.Row
import org.apache.spark.sql.DataFrame

// Function for computing the mean rank metric.
// Input:
// - prediction_scores_new: DataFrame with userIDs and corresponding recommendation lists.
// - training_set: DataFrame with observations in training set
// - validation_set: DataFrame with the observations needed for the evaluation of the metric.
// Output: Float corresponind to the mean_rank score.
def eval_model(predictions_scores_new: DataFrame, training_set: DataFrame, validation_set: DataFrame) : Float = {

  val predictions_scores = predictions_scores_new.withColumnRenamed("userID","userID_new") // Avoinding duplicate column names.
  val recommendations = predictions_scores.withColumn("recommendations", explode($"recommendations")) // Rearrange the recommendation lists.
                                      .select("userID_new","recommendations.artistID", "recommendations.rating")

  val recommendations_filtered = recommendations.join(training_set, training_set("userID")===recommendations("userID_new") && training_set("artistID")===recommendations("artistID"), "leftanti") // Erase artists appearing in the training for each user.

  // Compute ranking percentiles.
  val recommendations_percentiles = recommendations_filtered.withColumn("rank",percent_rank()
                                                            .over(Window.partitionBy("userID_new").orderBy(desc("rating"))))
  // Store everything in single DataFrame.
  val table_data = recommendations_percentiles.join(validation_set, recommendations_percentiles("userID_new")===validation_set("userID") && recommendations_percentiles("artistID")===validation_set("artistID"))

  // Compute the sum in the numerator for the metric.
  val numerator = table_data.withColumn("ru1rankui", $"rank"*$"play_count"*100.0)
                            .agg(sum("ru1rankui"))
                            .collect()(0)(0).asInstanceOf[Double]

  // Compute the sum in the denominator for the metric.
  val denumerator = table_data.agg(sum("play_count"))
                              .collect()(0)(0)
                              .asInstanceOf[Double]
  // Compute the mean percentile rank.
  val rank_score = numerator/denumerator
  rank_score.toFloat
}

import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.Dataset
import org.apache.spark.sql.Row
import org.apache.spark.sql.DataFrame
eval_model: (predictions_scores_new: org.apache.spark.sql.DataFrame, training_set: org.apache.spark.sql.DataFrame, validation_set: org.apache.spark.sql.DataFrame)Float

import org.apache.spark.ml.recommendation.ALS
// Number of iterations in the ALS algorithm
val numIter = 10


 val ranks = List(10,50,100,150) // Dimension of latent feature space
 val lambdas=List(0.1, 1.0, 2.0) // Regularization parameter
 val alphas=List(0.5, 1.0, 5.0) // Confidence parameter

// Loop over all parameter combinations
for ( alpha <- alphas ){
  for ( lambda <- lambdas ){
    for ( rank <- ranks ){
      val als = new ALS()
        .setRank(rank)
        .setMaxIter(numIter)
        .setRegParam(lambda)
        .setUserCol("userID")
        .setItemCol("artistID")
        .setRatingCol("play_count")
        .setImplicitPrefs(true) // Indicate we have implicit data
        .setAlpha(alpha)
        .setNonnegative(true) // Constrain to non-negative values

      // Fit the model
      val model = als.fit(training_set)

      model.setColdStartStrategy("drop") // This is to ensure we handle unseen users or unseen artist saftely during the prediction.
           .setUserCol("userID")
           .setItemCol("artistID")
      // Generate the recommendations
      val predictions_scores = model.recommendForUserSubset(validation_set,n_artists_new.toInt)

      // Evaluate the model
      println("rank=" + rank + ", alpha=" + alpha + ", lambda=" + lambda + ", mean_rank=" + eval_model(predictions_scores, training_set, validation_set))
    }
  }
}

// Retrain the best model.

val numIter_final=10
val rank_final=150
val alpha_final=0.5
val lambda_final=2.0
val als_final = new ALS()
        .setRank(rank_final)
        .setMaxIter(numIter_final)
        .setRegParam(lambda_final)
        .setUserCol("userID")
        .setItemCol("artistID")
        .setRatingCol("play_count")
        .setImplicitPrefs(true)
        .setAlpha(alpha_final)
        .setNonnegative(true)
val model_final = als_final.fit(training_set)
model_final.setColdStartStrategy("drop")
     .setUserCol("userID")
     .setItemCol("artistID")

// Evaluate on the validation set.
val predictions_scores_val = model_final.recommendForUserSubset(validation_set,n_artists_new.toInt)
println("Validation set: mean_rank=" + eval_model(predictions_scores_val, training_set, validation_set))

// Evaluate on the test set.
val predictions_scores_test = model_final.recommendForUserSubset(test_set,n_artists_new.toInt)
println("Test set: mean_rank=" + eval_model(predictions_scores_val, training_set, test_set))

Validation set: mean_rank=7.7979865
Test set: mean_rank=7.75016
numIter_final: Int = 10
rank_final: Int = 150
alpha_final: Double = 0.5
lambda_final: Double = 2.0
als_final: org.apache.spark.ml.recommendation.ALS = als_f778f6cc23db
model_final: org.apache.spark.ml.recommendation.ALSModel = als_f778f6cc23db
predictions_scores_val: org.apache.spark.sql.DataFrame = [userID: int, recommendations: array<struct<artistID:int,rating:float>>]
predictions_scores_test: org.apache.spark.sql.DataFrame = [userID: int, recommendations: array<struct<artistID:int,rating:float>>]

case class Rating(artistID: Int, rating: Float) // Simple class for getting the recommendations in suitable form.

// Generating random array of artistIDs.
val random = artist_data.select("artistID_2").distinct().orderBy(rand()).withColumn("idx",monotonically_increasing_id)
           .withColumn("rownumber",row_number.over(Window.orderBy(desc("idx")))).drop("idx").sort(desc("rownumber"))
          .collect.map(row =>Rating(row.getInt(0),row.getInt(1).toFloat))

val test_users = test_set.select("userID").distinct()

//Append the arrays to DataFrame.
val prediction_scores = user_artist_data.select("userID").distinct().withColumn("recommendations",typedLit(random))
                                        .join(test_users,"userID")

println("Random_model: mean_rank=" + eval_model(prediction_scores, training_set, test_set))

Random_model: mean_rank=50.860817

//Generating arrays of artistIDs w.r.t most plays.
val most_popular = artist_data.select("artistID_2", "total_plays_artist").sort(desc("total_plays_artist"))
                              .collect.map(row =>Rating(row.getInt(0),row.getLong(1).toFloat))
val test_users = test_set.select("userID").distinct()

//Append the arrays to DataFrame.
val prediction_scores = user_artist_data.select("userID").distinct().withColumn("recommendations",typedLit(most_popular))
                                        .join(test_users,"userID")

println("Popular_model: mean_rank=" + eval_model(prediction_scores, training_set, test_set))

Popular_model: mean_rank=24.553421

import org.apache.spark.ml.recommendation.ALSModel

// Function for showing the favorit artists for a given user based on the training set.
// Input:
// - userID: Int, the id of the user.
// - n: Int, number of top artists that should be presented.
// - user_artist_data: DataFrame with observations.
// - artist_names: Dataframe mapping artistIDs to actual artist names
// Output:
// - DataFrame with the users top artists
def userHistory(userID: Int, n: Int, user_artist_data: DataFrame, artist_names: DataFrame): DataFrame = {

  // Filter the userID and sort the artists w.r.t the play count. Append the actual artist names.
  val data = user_artist_data.filter($"userID"===userID).sort(desc("play_count")).join(artist_names, "artistID")
  data.select("userID","artistID","name").show(n)                            
  data.select("userID","artistID","name")
}

// Function for presenting recommended artist for a user.
// Input:
// - Model: ALSModel, the trained model
// - userID: DataFrame, with userID
// - n: Int, number of top artists that should be presented.
// - training_set: DataFrame used during the training.
// - artist_names: Dataframe mapping artistIDs to actual artist names
// Output:
// - DataFrame with the users recommended artists
def recommendToUser(model: ALSModel, userID: DataFrame, n: Int, training_set: DataFrame, artist_names: DataFrame) : DataFrame = {
  // Generate recommendations using the model.
  val recommendations = model.recommendForUserSubset(userID, n_artists_new.toInt).withColumn("recommendations", explode($"recommendations"))
                                      .select("userID","recommendations.artistID", "recommendations.rating").join(artist_names, "artistID").select("userID","artistID","name","rating")

  // Remove possible artists observed in the training set
  recommendations.join(training_set,training_set("userID")===recommendations("userID") && training_set("artistID")===recommendations("artistID"),"leftanti")
}

import org.apache.spark.ml.recommendation.ALSModel
userHistory: (userID: Int, n: Int, user_artist_data: org.apache.spark.sql.DataFrame, artist_names: org.apache.spark.sql.DataFrame)org.apache.spark.sql.DataFrame
recommendToUser: (model: org.apache.spark.ml.recommendation.ALSModel, userID: org.apache.spark.sql.DataFrame, n: Int, training_set: org.apache.spark.sql.DataFrame, artist_names: org.apache.spark.sql.DataFrame)org.apache.spark.sql.DataFrame

println("Listening history:")
// Print top 5 artists for userID 302
val sub_data = userHistory(302, 5, training_set, artist_names)

// Generate top 5 recommendations of undiscovered artists,
val recommendations = recommendToUser(model_final, sub_data, 5, training_set, artist_names)
println("Recommendations:")
recommendations.show(5)

Listening history:
+------+--------+--------------+
|userID|artistID|          name|
+------+--------+--------------+
|   302|      55| Kylie Minogue|
|   302|      89|     Lady Gaga|
|   302|     265|   Céline Dion|
|   302|     288|       Rihanna|
|   302|     299|Jennifer Lopez|
+------+--------+--------------+
only showing top 5 rows

Recommendations:
+------+--------+------------------+----------+
|userID|artistID|              name|    rating|
+------+--------+------------------+----------+
|   302|     289|    Britney Spears|0.88509434|
|   302|     292|Christina Aguilera| 0.8353728|
|   302|     300|        Katy Perry| 0.7862937|
|   302|      67|           Madonna| 0.7838166|
|   302|     295|           Beyoncé|0.76221865|
+------+--------+------------------+----------+
only showing top 5 rows

sub_data: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 1 more field]
recommendations: org.apache.spark.sql.DataFrame = [userID: int, artistID: int ... 2 more fields]

// Setup Wikipedia source
val lang = "sv"
val freeze_date = "20201120"
val wp_file = s"${lang}wiki-${freeze_date}-pages-articles.xml"
val wp_zip_remote = s"https://dumps.wikimedia.org/${lang}wiki/${freeze_date}/${wp_file}.bz2"

// Stopwords download soure
val stopwords_remote = "https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv"

// Setup base paths for storage
val group = "05_lda"  // directory name to separate file structure, allows to run different instances in parallel
val localpath=s"file:/databricks/driver/$group/"
val dir_local = s"$group/"
val dir = s"dbfs:/datasets/$group/"

// Some file names
val stopwords_file = "stopwords.csv"
val stopwords_store = s"${dir}${stopwords_file}"

val wp_store = s"${dir}${wp_file}"

val filtered_articles_store = s"${dir}filtered_articles_${lang}_${freeze_date}"
val lda_countVector_store = s"${dir}lda_countvector_${lang}_${freeze_date}"

val lda_model_store = s"${dir}lda_${lang}_${freeze_date}.model"

lang: String = sv
freeze_date: String = 20201120
wp_file: String = svwiki-20201120-pages-articles.xml
wp_zip_remote: String = https://dumps.wikimedia.org/svwiki/20201120/svwiki-20201120-pages-articles.xml.bz2
stopwords_remote: String = https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv
group: String = 05_lda
localpath: String = file:/databricks/driver/05_lda/
dir_local: String = 05_lda/
dir: String = dbfs:/datasets/05_lda/
stopwords_file: String = stopwords.csv
stopwords_store: String = dbfs:/datasets/05_lda/stopwords.csv
wp_store: String = dbfs:/datasets/05_lda/svwiki-20201120-pages-articles.xml
filtered_articles_store: String = dbfs:/datasets/05_lda/filtered_articles_sv_20201120
lda_countVector_store: String = dbfs:/datasets/05_lda/lda_countvector_sv_20201120
lda_model_store: String = dbfs:/datasets/05_lda/lda_sv_20201120.model

# Just check where local path is
pwd

/databricks/driver

// Run this only to force a clean start
/*
// Delete DBFS store
dbutils.fs.rm(dir,recurse=true)
// Delete local files
import org.apache.commons.io.FileUtils;
import java.io.File;
FileUtils.deleteDirectory(new File(dir_local))
*/

import org.apache.commons.io.FileUtils
import java.io.File

// Prepare DBFS store
dbutils.fs.mkdirs(dir)
display(dbutils.fs.ls(dir))
// Prepare local folder
import org.apache.commons.io.FileUtils;
import java.io.File;
FileUtils.forceMkdir(new File(dir_local));

// Check local files
import sys.process._  // For spawning shell commands
s"ls -l ./  ${dir_local}" !!

warning: there was one feature warning; for details, enable `:setting -feature' or `:replay -feature'
import sys.process._
res19: String =
"./:
total 24
drwxr-xr-x 2 root root 4096 Jan 26 04:22 05_lda
drwxr-xr-x 2 root root 4096 Jan  1  1970 conf
-rw-r--r-- 1 root root  733 Jan 26 03:34 derby.log
drwxr-xr-x 3 root root 4096 Jan 26 03:33 eventlogs
drwxr-xr-x 2 root root 4096 Jan 26 04:15 ganglia
drwxr-xr-x 2 root root 4096 Jan 26 04:03 logs

05_lda/:
total 0
"

// Download stopwords to store.
val local = s"${dir_local}${stopwords_file}"
val remote = stopwords_remote
try
{
  // -nc prevents download if already exist. !! converts wget output to string.
  s"wget -nc -O${local} ${remote}" !!
}
catch
{
  case e: Throwable => println("Exception: " + e)
}

--2021-01-26 04:23:45--  https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.52.133
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.52.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1936 (1.9K) [text/plain]
Saving to: ‘05_lda/stopwords.csv’

     0K .                                                     100% 15.9M=0s

2021-01-26 04:23:45 (15.9 MB/s) - ‘05_lda/stopwords.csv’ saved [1936/1936]

warning: there was one feature warning; for details, enable `:setting -feature' or `:replay -feature'
local: String = 05_lda/stopwords.csv
remote: String = https://raw.githubusercontent.com/peterdalle/svensktext/master/stoppord/stoppord.csv
res20: Any = ""

// Check local stopwords file
s"ls -l ${dir_local}" !!

warning: there was one feature warning; for details, enable `:setting -feature' or `:replay -feature'
res21: String =
"total 4
-rw-r--r-- 1 root root 1936 Jan 26 04:23 stopwords.csv
"

dbutils.fs.cp(localpath + stopwords_file, dir)

res22: Boolean = true

display(dbutils.fs.ls(stopwords_store))

// Download Wikipedia xml to store. -nc prevents overwrite if already exist. !! converts wget output to string.
val local = s"${dir_local}${wp_file}.bz2"
val remote = wp_zip_remote
try
{
  s"wget -nc -O${local} ${remote}" !!
}
catch
{
  case e: Throwable => println("Exception: " + e)
}

// Unpack bz2 to xml.
val local = s"${dir_local}${wp_file}.bz2"
try
{
  // -d unpack, -k keep source file, -v activate some output. !! converts output to string.
  s"bzip2 -dk -f -v ${local}" !!
}
catch
{
  case e: Throwable => println("Exception: " + e)
}

  05_lda/svwiki-20201120-pages-articles.xml.bz2: done
warning: there was one feature warning; for details, enable `:setting -feature' or `:replay -feature'
local: String = 05_lda/svwiki-20201120-pages-articles.xml.bz2
res27: Any = ""

s"ls ${dir_local}" !!

warning: there was one feature warning; for details, enable `:setting -feature' or `:replay -feature'
res28: String =
"stopwords.csv
svwiki-20201120-pages-articles.xml
svwiki-20201120-pages-articles.xml.bz2
"

val local = s"${localpath}${wp_file}"
dbutils.fs.cp(local, dir)

local: String = file:/databricks/driver/05_lda/svwiki-20201120-pages-articles.xml
res29: Boolean = true

display(dbutils.fs.ls(wp_store))

// Discard the local files no longer needed
import org.apache.commons.io.FileUtils;
import java.io.File;
FileUtils.cleanDirectory(new File(dir_local))

import org.apache.commons.io.FileUtils
import java.io.File

// ---------------spark_import
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext

// ----------------xml_loader_import
import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Text
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{ LongWritable, Text }
import com.databricks.spark.xml._

import org.apache.hadoop.conf.Configuration

// ---- Matching
import scala.util.matching.UnanchoredRegex

import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Text
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{LongWritable, Text}
import com.databricks.spark.xml._
import org.apache.hadoop.conf.Configuration
import scala.util.matching.UnanchoredRegex

def readWikiDump(sc: SparkContext, file: String) : RDD[String] = {
  val conf = new Configuration()
  conf.set(XmlInputFormat.START_TAG_KEY, "<page>")
  conf.set(XmlInputFormat.END_TAG_KEY, "</page>")
  val rdd = sc.newAPIHadoopFile(file,
                                classOf[XmlInputFormat],
                                classOf[LongWritable],
                                classOf[Text],
                                conf)
  rdd.map{case (k,v) => (new String(v.copyBytes()))}
}

readWikiDump: (sc: org.apache.spark.SparkContext, file: String)org.apache.spark.rdd.RDD[String]

// Read the wiki dump
val dump = readWikiDump(sc, wp_store)

val n_wp_pages = dump.count()
dump.take(5)

val redirection = "<redirect title=".r
def is_not_redirect(source : String) = { redirection.findFirstIn(source) == None }

val ns0 = "<ns>0</ns>".r
def is_article(source : String) = { ns0.findFirstIn(source) != None }

val autogenerated = "robotskapad|Robotskapad".r
def is_not_autogenerated(source : String) = { autogenerated.findFirstIn(source) == None }

val articles = dump.filter(is_not_redirect).filter(is_article).filter(is_not_autogenerated)

val n_articles = articles.count()
articles.take(5)

val id_regexp = raw"<id>([0-9]*)</id>".r
def extract_id(source : String) = {
  val m = id_regexp.findFirstMatchIn(source).get
  m.group(1).toLong
}

val title_regexp = raw"<title>([^\<\>]*)</title>".r
def extract_title(source : String) = {
  val m = title_regexp.findFirstMatchIn(source).get
  m.group(1)
}

val labelled_articles = articles.map(source => (extract_id(source), extract_title(source), source))

labelled_articles.take(5).map({case(id, title, contents) => title})

id_regexp: scala.util.matching.Regex = <id>([0-9]*)</id>
extract_id: (source: String)Long
title_regexp: scala.util.matching.Regex = <title>([^\<\>]*)</title>
extract_title: (source: String)String
labelled_articles: org.apache.spark.rdd.RDD[(Long, String, String)] = MapPartitionsRDD[1021] at map at command-2629358840275853:13
res11: Array[String] = Array(Amager, Afrika, Amerika, Abbekås, Alingsås)

def filter_xml(source : String) = {
  val noAmp = raw"&".r.replaceAllIn(source, "&")
  val noNbsp = raw" ".r.replaceAllIn(noAmp, " ")
  val noQuot = raw""".r.replaceAllIn(noNbsp, "")
  val noLt = raw"&lt;".r.replaceAllIn(noQuot, "<")
  val noGt = raw"&gt;".r.replaceAllIn(noLt, ">")



  val tags = Seq("id", "ns", "parentid", "timestamp", "contributor", "comment", "model", "format", "sha1")
  var current = noGt
  for (tag <- tags) {
    val start = s"<$tag>".r
    val end = s"</$tag>".r
    while (start.findAllIn(current).hasNext) {
      val a = start.findAllIn(current).start
      val b = end.findAllIn(current).start
      val len = current.length
      current = current.substring(0, a) + current.substring(b+tag.length+3, current.length)
      assert (current.length < len)
    }
  }
  current = raw"<[^<>]*>".r.replaceAllIn(current, "")

  // A loop to remove innermost curly braces, then outer such things.
  val curly = raw"\{[^\{\}]*\}".r
  while (curly.findAllIn(current).hasNext)
  {
     current = curly.replaceAllIn(current, "")
  }

  current
  }


val filtered_xml = labelled_articles.map({ case(id, title, source) => (id, title, filter_xml(source)) })

filtered_xml.take(5)

def sanitize(source: String) : String = {
  val noDollar = source.replaceAllLiterally("$", "")
  val noBackSlash = noDollar.replaceAllLiterally(raw"\", "")
  noBackSlash
}

val sanitized_xml = filtered_xml.map({case(id, title, source) => (id, title, sanitize(source))})

sanitize: (source: String)String
sanitized_xml: org.apache.spark.rdd.RDD[(Long, String, String)] = MapPartitionsRDD[1026] at map at command-2629358840275860:7

def filterMarkup(source: String) : String = {

  // Loop to remove some specific sections
  val sections = Seq("Se även", "Referenser", "Noter", "Källor", "Externa länkar")
  var current = source
  for (section <- sections) {
    val regexp = s"[=]+ *$section *[=]+".r
    if (regexp.findAllIn(current).hasNext)
    {
      val a = regexp.findAllIn(current).start
      val pre = current.substring(0, a)
      var post = current.substring(a, current.length)
      post = regexp.replaceAllIn(post, "")
      if ("^=".r.findAllIn(post).hasNext) {
        val b = "^=".r.findAllIn(post).start
        post = post.substring(b, post.length)
      }
      else {
        post = ""        
      }
      current = pre + post
    }
  }

  // Unnamed internal links appear in figure captions, confusing later matching
  val noUnnamedInternalLinks = raw"\[\[([^\|\]]*)\]\]".r.replaceAllIn(current, m => m.group(1))

  // Loop to remove internal links and their properties
  current = noUnnamedInternalLinks
  val regexp = raw"\[\[[^\|\[]*\|([^\[\]]*)\]\]".r
  while (regexp.findAllIn(current).hasNext) {
    val len = current.length
    current = regexp.replaceAllIn(current, m => "[[" + m.group(1) + "]]")
    assert (current.length < len)
  }
  val noNamedInternalLinks = current

  val noInternalLinks = raw"\[\[([^\]]*)\]\]".r.replaceAllIn(noNamedInternalLinks, m => m.group(1))
  val noNamedExternalLinks = raw"\[[^ \[]*\ ([^\]]*)\]".r.replaceAllIn(noInternalLinks, m => m.group(1))
  val noExternalLinks = raw"\[([^\]]*)\]".r.replaceAllIn(noNamedExternalLinks, m => m.group(1))


 val lowerCase = noExternalLinks.toLowerCase  

  val noQuoteChars = raw"'".r.replaceAllIn(lowerCase, "")
  // Loop to remove double whitespace characters
  current = noQuoteChars
  val doubleWhitespace = raw"\s(\s)".r
  while (doubleWhitespace.findAllIn(current).hasNext) {
    current = doubleWhitespace.replaceAllIn(current, m => m.group(1))
  }
  val noDoubleWhitespace = current
  noDoubleWhitespace
}


val filtered = sanitized_xml.map({case (id, title, source) => (id, title, filterMarkup(source))})

filtered.take(2)

// dbutils.fs.rm(filtered_articles_store, recurse=true)

res19: Boolean = true

val outputs = filtered.map({case (id, title, contents) => (id, title.replaceAllLiterally(",", "**COMMA**"), contents.replaceAllLiterally("\n", "**NEWLINE**"))})
outputs.saveAsTextFile(filtered_articles_store);

outputs: org.apache.spark.rdd.RDD[(Long, String, String)] = MapPartitionsRDD[1038] at map at command-4443336225071544:1

val format = "^\\(([^,]*),([^,]*),(.*)\\)$".r
def parse_line(s : String) = {
  assert (s(0) == '(')
  val firstComma = s.indexOf(",")
  val id = s.substring(1, firstComma).toLong
  val afterId = s.substring(firstComma+1, s.length)
  val nextComma = afterId.indexOf(",")
  val title = afterId.substring(0, nextComma)
  val contents = afterId.substring(nextComma+1, afterId.length-1)

  assert (afterId(afterId.length-1) == ')')
  (id, title.replaceAllLiterally("**COMMA**", ","), contents.replaceAllLiterally("**NEWLINE**", "\n"))
}

val filtered = spark.sparkContext.textFile(filtered_articles_store).map(parse_line)
val n_articles = filtered.count()

format: scala.util.matching.Regex = ^\(([^,]*),([^,]*),(.*)\)$
parse_line: (s: String)(Long, String, String)
filtered: org.apache.spark.rdd.RDD[(Long, String, String)] = MapPartitionsRDD[1042] at map at command-4443336225071727:15
n_articles: Long = 813245

val corpus_df = filtered.toDF("id", "title", "contents")

corpus_df: org.apache.spark.sql.DataFrame = [id: bigint, title: string ... 1 more field]

display(corpus_df.sample(0.0001))

import org.apache.spark.ml.feature.RegexTokenizer

// Set params for RegexTokenizer
val tokenizer = new RegexTokenizer()
  .setGaps(true) // pattern below identifies non-word alphabet
  .setPattern("[^a-zåäö]+") // break word and remove any detected non-word character(s).
  .setMinTokenLength(4) // Filter away tokens with length < 4
  .setInputCol("contents") // name of the input column
  .setOutputCol("tokens") // name of the output column

// Tokenize document
val tokenized_df = tokenizer.transform(corpus_df)

import org.apache.spark.ml.feature.RegexTokenizer
tokenizer: org.apache.spark.ml.feature.RegexTokenizer = RegexTokenizer: uid=regexTok_5d41547188e0, minTokenLength=4, gaps=true, pattern=[^a-zåäö]+, toLowercase=true
tokenized_df: org.apache.spark.sql.DataFrame = [id: bigint, title: string ... 2 more fields]

display(tokenized_df.sample(0.0001))

val swedish_stopwords = sc.textFile(stopwords_store).collect()
val wikipedia_stopwords = Array("label", "note", "area", "unit", "type", "mark", "long", "right", "kartposition", "quot", "text", "title", "page", "timestamp", "revision", "name", "username", "sha1", "format", "coord", "left", "center", "align", "region", "nasa", "source", "mouth", "species", "highest", "style", "kategori", "http", "wikipedia", "referenser", "källor", "noter")
// Combine newly identified stopwords to our exising list of stopwords
val stopwords = swedish_stopwords.union(wikipedia_stopwords)

import org.apache.spark.ml.feature.StopWordsRemover

val remover = new StopWordsRemover()
  .setStopWords(stopwords)
  .setInputCol("tokens")
  .setOutputCol("filtered")

val filtered_df = remover.transform(tokenized_df)

import org.apache.spark.ml.feature.StopWordsRemover
remover: org.apache.spark.ml.feature.StopWordsRemover = StopWordsRemover: uid=stopWords_ecf00fa02fd5, numStopWords=366, locale=en, caseSensitive=false
filtered_df: org.apache.spark.sql.DataFrame = [id: bigint, title: string ... 3 more fields]

import org.apache.spark.ml.feature.CountVectorizer

val vectorizer = new CountVectorizer()
  .setInputCol("filtered")
  .setOutputCol("features")
  .setVocabSize(10000)
  .setMinDF(5) // the minimum number of different documents a term must appear in to be included in the vocabulary.
  .fit(filtered_df)

import org.apache.spark.ml.feature.CountVectorizer
vectorizer: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVec_ad38aa4e1641, vocabularySize=10000

import org.apache.spark.ml.linalg.Vector

val countVectors = vectorizer.transform(filtered_df).select("id", "features")
val lda_countVector = countVectors.map { case Row(id: Long, countVector: Vector) => (id, countVector) }
val lda_countVector_mllib = lda_countVector.map { case (id, vector) => (id, org.apache.spark.mllib.linalg.Vectors.fromML(vector)) }.rdd

lda_countVector_mllib.take(1)

import org.apache.spark.ml.linalg.Vector
countVectors: org.apache.spark.sql.DataFrame = [id: bigint, features: vector]
lda_countVector: org.apache.spark.sql.Dataset[(Long, org.apache.spark.ml.linalg.Vector)] = [_1: bigint, _2: vector]
lda_countVector_mllib: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[1088] at rdd at command-4443336225071537:5
res41: Array[(Long, org.apache.spark.mllib.linalg.Vector)] = Array((1,(10000,[1,2,8,9,15,17,20,26,37,38,39,42,48,50,51,55,64,67,68,71,75,80,81,85,86,94,95,96,106,114,117,118,128,129,140,149,150,156,161,189,195,208,216,297,299,332,336,399,409,427,452,457,459,461,465,535,540,572,603,612,686,739,747,806,849,904,922,983,1002,1013,1037,1081,1128,1167,1170,1175,1272,1274,1295,1334,1374,1387,1474,1578,1593,1602,1707,1750,1764,1792,1835,1852,1927,2022,2027,2114,2182,2268,2358,2612,2726,2766,2929,3316,3386,3415,3468,3486,3588,3593,3634,3693,3735,4156,4184,4335,4339,4470,4579,4675,4691,4851,5046,5083,5118,5186,5355,5653,5712,5801,6051,6080,6140,6751,7381,7491,7602,8758,8892,8944,8990,9148,9185,9375,9416,9737,9779,9870],[7.0,2.0,1.0,3.0,3.0,1.0,1.0,4.0,4.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,3.0,1.0,1.0,1.0,1.0,1.0,5.0,2.0,2.0,1.0,4.0,2.0,1.0,4.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,2.0,2.0,1.0,1.0,1.0,1.0,3.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,4.0,1.0,3.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,2.0,1.0,1.0,1.0,4.0,1.0,1.0,1.0,1.0,2.0,1.0,2.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,3.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0])))

println(lda_countVector_store)

// lda_countVector_mllib.saveAsTextFile(lda_countVector_store);

dbfs:/datasets/05_lda/lda_countvector_sv_20201120

val numTopics = 50
val maxIterations = 20

numTopics: Int = 50
maxIterations: Int = 20

import org.apache.spark.mllib.clustering.{LDA, OnlineLDAOptimizer}

val lda = new LDA()
  .setOptimizer(new OnlineLDAOptimizer().setMiniBatchFraction(0.8))
  .setK(numTopics)
  .setMaxIterations(maxIterations)
  .setDocConcentration(-1) // use default values
  .setTopicConcentration(-1) // use default values

import org.apache.spark.mllib.clustering.{LDA, OnlineLDAOptimizer}
lda: org.apache.spark.mllib.clustering.LDA = org.apache.spark.mllib.clustering.LDA@54e767e2

val ldaModel = lda.run(lda_countVector_mllib)

ldaModel: org.apache.spark.mllib.clustering.LDAModel = org.apache.spark.mllib.clustering.LocalLDAModel@5c324bb0

// Save the model to file for later use.
dbutils.fs.rm(lda_model_store, recurse=true)
ldaModel.save(sc, lda_model_store)

val maxTermsPerTopic = 8
val topicIndices = ldaModel.describeTopics(maxTermsPerTopic)
val vocabList = vectorizer.vocabulary
val topics = topicIndices.map { case (terms, termWeights) =>
  terms.map(vocabList(_)).zip(termWeights)
}

println(s"$numTopics topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
  println(s"TOPIC $i")
  topic.foreach { case (term, weight) => println(s"$term\t$weight") }
  println(s"==========")
}

Zip topic terms with topic IDs
val termArray = topics.zipWithIndex

// Transform data into the form (term, probability, topicId)
val termRDD = sc.parallelize(termArray)
val termRDD2 = termRDD.flatMap( (x: (Array[(String, Double)], Int) ) => {
  val arrayOfTuple = x._1
  val topicId = x._2
  arrayOfTuple.map(el => (el._1, el._2, topicId))
})

termRDD: org.apache.spark.rdd.RDD[(Array[(String, Double)], Int)] = ParallelCollectionRDD[1881] at parallelize at command-4443336225072171:2
termRDD2: org.apache.spark.rdd.RDD[(String, Double, Int)] = MapPartitionsRDD[1882] at flatMap at command-4443336225072171:3

// Create DF with proper column names
val termDF = termRDD2.toDF.withColumnRenamed("_1", "term").withColumnRenamed("_2", "probability").withColumnRenamed("_3", "topicId")

termDF: org.apache.spark.sql.DataFrame = [term: string, probability: double ... 1 more field]

 display(termDF)

Create JSON data
val rawJson = termDF.toJSON.collect().mkString(",\n")

displayHTML(s"""
<!DOCTYPE html>
<meta charset="utf-8">
<style>

circle {
  fill: rgb(31, 119, 180);
  fill-opacity: 0.5;
  stroke: rgb(31, 119, 180);
  stroke-width: 1px;
}

.leaf circle {
  fill: #ff7f0e;
  fill-opacity: 1;
}

text {
  font: 14px sans-serif;
}

</style>
<body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script>

var json = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson}
     ]
    }
   ]
};

var r = 1000,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble = d3.layout.pack()
    .sort(null)
    .size([r, r])
    .padding(1.5);

var vis = d3.select("body").append("svg")
    .attr("width", r)
    .attr("height", r)
    .attr("class", "bubble");


var node = vis.selectAll("g.node")
    .data(bubble.nodes(classes(json))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });

// Returns a flattened hierarchy containing all leaf nodes under the root.
function classes(root) {
  var classes = [];

  function recurse(term, node) {
    if (node.children) node.children.forEach(function(child) { recurse(node.term, child); });
    else classes.push({topicName: node.topicId, className: node.term, value: node.probability});
  }

  recurse(null, root);
  return {children: classes};
}
</script>
""")

// Load the model
import org.apache.spark.mllib.clustering.LocalLDAModel
val model = LocalLDAModel.load(sc, lda_model_store)

import org.apache.spark.mllib.clustering.LocalLDAModel
model: org.apache.spark.mllib.clustering.LocalLDAModel = org.apache.spark.mllib.clustering.LocalLDAModel@6067f4d5

// Load the countvector
// val lda_countVector_mllib_loaded = sc.textFile("lda_countvector_sv_20201208") // This doesn't work, need to figure out how to parse the loaded rdd
// print(lda_countVector_mllib_loaded)

val numsample = 20
val sample = lda_countVector_mllib.takeSample(false, numsample)

val sampleRdd = sc.makeRDD(sample)
val topicDistributions = model.topicDistributions(sampleRdd)
val MLTopics = topicDistributions.map({case(id, vs) => (id, vs.argmax)})
val ids = MLTopics.map({case(id, vs)=>id}).collect()
val selection = filtered.filter({case(id, title, contents) => ids.contains(id)}).map({case(id, title, contents) => (id, title)}).collect()
val allTopicProbs = model.topicDistributions(lda_countVector_mllib)

sampleRdd: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = ParallelCollectionRDD[2329] at makeRDD at command-4443336225072182:1
topicDistributions: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[2330] at map at LDAModel.scala:373
MLTopics: org.apache.spark.rdd.RDD[(Long, Int)] = MapPartitionsRDD[2331] at map at command-4443336225072182:3
ids: Array[Long] = Array(2779114, 226060, 1613204, 4970370, 700470, 636341, 4987605, 766457, 4965484, 82392, 7982574, 7973374, 367933, 1204225, 1225253, 503655, 821159, 8299631, 4684122, 1695311)
selection: Array[(Long, String)] = Array((82392,Senmedeltiden), (226060,Río Verde, San Luis Potosí), (367933,Kontrollsumma), (503655,Adam Kårsnäs), (636341,Live at McCabe's Guitar Shop), (700470,R2-0), (766457,Pakistanska muslimska förbundet - L), (821159,Fisher Athletic FC), (1204225,Financial Institutions Reform, Recovery and Enforcement Act of 1989), (1225253,Gardar), (1613204,Bessan), (1695311,Borneol), (2779114,Sara Branham Matthews), (4684122,Cecil Blachford), (4965484,Riddarsynerätt), (4970370,Kenyaskratthärfågel), (4987605,Rihannas diskografi), (7973374,Ahtisaariplanen), (7982574,James S. Brown), (8299631,Mount Roper))
allTopicProbs: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[2335] at map at LDAModel.scala:373

for ( (article_id, topic_id) <- MLTopics.collect() ) {
  val title = selection.filter({case(id, title) => id == article_id})(0)._2
  val topicProbs = allTopicProbs.map({case(id, vs) => (id, vs(topic_id))})
  val top5articles = topicProbs.takeOrdered(5)(Ordering[Double].reverse.on(_._2))
  val top5ids = top5articles.map(_._1)
  val top5titles = filtered.filter({case(id, title, contents) => top5ids.contains(id) }).map({case(id, title, contents) => (title)}).collect
  println(s"Top 5 articles on the same topic as ${title}:")

  for (t <- top5titles) println(s" * ${t}")
  println("")
}

displayHTML("""<iframe width="99%" height="340" src="https://my.matterport.com/show/?m=yYCddcrq6Zj" frameborder="0" allowfullscreen allow="xr-spatial-tracking"></iframe>""")

displayHTML("""<iframe width="99%" height="340" src="https://videos.cern.ch/video/OPEN-VIDEO-2018-041-001" frameborder="0" allowfullscreen allow="xr-spatial-tracking"></iframe>""")

displayHTML("""<iframe width="99%" height="340" src="https://videos.cern.ch/video/CERN-VIDEO-2011-192-001" frameborder="0" allowfullscreen allow="xr-spatial-tracking"></iframe>""")

displayHTML("""<a href="https://arxiv.org/pdf/2010.07106.pdf">
<img border="0" alt="UCluster paper" src="https://paperswithcode.com/static/thumbs/2010.07106.jpg" width="300" height="300">
</a>""")

dbutils.fs.ls("dbfs:///FileStore/06_LHC")

res0: Seq[com.databricks.backend.daemon.dbutils.FileInfo] = WrappedArray(FileInfo(dbfs:/FileStore/06_LHC/LICENSE, LICENSE, 1071), FileInfo(dbfs:/FileStore/06_LHC/README.md, README.md, 3150), FileInfo(dbfs:/FileStore/06_LHC/data/, data/, 0), FileInfo(dbfs:/FileStore/06_LHC/h5/, h5/, 0), FileInfo(dbfs:/FileStore/06_LHC/models/, models/, 0), FileInfo(dbfs:/FileStore/06_LHC/scripts/, scripts/, 0), FileInfo(dbfs:/FileStore/06_LHC/utils/, utils/, 0))

pip install h5py

rm -r 06_LHC
mkdir 06_LHC

cd 06_LHC

wget https://github.com/dgedon/ProjectParticleClusteringv2/archive/main.zip
unzip main.zip
mv ProjectParticleClusteringv2-main/* .
rm -r ProjectParticleClusteringv2-main/ main.zip

cd 06_LHC
mkdir data
cd data


wget https://zenodo.org/record/3602254/files/hls4ml_LHCjet_100p_train.tar.gz
#mkdir data
#mv hls4ml_LHCjet_100p_train.tar.gz data
#cd data
tar --no-same-owner -xvf hls4ml_LHCjet_100p_train.tar.gz

cd 06_LHC/data
wget https://zenodo.org/record/3602254/files/hls4ml_LHCjet_100p_val.tar.gz
tar --no-same-owner -xvf hls4ml_LHCjet_100p_val.tar.gz

cd 06_LHC/scripts
python prepare_data_multi.py --dir ../data/

cd 06_LHC/scripts
python prepare_data_multi.py --dir ../data/ --make_eval

rm -r ../data

dbutils.fs.cp ("file:////databricks/driver/06_LHC", "dbfs:///FileStore/06_LHC", recurse=true) 

dbutils.fs.cp("dbfs:///FileStore/06_LHC", "file:////databricks/driver/06_LHC", recurse=true)

res0: Boolean = true

ls 06_LHC/

ls 06_LHC

LICENSE
README.md
data
h5
models
scripts
utils

import argparse
from argparse import Namespace
from datetime import datetime
import numpy as np
import tensorflow as tf
import socket
import os
import sys
from sklearn.cluster import KMeans
from tqdm import tqdm
#Added matplot for accuracy
import matplotlib.pyplot as plt

np.set_printoptions(edgeitems=1000)

from scipy.optimize import linear_sum_assignment

BASE_DIR = os.path.join(os.getcwd(), '06_LHC','scripts')  
#os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR, '..', 'models'))
sys.path.append(os.path.join(BASE_DIR, '..', 'utils'))
import provider
import gapnet_classify as MODEL

parserdict = {'max_dim': 3, #help='Dimension of the encoding layer [Default: 3]')
              'n_clusters': 3, #help='Number of clusters [Default: 3]')
              'gpu': 0, #help='GPU to use [default: GPU 0]')
              'model': 'gapnet_clasify', #help='Model name [default: gapnet_classify]')
              'log_dir': 'log', #help='Log dir [default: log]')
              'num_point': 100, #help='Point Number [default: 100]')
              'max_epoch': 100, #help='Epoch to run [default: 200]')
              'epochs_pretrain': 10, #help='Epochs to for pretraining [default: 10]')
              'batch_size': 1024, #help='Batch Size during training [default: 512]')
              'learning_rate': 0.001, #help='Initial learning rate [default: 0.01]')

              'momentum': 0.9, #help='Initial momentum [default: 0.9]')
              'optimizer': 'adam', #help='adam or momentum [default: adam]')
              'decay_step': 500000, #help='Decay step for lr decay [default: 500000]')
              'wd': 0.0, #help='Weight Decay [Default: 0.0]')
              'decay_rate': 0.5, #help='Decay rate for lr decay [default: 0.5]')
              'output_dir': 'train_results', #help='Directory that stores all training logs and trained models')
              'data_dir': os.path.join(os.getcwd(),'06_LHC', 'h5'), # '../h5', #help='directory with data [default: hdf5_data]')
              'nfeat': 8, #help='Number of features [default: 8]')
              'ncat': 20, #help='Number of categories [default: 20]')
             }

FLAGS = Namespace(**parserdict)
H5_DIR = FLAGS.data_dir

EPOCH_CNT = 0
MAX_PRETRAIN = FLAGS.epochs_pretrain
BATCH_SIZE = FLAGS.batch_size
NUM_POINT = FLAGS.num_point
NUM_FEAT = FLAGS.nfeat
NUM_CLASSES = FLAGS.ncat
MAX_EPOCH = FLAGS.max_epoch
BASE_LEARNING_RATE = FLAGS.learning_rate
GPU_INDEX = FLAGS.gpu
MOMENTUM = FLAGS.momentum
OPTIMIZER = FLAGS.optimizer
DECAY_STEP = FLAGS.decay_step
DECAY_RATE = FLAGS.decay_rate

# MODEL = importlib.import_module(FLAGS.model) # import network module
MODEL_FILE = os.path.join(BASE_DIR, 'models', FLAGS.model + '.py')
LOG_DIR = os.path.join('..', 'logs', FLAGS.log_dir)

if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR)
os.system('cp %s.py %s' % (MODEL_FILE, LOG_DIR))  # bkp of model def
os.system('cp train_kmeans.py %s' % (LOG_DIR))  # bkp of train procedure

BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

LEARNING_RATE_CLIP = 1e-5
HOSTNAME = socket.gethostname()

TRAIN_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'train_files_wztop.txt'))
TEST_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'test_files_wztop.txt'))

def get_learning_rate(batch):
    learning_rate = tf.compat.v1.train.exponential_decay(
        BASE_LEARNING_RATE,  # Base learning rate.
        batch * BATCH_SIZE,  # Current index into the dataset.
        DECAY_STEP,  # Decay step.
        DECAY_RATE,  # Decay rate.
        staircase=True)
    learning_rate = tf.maximum(learning_rate, LEARNING_RATE_CLIP)  # CLIP THE LEARNING RATE!
    return learning_rate


def get_bn_decay(batch):
    bn_momentum = tf.compat.v1.train.exponential_decay(
        BN_INIT_DECAY,
        batch * BATCH_SIZE,
        BN_DECAY_DECAY_STEP,
        BN_DECAY_DECAY_RATE,
        staircase=True)
    bn_decay = tf.minimum(BN_DECAY_CLIP, 1 - bn_momentum)
    return bn_decay

def train():
    with tf.Graph().as_default():
        with tf.device('/gpu:' + str(GPU_INDEX)):
            #ADDED THIS TO RECORD ACCURACY
            epochs_acc = []
            pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT, NUM_FEAT)

            is_training_pl = tf.compat.v1.placeholder(tf.bool, shape=())

            # Note the global_step=batch parameter to minimize.
            # That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
            batch = tf.Variable(0)
            alpha = tf.compat.v1.placeholder(dtype=tf.float32, shape=())
            bn_decay = get_bn_decay(batch)
            tf.compat.v1.summary.scalar('bn_decay', bn_decay)
            print("--- Get model and loss")

            pred, max_pool = MODEL.get_model(pointclouds_pl, is_training=is_training_pl,
                                             bn_decay=bn_decay,
                                             num_class=NUM_CLASSES, weight_decay=FLAGS.wd,
                                             )

            class_loss = MODEL.get_focal_loss(pred, labels_pl, NUM_CLASSES)
            mu = tf.Variable(tf.zeros(shape=(FLAGS.n_clusters, FLAGS.max_dim)), name="mu",
                             trainable=True)  # k centroids
            kmeans_loss, stack_dist = MODEL.get_loss_kmeans(max_pool, mu, FLAGS.max_dim,
                                                            FLAGS.n_clusters, alpha)

            full_loss = kmeans_loss + class_loss

            print("--- Get training operator")
            # Get training operator
            learning_rate = get_learning_rate(batch)
            tf.compat.v1.summary.scalar('learning_rate', learning_rate)
            if OPTIMIZER == 'momentum':
                optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate, momentum=MOMENTUM)
            elif OPTIMIZER == 'adam':
                optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)

            train_op_full = optimizer.minimize(full_loss, global_step=batch)
            train_op = optimizer.minimize(class_loss, global_step=batch)

            # Add ops to save and restore all the variables.
            saver = tf.compat.v1.train.Saver()

        # Create a session
        config = tf.compat.v1.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        config.log_device_placement = False
        sess = tf.compat.v1.Session(config=config)

        sess.run(tf.compat.v1.global_variables_initializer())

        # Add summary writers
        merged = tf.compat.v1.summary.merge_all()
        train_writer = tf.compat.v1.summary.FileWriter(os.path.join(LOG_DIR, 'train'), sess.graph)
        test_writer = tf.compat.v1.summary.FileWriter(os.path.join(LOG_DIR, 'test'), sess.graph)

        # Init variables
        print("Total number of weights for the model: ",
              np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]))
        ops = {'pointclouds_pl': pointclouds_pl,
               'labels_pl': labels_pl,
               'is_training_pl': is_training_pl,
               'max_pool': max_pool,
               'pred': pred,
               'alpha': alpha,
               'mu': mu,
               'stack_dist': stack_dist,
               'class_loss': class_loss,
               'kmeans_loss': kmeans_loss,
               'train_op': train_op,
               'train_op_full': train_op_full,
               'merged': merged,
               'step': batch,
               'learning_rate': learning_rate
               }

        for epoch in range(MAX_EPOCH):
            print('\n**** EPOCH %03d ****' % (epoch))
            sys.stdout.flush()

            is_full_training = epoch > MAX_PRETRAIN
            max_pool = train_one_epoch(sess, ops, train_writer, is_full_training)
            if epoch == MAX_PRETRAIN:
                centers = KMeans(n_clusters=FLAGS.n_clusters).fit(np.squeeze(max_pool))
                centers = centers.cluster_centers_
                sess.run(tf.compat.v1.assign(mu, centers))

            #eval_one_epoch(sess, ops, test_writer, is_full_training)
            #Added this line to record accuracy
            epoch_acc = eval_one_epoch(sess, ops, test_writer, is_full_training)
            epochs_acc.append(epoch_acc)
            if is_full_training:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'cluster.ckpt'))
            else:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))
            print("Model saved in file: %s" % save_path)
        plt.plot(epochs_acc)
        plt.ylabel('Validation accuracy')
        plt.xlabel('epochs')
        plt.show()
        plt.savefig('single_maching.png')

def get_batch(data, label, start_idx, end_idx):
    batch_label = label[start_idx:end_idx]
    batch_data = data[start_idx:end_idx, :, :]
    return batch_data, batch_label


def cluster_acc(y_true, y_pred):
    """
    Calculate clustering accuracy. Require scikit-learn installed
    """
    y_true = y_true.astype(np.int64)
    D = max(y_pred.max(), y_true.max()) + 1
    w = np.zeros((D, D), dtype=np.int64)
    for i in range(y_pred.size):
        w[y_pred[i], y_true[i]] += 1
    ind = linear_sum_assignment(w.max() - w)
    ind = np.asarray(ind)
    ind = np.transpose(ind)
    return sum([w[i, j] for i, j in ind]) * 1.0 / y_pred.size

def train_one_epoch(sess, ops, train_writer, is_full_training):
    """ ops: dict mapping from string to tf ops """
    is_training = True

    train_idxs = np.arange(0, len(TRAIN_FILES))

    acc = loss_sum = 0
    y_pool = []
    for fn in range(len(TRAIN_FILES)):
        # print('----' + str(fn) + '-----')
        current_file = os.path.join(H5_DIR, TRAIN_FILES[train_idxs[fn]])
        current_data, current_label, current_cluster = provider.load_h5_data_label_seg(current_file)

        current_label = np.squeeze(current_label)

        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        # num_batches = 5
        print(str(datetime.now()))

        # initialise progress bar
        process_desc = "TRAINING: Loss {:2.3e}"
        progress_bar = tqdm(initial=0, leave=True, total=num_batches,
                            desc=process_desc.format(0),
                            position=0)
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx + 1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label, start_idx, end_idx)
            cur_batch_size = end_idx - start_idx

            # print(batch_weight)
            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['labels_pl']: batch_label,
                         ops['is_training_pl']: is_training,
                         ops['alpha']: 2 * (EPOCH_CNT - MAX_PRETRAIN + 1),}
            if is_full_training:
                summary, step, _, loss_val, dist, lr = sess.run([ops['merged'], ops['step'],
                                                                 ops['train_op_full'], ops['kmeans_loss'],
                                                                 ops['stack_dist'], ops['learning_rate']],
                                                                feed_dict=feed_dict)

                batch_cluster = np.array([np.where(r == 1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)

                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc += cluster_acc(batch_cluster, cluster_assign)
            else:
                summary, step, _, loss_val, max_pool, lr = sess.run([ops['merged'], ops['step'],
                                                                     ops['train_op'], ops['class_loss'],
                                                                     ops['max_pool'], ops['learning_rate']],
                                                                    feed_dict=feed_dict)

                if len(y_pool) == 0:
                    y_pool = np.squeeze(max_pool)
                else:
                    y_pool = np.concatenate((y_pool, np.squeeze(max_pool)), axis=0)

            loss_sum += np.mean(loss_val)

            train_writer.add_summary(summary, step)

            # Update train bar
            process_desc.format(loss_val)
            progress_bar.update(1)
        progress_bar.close()

    print('learning rate: %f' % (lr))
    print('train mean loss: %f' % (loss_sum / float(num_batches)))
    print('train clustering accuracy: %f' % (acc / float(num_batches)))
    return y_pool


def eval_one_epoch(sess, ops, test_writer, is_full_training):
    """ ops: dict mapping from string to tf ops """
    global EPOCH_CNT
    is_training = False
    test_idxs = np.arange(0, len(TEST_FILES))
    # Test on all data: last batch might be smaller than BATCH_SIZE
    loss_sum = acc = 0
    acc_kmeans = 0

    for fn in range(len(TEST_FILES)):
        # print('----' + str(fn) + '-----')
        current_file = os.path.join(H5_DIR, TEST_FILES[test_idxs[fn]])
        current_data, current_label, current_cluster = provider.load_h5_data_label_seg(current_file)
        current_label = np.squeeze(current_label)

        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        
        process_desc = "VALIDATION: Loss {:2.3e}"
        progress_bar = tqdm(initial=0, leave=True, total=num_batches,
                        desc=process_desc.format(0),
                        position=0)
        # num_batches = 5
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx + 1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label, start_idx, end_idx)
            cur_batch_size = end_idx - start_idx

            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['is_training_pl']: is_training,
                         ops['labels_pl']: batch_label,
                         ops['alpha']: 2 * (EPOCH_CNT - MAX_PRETRAIN + 1),}

            if is_full_training:
                summary, step, loss_val, max_pool, dist, mu = sess.run([ops['merged'], ops['step'],
                                                                        ops['kmeans_loss'],
                                                                        ops['max_pool'], ops['stack_dist'],
                                                                        ops['mu']],
                                                                       feed_dict=feed_dict)
                if batch_idx == 0:
                    print("mu: {}".format(mu))
                batch_cluster = np.array([np.where(r == 1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)
                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc += cluster_acc(batch_cluster, cluster_assign)

            else:
                summary, step, loss_val = sess.run([ops['merged'], ops['step'],
                                                    ops['class_loss']],
                                                   feed_dict=feed_dict)

            test_writer.add_summary(summary, step)

            loss_sum += np.mean(loss_val)
            
            # Update train bar
            process_desc.format(loss_val)
            progress_bar.update(1)
        progress_bar.close()

    total_loss = loss_sum * 1.0 / float(num_batches)
    print('test mean loss: %f' % (total_loss))
    print('testing clustering accuracy: %f' % (acc / float(num_batches)))
    
    #Added this to save accuracy
    return acc/float(num_batches)
    
    
    EPOCH_CNT += 1

train()

ls 06_LHC/

LICENSE
README.md
data
h5
models
scripts
utils

import argparse
from argparse import Namespace
from datetime import datetime
import numpy as np
import tensorflow as tf
import socket
import os
import sys
from sklearn.cluster import KMeans
from tqdm import tqdm
import h5py
#Added matplot for accuracy
import matplotlib.pyplot as plt

np.set_printoptions(edgeitems=1000)

from scipy.optimize import linear_sum_assignment

BASE_DIR = os.path.join(os.getcwd(), '06_LHC','scripts')  
#os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR, '..', 'models'))
sys.path.append(os.path.join(BASE_DIR, '..', 'utils'))
import provider
import gapnet_classify as MODEL

parserdict = {'max_dim': 3, #help='Dimension of the encoding layer [Default: 3]')
              'n_clusters': 3, #help='Number of clusters [Default: 3]')
              'gpu': 0, #help='GPU to use [default: GPU 0]')
              'model': 'gapnet_clasify', #help='Model name [default: gapnet_classify]')
              'log_dir': 'log', #help='Log dir [default: log]')
              'num_point': 100, #help='Point Number [default: 100]')
              'max_epoch': 100, #help='Epoch to run [default: 200]')
              'epochs_pretrain': 20, #help='Epochs to for pretraining [default: 10]')
              'batch_size': 1024, #help='Batch Size during training [default: 512]')
              'learning_rate': 0.001, #help='Initial learning rate [default: 0.01]')

              'momentum': 0.9, #help='Initial momentum [default: 0.9]')
              'optimizer': 'adam', #help='adam or momentum [default: adam]')
              'decay_step': 500000, #help='Decay step for lr decay [default: 500000]')
              'wd': 0.0, #help='Weight Decay [Default: 0.0]')
              'decay_rate': 0.5, #help='Decay rate for lr decay [default: 0.5]')
              'output_dir': 'train_results', #help='Directory that stores all training logs and trained models')
              'data_dir': os.path.join(os.getcwd(),'06_LHC', 'h5'), # '../h5', #help='directory with data [default: hdf5_data]')
              'nfeat': 8, #help='Number of features [default: 8]')
              'ncat': 20, #help='Number of categories [default: 20]')
             }

FLAGS = Namespace(**parserdict)
H5_DIR = FLAGS.data_dir

EPOCH_CNT = 0
MAX_PRETRAIN = FLAGS.epochs_pretrain
BATCH_SIZE = FLAGS.batch_size
NUM_POINT = FLAGS.num_point
NUM_FEAT = FLAGS.nfeat
NUM_CLASSES = FLAGS.ncat
MAX_EPOCH = FLAGS.max_epoch
BASE_LEARNING_RATE = FLAGS.learning_rate
GPU_INDEX = FLAGS.gpu
MOMENTUM = FLAGS.momentum
OPTIMIZER = FLAGS.optimizer
DECAY_STEP = FLAGS.decay_step
DECAY_RATE = FLAGS.decay_rate

# MODEL = importlib.import_module(FLAGS.model) # import network module
MODEL_FILE = os.path.join(BASE_DIR, 'models', FLAGS.model + '.py')
LOG_DIR = os.path.join(os.getcwd(), '06_LHC', 'logs', FLAGS.log_dir)

if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR)
os.system('cp %s.py %s' % (MODEL_FILE, LOG_DIR))  # bkp of model def
os.system('cp train_kmeans.py %s' % (LOG_DIR))  # bkp of train procedure

BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

LEARNING_RATE_CLIP = 1e-5
HOSTNAME = socket.gethostname()

TRAIN_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'train_files_wztop.txt'))
TEST_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'test_files_wztop.txt'))

def get_learning_rate(batch):
    learning_rate = tf.compat.v1.train.exponential_decay(
        BASE_LEARNING_RATE,  # Base learning rate.
        batch * BATCH_SIZE,  # Current index into the dataset.
        DECAY_STEP,  # Decay step.
        DECAY_RATE,  # Decay rate.
        staircase=True)
    learning_rate = tf.minimum(learning_rate, LEARNING_RATE_CLIP)  # CLIP THE LEARNING RATE!
    return learning_rate


def get_bn_decay(batch):
    bn_momentum = tf.compat.v1.train.exponential_decay(
        BN_INIT_DECAY,
        batch * BATCH_SIZE,
        BN_DECAY_DECAY_STEP,
        BN_DECAY_DECAY_RATE,
        staircase=True)
    bn_decay = tf.minimum(BN_DECAY_CLIP, 1 - bn_momentum)
    return bn_decay

import os
import time
 
checkpoint_dir = '/dbfs/databricks/driver/06_LHC/logs/train/{}/'.format(time.time())
 
os.makedirs(checkpoint_dir)

def load_h5_hvd(h5_filename, rank=0, size=1):
    f = h5py.File(h5_filename, 'r')
    data = f['data'][rank::size]
    label = f['pid'][rank::size]
    seg = f['label'][rank::size]
    print("loaded {0} events".format(len(data)))

    return (data, label, seg)

def train_hvd():
    import horovod.tensorflow as hvd
    import tensorflow as tf
    import shutil

    
    # do all the imports here again in order for hvd to work nicely
    import horovod.tensorflow as hvd
    import argparse, shlex
    from datetime import datetime
    import numpy as np
    import tensorflow as tf
    import socket
    import os
    import sys
    from sklearn.cluster import KMeans
    from tqdm import tqdm
    np.set_printoptions(edgeitems=1000)
    from scipy.optimize import linear_sum_assignment
    BASE_DIR = os.path.join(os.getcwd(), '06_LHC','scripts')
    sys.path.append(BASE_DIR)
    sys.path.append(os.path.join(BASE_DIR, '..', 'models'))
    sys.path.append(os.path.join(BASE_DIR, '..', 'utils'))
    
    # HOROVOD: initialize Horovod.
    hvd.init()
    
    # HOROVOD: Copy files from local to each single GPU directory
    src = "/dbfs/FileStore/06_LHC"
    dst = os.path.join(os.getcwd(), '06_LHC')
    print("Copying data/files to local horovod folder...")
    shutil.copytree(src, dst)
    print("Done with copying!")
    
    import provider
    import gapnet_classify as MODEL
    
    
    with tf.Graph().as_default():
        with tf.device('/gpu:' + str(GPU_INDEX)):
            #ADDED THIS TO RECORD ACCURACY
            epochs_acc = []          
            pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT, NUM_FEAT)

            is_training_pl = tf.compat.v1.placeholder(tf.bool, shape=())

            # Note the global_step=batch parameter to minimize.
            # That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
            batch = tf.Variable(0)
            alpha = tf.compat.v1.placeholder(dtype=tf.float32, shape=())
            bn_decay = get_bn_decay(batch)
            tf.compat.v1.summary.scalar('bn_decay', bn_decay)
            print("--- Get model and loss")

            pred, max_pool = MODEL.get_model(pointclouds_pl, is_training=is_training_pl,
                                             bn_decay=bn_decay,
                                             num_class=NUM_CLASSES, weight_decay=FLAGS.wd,
                                             )

            class_loss = MODEL.get_focal_loss(pred, labels_pl, NUM_CLASSES)
            mu = tf.Variable(tf.zeros(shape=(FLAGS.n_clusters, FLAGS.max_dim)), name="mu",
                             trainable=True)  # k centroids
            kmeans_loss, stack_dist = MODEL.get_loss_kmeans(max_pool, mu, FLAGS.max_dim,
                                                            FLAGS.n_clusters, alpha)

            full_loss = kmeans_loss + class_loss

            print("--- Get training operator")
            # Get training operator
            learning_rate = get_learning_rate(batch)
            # HOROVOD: scale learning rate from hvd dependent number of processes (=hvd.size)
            tf.compat.v1.summary.scalar('learning_rate', learning_rate * hvd.size())
            if OPTIMIZER == 'momentum':
                optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate * hvd.size(), momentum=MOMENTUM)
            elif OPTIMIZER == 'adam':
                optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate * hvd.size())
            # HOROVOD: add Horovod Distributed Optimizer
            optimizer = hvd.DistributedOptimizer(optimizer)

            global_step = tf.compat.v1.train.get_or_create_global_step() 
            train_op_full = optimizer.minimize(full_loss, global_step=global_step) #batch)
            train_op = optimizer.minimize(class_loss, global_step=global_step) #batch)

            # Add ops to save and restore all the variables.
            saver = tf.compat.v1.train.Saver()
             
        # HOROVOD
        hooks = [
          # Horovod: BroadcastGlobalVariablesHook broadcasts initial variable states
          # from rank 0 to all other processes. This is necessary to ensure consistent
          # initialization of all workers when training is started with random weights
          # or restored from a checkpoint.
          hvd.BroadcastGlobalVariablesHook(0),
          
          #checkpoint_dir_mod = checkpoint_dir if hvd.rank() == 0 else None
          
          tf.compat.v1.train.CheckpointSaverHook(checkpoint_dir=checkpoint_dir,
                                                 checkpoint_basename='cluster.ckpt',
                                                 save_steps=1_000
                                                ),

          # this one basically prints every n steps the "step" and the "loss". Output is cleaner without
          # tf.compat.v1.train.LoggingTensorHook(tensors={'step': global_step, 'loss': full_loss}, every_n_iter=75),
        ]

        # Create a session
        config = tf.compat.v1.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        config.log_device_placement = False
        config.gpu_options.visible_device_list = str(hvd.local_rank())
        
        # global variable initializer must be defined before session definition
        init_global_step = tf.compat.v1.global_variables_initializer()
        
        # MonitoredTrainingSession
        # takes care of session initialization,
        # restoring from a checkpoint, saving to a checkpoint, and closing when done
        # or an error occurs.
        #checkpoint_dir_mod = checkpoint_dir if hvd.rank() == 0 else None
        sess = tf.compat.v1.train.MonitoredTrainingSession(checkpoint_dir=checkpoint_dir,
                                                           hooks=hooks,
                                                           config=config)

        # get one batch_data from the training files in oder to inintialise the session
        train_idxs = np.arange(0, len(TRAIN_FILES))
        current_file = os.path.join(os.getcwd(), '06_LHC', 'h5', TRAIN_FILES[train_idxs[0]])
        current_data, current_label, current_cluster = load_h5_hvd(current_file, hvd.rank(), hvd.size())
        batch_data, batch_label = get_batch(current_data, current_label, 0, BATCH_SIZE)
        # 
        feed_dict = {pointclouds_pl: batch_data,
                     labels_pl: batch_label,
                     is_training_pl: False,
                     alpha: 2 * (EPOCH_CNT - MAX_PRETRAIN + 1),}
        #NOT SO CLEAR THAT init_global_step IS NECESSARY. 
        sess.run(init_global_step, feed_dict=feed_dict)

        # hels with merging: CHANGE THIS IF POSSIBLE
        sess.graph._unsafe_unfinalize()
        # Add summary writers
        merged = tf.compat.v1.summary.merge_all()
        train_writer = tf.compat.v1.summary.FileWriter(os.path.join(LOG_DIR, 'train'), sess.graph)
        test_writer = tf.compat.v1.summary.FileWriter(os.path.join(LOG_DIR, 'test'), sess.graph)
        
        # Init variables
        print("Total number of weights for the model: ", np.sum([np.prod(v.get_shape().as_list()) for v in tf.compat.v1.trainable_variables()]))
        ops = {'pointclouds_pl': pointclouds_pl,
          'labels_pl': labels_pl,
          'is_training_pl': is_training_pl,
          'max_pool': max_pool,
          'pred': pred,
          'alpha': alpha,
          'mu': mu,
          'stack_dist': stack_dist,
          'class_loss': class_loss,
          'kmeans_loss': kmeans_loss,
          'train_op': train_op,
          'train_op_full': train_op_full,
          'merged': merged,
          'step': batch,
          'learning_rate': learning_rate
        }

        for epoch in range(MAX_EPOCH):
            print('\n**** EPOCH %03d ****' % (epoch))
            sys.stdout.flush()
            
            is_full_training = epoch > MAX_PRETRAIN
            max_pool = train_one_epoch(sess, ops, train_writer, hvd.rank(), hvd.size(), is_full_training)
            if epoch == MAX_PRETRAIN:
                centers = KMeans(n_clusters=FLAGS.n_clusters).fit(np.squeeze(max_pool))
                centers = centers.cluster_centers_
                sess.run(tf.compat.v1.assign(mu, centers))
            
            #eval_one_epoch(sess, ops, test_writer, hvd.rank(), hvd.size(), is_full_training)
            #Added these lines to record accuracy
            epoch_acc = eval_one_epoch(sess, ops, test_writer, hvd.rank(), hvd.size(), is_full_training)
            epochs_acc.append(epoch_acc)
            """if is_full_training:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'cluster.ckpt'))
            else:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))"""
            #print("Model saved in file: %s" % save_path)
    return epochs_acc

def get_batch(data, label, start_idx, end_idx):
    batch_label = label[start_idx:end_idx]
    batch_data = data[start_idx:end_idx, :, :]
    return batch_data, batch_label


def cluster_acc(y_true, y_pred):
    """
    Calculate clustering accuracy. Require scikit-learn installed
    """
    y_true = y_true.astype(np.int64)
    D = max(y_pred.max(), y_true.max()) + 1
    w = np.zeros((D, D), dtype=np.int64)
    for i in range(y_pred.size):
        w[y_pred[i], y_true[i]] += 1
    ind = linear_sum_assignment(w.max() - w)
    ind = np.asarray(ind)
    ind = np.transpose(ind)
    return sum([w[i, j] for i, j in ind]) * 1.0 / y_pred.size

def train_one_epoch(sess, ops, train_writer, hvd_rank, hvd_size, is_full_training):
    """ ops: dict mapping from string to tf ops """
    is_training = True

    train_idxs = np.arange(0, len(TRAIN_FILES))

    acc = loss_sum = 0
    y_pool = []
    for fn in range(len(TRAIN_FILES)):
        # print('----' + str(fn) + '-----')
        current_file = os.path.join(os.getcwd(), '06_LHC', 'h5', TRAIN_FILES[train_idxs[fn]])
        current_data, current_label, current_cluster = load_h5_hvd(current_file, hvd_rank, hvd_size)

        current_label = np.squeeze(current_label)

        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        # num_batches = 5
        print(str(datetime.now()))

        # initialise progress bar
        #process_desc = "TRAINING: Loss {:2.3e}"
        #progress_bar = tqdm(initial=0, leave=True, total=num_batches,
        #                    desc=process_desc.format(0),
        #                    position=0)
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx + 1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label, start_idx, end_idx)
            cur_batch_size = end_idx - start_idx

            # print(batch_weight)
            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['labels_pl']: batch_label,
                         ops['is_training_pl']: is_training,
                         ops['alpha']: 2 * (EPOCH_CNT - MAX_PRETRAIN + 1),}
            if is_full_training:
                summary, step, _, loss_val, dist, lr = sess.run([ops['merged'], ops['step'],
                                                                 ops['train_op_full'], ops['kmeans_loss'],
                                                                 ops['stack_dist'], ops['learning_rate']],
                                                                feed_dict=feed_dict)

                batch_cluster = np.array([np.where(r == 1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)

                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc += cluster_acc(batch_cluster, cluster_assign)
            else:
                summary, step, _, loss_val, max_pool, lr = sess.run([ops['merged'], ops['step'],
                                                                     ops['train_op'], ops['class_loss'],
                                                                     ops['max_pool'], ops['learning_rate']],
                                                                    feed_dict=feed_dict)

                if len(y_pool) == 0:
                    y_pool = np.squeeze(max_pool)
                else:
                    y_pool = np.concatenate((y_pool, np.squeeze(max_pool)), axis=0)

            loss_sum += np.mean(loss_val)

            #train_writer.add_summary(summary, step)
            if hvd_rank == 0:
                train_writer.add_summary(summary, step)
              
              

            # Update train bar
            #process_desc.format(loss_val)
            #progress_bar.update(1)
        #progress_bar.close()

    print('learning rate: %f' % (lr))
    print('train mean loss: %f' % (loss_sum / float(num_batches)))
    #if is_full_training:
    print('train clustering accuracy: %f' % (acc / float(num_batches)))
    return y_pool


def eval_one_epoch(sess, ops, test_writer, hvd_rank, hvd_size, is_full_training):
    """ ops: dict mapping from string to tf ops """
    global EPOCH_CNT
    is_training = False
    test_idxs = np.arange(0, len(TEST_FILES))
    # Test on all data: last batch might be smaller than BATCH_SIZE
    loss_sum = acc = 0
    acc_kmeans = 0

    for fn in range(len(TEST_FILES)):
        # print('----' + str(fn) + '-----')
        current_file = os.path.join(os.getcwd(), '06_LHC', 'h5', TEST_FILES[test_idxs[fn]])
        current_data, current_label, current_cluster = load_h5_hvd(current_file, hvd_rank, hvd_size)
        current_label = np.squeeze(current_label)

        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        
        """process_desc = "VALIDATION: Loss {:2.3e}"
        progress_bar = tqdm(initial=0, leave=True, total=num_batches,
                        desc=process_desc.format(0),
                        position=0)"""
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx + 1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label, start_idx, end_idx)
            cur_batch_size = end_idx - start_idx

            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['is_training_pl']: is_training,
                         ops['labels_pl']: batch_label,
                         ops['alpha']: 2 * (EPOCH_CNT - MAX_PRETRAIN + 1),}

            if is_full_training:
                summary, step, loss_val, max_pool, dist, mu = sess.run([ops['merged'], ops['step'],
                                                                        ops['kmeans_loss'],
                                                                        ops['max_pool'], ops['stack_dist'],
                                                                        ops['mu']],
                                                                       feed_dict=feed_dict)
                
                batch_cluster = np.array([np.where(r == 1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)
                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc += cluster_acc(batch_cluster, cluster_assign)

            else:
                summary, step, loss_val = sess.run([ops['merged'], ops['step'],
                                                    ops['class_loss']],
                                                   feed_dict=feed_dict)

            #test_writer.add_summary(summary, step)
            if hvd_rank == 0:
                test_writer.add_summary(summary, step)

            loss_sum += np.mean(loss_val)
            
            """# Update train bar
            process_desc.format(loss_val)
            progress_bar.update(1)"""
        #progress_bar.close()

    total_loss = loss_sum * 1.0 / float(num_batches)
    print('test mean loss: %f' % (total_loss))
    #if is_full_training:
    print('testing clustering accuracy: %f' % (acc / float(num_batches)))

    return acc/float(num_batches)
    
    EPOCH_CNT += 1
    

from sparkdl import HorovodRunner
 
hr = HorovodRunner(np=2)
epochs_acc=hr.run(train_hvd)
plt.plot(epochs_acc)
plt.ylabel('Validation accuracy')
plt.xlabel('epochs')
plt.show()
plt.savefig('distributed.png')

print(epochs_acc)

checkpoint_dir

import argparse
from argparse import Namespace
from math import *
import numpy as np
from datetime import datetime
import json
import os, ast
import sys
import socket
from sklearn.cluster import KMeans
np.set_printoptions(edgeitems=1000)
from scipy.optimize import linear_sum_assignment
import h5py
import tensorflow as tf
from tqdm import tqdm


from scipy.optimize import linear_sum_assignment

BASE_DIR = os.path.join(os.getcwd(), '06_LHC','scripts')  
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR, '..', 'models'))
sys.path.append(os.path.join(BASE_DIR, '..', 'utils'))
import provider
import gapnet_classify as MODEL

parserdict = {'gpu':0, #help='GPUs to use [default: 0]')
              'n_clusters':3,# type=int, default=3, #help='Number of clusters [Default: 3]')
              'max_dim':3, #type=int, default=3, #help='Dimension of the encoding layer [Default: 3]')
              'log_dir': 'log',#default='log', #help='Log dir [default: log]')
              'batch':1024,# type=int, default=512, #help='Batch Size  during training [default: 512]')
              'num_point':100, # type=int, default=100, #help='Point Number [default: 100]')
              'data_dir':'../h5/', #default='../h5', #help='directory with data [default: ../h5]')
              'nfeat':8,# type=int, default=8, #help='Number of features [default: 8]')
              'ncat':20, # type=int, default=20, #help='Number of categories [default: 20]')
              'name': "evaluation", #default="", #help='name of the output file')
              'h5_folder':'../h5/', #default="../h5/", #help='folder to store output files')
              'full_train':True,# default=False, action='store_true',
                    #help='load full training results [default: False]')
              'checkpoint_folder':'/dbfs/databricks/driver/06_LHC/logs/train/', #help: The folder where the checkpoint is saved. The script
                   #will retrieved the latest checkpoint created here.
             }

FLAGS = Namespace(**parserdict)
#LOG_DIR = os.path.join('..', 'logs', FLAGS.log_dir)
LOG_DIR = os.path.join(os.getcwd(), '06_LHC', 'logs', FLAGS.log_dir)
DATA_DIR = FLAGS.data_dir
H5_DIR = os.path.join(BASE_DIR, DATA_DIR)
H5_OUT = FLAGS.h5_folder
CHECKPOINT_PATH = FLAGS.checkpoint_folder
if not os.path.exists(H5_OUT): os.mkdir(H5_OUT)

#Calculate the clustering accuracy
def cluster_acc(y_true, y_pred):
    """
    Calculate clustering accuracy. Require scikit-learn installed
    """
    y_true = y_true.astype(np.int64)
    D = max(y_pred.max(), y_true.max()) + 1
    w = np.zeros((D, D), dtype=np.int64)
    for i in range(y_pred.size):
        w[y_pred[i], y_true[i]] += 1
    ind = linear_sum_assignment(w.max() - w)
    ind = np.asarray(ind)
    ind = np.transpose(ind)
    return sum([w[i, j] for i, j in ind]) * 1.0 / y_pred.size

# Find the latest checkpoint (the training script saves one every 1000th step)
def find_ckpt(path,base):
  files = os.listdir(os.path.join(path,os.listdir(path)[-1]))
  s=base+".ckpt-"
  ckpts = [r for r in files if s in r]
  numbers = [int(r.split('.')[1].split('-')[1]) for r in ckpts]
  ckpt = base+'.ckpt-'+str(np.max(numbers))
  return os.path.join(path,os.listdir(path)[-1],ckpt)

ls /dbfs/databricks/driver/06_LHC/logs/train/

1608312111.5291479
1608312856.6910331
1608316348.7265332
1608316368.303585
1608316645.5256958
1608317160.1003277
1608317514.0408154
1608317816.3961644
1608545228.287947
1609765453.8899894
1614197922.0756063
1614199473.6461
1614199773.0030868
1614199944.5590024
1614205764.5543048
1614253383.643093
1614266637.6848218
1614283969.6952937
1614352318.071854
1614372841.6587343
1614427683.679712
1614435113.1720457
1614435261.5545285
1614447752.0912156
1614447848.5286212
1614516429.048815
1614516851.6496608
1618854193.501687

NUM_POINT = FLAGS.num_point
BATCH_SIZE = FLAGS.batch
NFEATURES = FLAGS.nfeat
FULL_TRAINING = FLAGS.full_train

NUM_CATEGORIES = FLAGS.ncat
# Only used to get how many parts per categor
print('#### Batch Size : {0}'.format(BATCH_SIZE))
print('#### Point Number: {0}'.format(NUM_POINT))
print('#### Using GPUs: {0}'.format(FLAGS.gpu))

print('### Starting evaluation')

EVALUATE_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'evaluate_files_wztop.txt'))


def eval():
    with tf.Graph().as_default():
        with tf.device('/gpu:' + str(FLAGS.gpu)):
            pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT, NFEATURES)
            batch = tf.Variable(0, trainable=False)
            alpha = tf.compat.v1.placeholder(tf.float32, shape=())
            is_training_pl = tf.compat.v1.placeholder(tf.bool, shape=())
            pred, max_pool = MODEL.get_model(pointclouds_pl, is_training=is_training_pl, num_class=NUM_CATEGORIES)
            mu = tf.Variable(tf.zeros(shape=(FLAGS.n_clusters, FLAGS.max_dim)), name="mu",
                             trainable=False)  # k centroids

            classify_loss = MODEL.get_focal_loss(pred, labels_pl, NUM_CATEGORIES)
            kmeans_loss, stack_dist = MODEL.get_loss_kmeans(max_pool, mu, FLAGS.max_dim,
                                                            FLAGS.n_clusters, alpha)

            saver = tf.compat.v1.train.Saver()

        config = tf.compat.v1.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        sess = tf.compat.v1.Session(config=config)

        if FULL_TRAINING:
            #saver.restore(sess, os.path.join(LOG_DIR, 'cluster.ckpt'))
            saver.restore(sess, find_ckpt(CHECKPOINT_PATH,'cluster'))
        else:
            saver.restore(sess, find_ckpt(CHECKPOINT_PATH,'model'))
            #saver.restore(sess, os.path.join(LOG_DIR, 'model.ckpt'))
        print('model restored')

        ops = {'pointclouds_pl': pointclouds_pl,
               'labels_pl': labels_pl,
               'stack_dist': stack_dist,
               'kmeans_loss': kmeans_loss,
               'pred': pred,
               'alpha': alpha,
               'max_pool': max_pool,
               'is_training_pl': is_training_pl,
               'classify_loss': classify_loss, }

        eval_one_epoch(sess, ops)


def get_batch(data, label, start_idx, end_idx):
    batch_label = label[start_idx:end_idx]
    batch_data = data[start_idx:end_idx, :, :]
    return batch_data, batch_label


def eval_one_epoch(sess, ops):
    is_training = False

    eval_idxs = np.arange(0, len(EVALUATE_FILES))
    y_val = []
    acc = 0
    
    for fn in range(len(EVALUATE_FILES)):
        current_file = os.path.join(H5_DIR, EVALUATE_FILES[eval_idxs[fn]])
        current_data, current_label, current_cluster = provider.load_h5_data_label_seg(current_file)
        adds = provider.load_add(current_file, ['masses'])

        current_label = np.squeeze(current_label)

        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
       # num_batches = 5

        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx + 1) * BATCH_SIZE

            batch_data, batch_label = get_batch(current_data, current_label, start_idx, end_idx)
            batch_cluster = current_cluster[start_idx:end_idx]
            cur_batch_size = end_idx - start_idx

            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['labels_pl']: batch_label,
                         ops['alpha']: 1,  # No impact on evaluation,
                         ops['is_training_pl']: is_training,
                         }

            loss, dist, max_pool = sess.run([ops['kmeans_loss'], ops['stack_dist'],
                                             ops['max_pool']], feed_dict=feed_dict)
            cluster_assign = np.zeros((cur_batch_size), dtype=int)
            for i in range(cur_batch_size):
                index_closest_cluster = np.argmin(dist[:, i])
                cluster_assign[i] = index_closest_cluster

            batch_cluster = np.array([np.where(r == 1)[0][0] for r in current_cluster[start_idx:end_idx]])

            if len(y_val) == 0:
                y_val = batch_cluster
                y_assign = cluster_assign
                y_pool = np.squeeze(max_pool)
                y_mass = adds['masses'][start_idx:end_idx]
            else:
                y_val = np.concatenate((y_val, batch_cluster), axis=0)
                y_assign = np.concatenate((y_assign, cluster_assign), axis=0)
                y_pool = np.concatenate((y_pool, np.squeeze(max_pool)), axis=0)
                y_mass = np.concatenate((y_mass, adds['masses'][start_idx:end_idx]), axis=0)

    with h5py.File(os.path.join(H5_OUT, '{0}.h5'.format(FLAGS.name)), "w") as fh5:
        dset = fh5.create_dataset("pid", data=y_val)  # Real jet categories
        dset = fh5.create_dataset("label", data=y_assign)  # Cluster labeling
        dset = fh5.create_dataset("max_pool", data=y_pool)
        dset = fh5.create_dataset("masses", data=y_mass)
    
    print("Clustering accuracy is ",cluster_acc(y_val,y_assign))

################################################


if __name__ == '__main__':
    eval()

#### Batch Size : 1024
#### Point Number: 100
#### Using GPUs: 0
### Starting evaluation
INFO:tensorflow:Restoring parameters from /dbfs/databricks/driver/06_LHC/logs/train/1614516851.6496608/cluster.ckpt-13000
model restored
loaded 143984 events
loaded 302664 events
loaded 75666 events
Clustering accuracy is  0.5005421075295275

os.listdir(H5_OUT)
from tsne import bh_sne

import os

ls

06_LHC
conf
derby.log
eventlogs
ganglia
logs

#%sh
#wget https://github.com/olgeet/UCluster/archive/refs/heads/master.zip
#unzip master.zip

rm -r 06_LHC_TF1
mkdir 06_LHC_TF1
cd 06_LHC_TF1
mv ../UCluster-master/* .
rm -r ../UCluster-master/ ../master.zip
cp ../06_LHC/h5/* ./h5/

rm: cannot remove '06_LHC_TF1': No such file or directory

ls 06_LHC_TF1/h5

eval_multi_20v_100P.h5
evaluate_files_RD.txt
evaluate_files_b1.txt
evaluate_files_b2.txt
evaluate_files_b3.txt
evaluate_files_gwztop.txt
evaluate_files_wztop.txt
test_files_RD.txt
test_files_b1.txt
test_files_b2.txt
test_files_b3.txt
test_files_gwztop.txt
test_files_wztop.txt
test_multi_20v_100P.h5
train_files_RD.txt
train_files_b1.txt
train_files_b2.txt
train_files_b3.txt
train_files_gwztop.txt
train_files_wztop.txt
train_multi_20v_100P.h5

from argparse import Namespace
import math
import subprocess
from datetime import datetime
import numpy as np
import tensorflow as tf
import socket
import importlib
import os,ast
import sys
from sklearn.cluster import KMeans
import h5py
np.set_printoptions(edgeitems=1000)

from scipy.optimize import linear_sum_assignment
BASE_DIR = os.path.join(os.getcwd(), '06_LHC_TF1','scripts') 
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR,'..', 'models'))
sys.path.append(os.path.join(BASE_DIR,'..' ,'utils'))
import provider
import gapnet_classify as MODEL

print(tf.__version__)

1.15.4

parserdict = {'max_dim': 3, #help='Dimension of the encoding layer [Default: 3]')
              'n_clusters': 3, #help='Number of clusters [Default: 3]')
              'gpu': 0, #help='GPU to use [default: GPU 0]')
              'model': 'gapnet_clasify', #help='Model name [default: gapnet_classify]')
              'log_dir': 'log', #help='Log dir [default: log]')
              'num_point': 100, #help='Point Number [default: 100]')
              'max_epoch': 100, #help='Epoch to run [default: 200]')
              'epochs_pretrain': 20, #help='Epochs to for pretraining [default: 10]')
              'batch_size': 1024, #help='Batch Size during training [default: 512]')
              'learning_rate': 0.001, #help='Initial learning rate [default: 0.01]')

              'momentum': 0.9, #help='Initial momentum [default: 0.9]')
              'optimizer': 'adam', #help='adam or momentum [default: adam]')
              'decay_step': 500000, #help='Decay step for lr decay [default: 500000]')
              'wd': 0.0, #help='Weight Decay [Default: 0.0]')
              'decay_rate': 0.5, #help='Decay rate for lr decay [default: 0.5]')
              'output_dir': 'train_results', #help='Directory that stores all training logs and trained models')
              'data_dir': os.path.join(os.getcwd(),'06_LHC_TF1', 'h5'), # '../h5', #help='directory with data [default: hdf5_data]')
              'nfeat': 8, #help='Number of features [default: 8]')
              'ncat': 20, #help='Number of categories [default: 20]')
             }

FLAGS = Namespace(**parserdict)
H5_DIR = FLAGS.data_dir

EPOCH_CNT = 0
MAX_PRETRAIN = FLAGS.epochs_pretrain
BATCH_SIZE = FLAGS.batch_size
NUM_POINT = FLAGS.num_point
NUM_FEAT = FLAGS.nfeat
NUM_CLASSES = FLAGS.ncat
MAX_EPOCH = FLAGS.max_epoch
BASE_LEARNING_RATE = FLAGS.learning_rate
GPU_INDEX = FLAGS.gpu
MOMENTUM = FLAGS.momentum
OPTIMIZER = FLAGS.optimizer
DECAY_STEP = FLAGS.decay_step
DECAY_RATE = FLAGS.decay_rate

# MODEL = importlib.import_module(FLAGS.model) # import network module
MODEL_FILE = os.path.join(BASE_DIR, 'models', FLAGS.model + '.py')
LOG_DIR = os.path.join('..', 'logs', FLAGS.log_dir)

if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR)
os.system('cp %s.py %s' % (MODEL_FILE, LOG_DIR))  # bkp of model def
os.system('cp train_kmeans.py %s' % (LOG_DIR))  # bkp of train procedure

BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

LEARNING_RATE_CLIP = 1e-5
HOSTNAME = socket.gethostname()

TRAIN_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'train_files_wztop.txt'))
TEST_FILES = provider.getDataFiles(os.path.join(H5_DIR, 'test_files_wztop.txt'))

def log_string(out_str):
    #LOG_FOUT.write(out_str+'\n')
    #LOG_FOUT.flush()
    print(out_str)

def get_learning_rate(batch):
    learning_rate = tf.train.exponential_decay(
                        BASE_LEARNING_RATE,  # Base learning rate.
                        batch * BATCH_SIZE,  # Current index into the dataset.
                        DECAY_STEP,          # Decay step.
                        DECAY_RATE,          # Decay rate.
                        staircase=True)
    learning_rate = tf.maximum(learning_rate, LEARNING_RATE_CLIP) # CLIP THE LEARNING RATE!
    return learning_rate        


def get_bn_decay(batch):
    bn_momentum = tf.train.exponential_decay(
                      BN_INIT_DECAY,
                      batch*BATCH_SIZE,
                      BN_DECAY_DECAY_STEP,
                      BN_DECAY_DECAY_RATE,
                      staircase=True)
    bn_decay = tf.minimum(BN_DECAY_CLIP, 1 - bn_momentum)
    return bn_decay

def train():
    with tf.Graph().as_default():
        with tf.device('/gpu:'+str(GPU_INDEX)):
            pointclouds_pl,  labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT,NUM_FEAT) 

            is_training_pl = tf.placeholder(tf.bool, shape=())

            # Note the global_step=batch parameter to minimize. 
            # That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
            batch = tf.Variable(0)
            alpha = tf.placeholder(dtype=tf.float32, shape=())
            bn_decay = get_bn_decay(batch)
            tf.summary.scalar('bn_decay', bn_decay)
            print("--- Get model and loss")

            pred , max_pool = MODEL.get_model(pointclouds_pl, is_training=is_training_pl,
                                              bn_decay=bn_decay,
                                              num_class=NUM_CLASSES, weight_decay=FLAGS.wd,
            )
                                                                                        

            class_loss = MODEL.get_focal_loss(pred, labels_pl,NUM_CLASSES)
            mu = tf.Variable(tf.zeros(shape=(FLAGS.n_clusters,FLAGS.max_dim)),name="mu",trainable=True) #k centroids
            kmeans_loss, stack_dist= MODEL.get_loss_kmeans(max_pool,mu,  FLAGS.max_dim,
                                                            FLAGS.n_clusters,alpha)
            
            full_loss = 10*kmeans_loss + class_loss


            
            print("--- Get training operator")
            # Get training operator
            learning_rate = get_learning_rate(batch)
            tf.summary.scalar('learning_rate', learning_rate)
            if OPTIMIZER == 'momentum':
                optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=MOMENTUM)
            elif OPTIMIZER == 'adam':
                optimizer = tf.train.AdamOptimizer(learning_rate)

            
            train_op_full = optimizer.minimize(full_loss, global_step=batch)
            train_op = optimizer.minimize(class_loss, global_step=batch)
            
            # Add ops to save and restore all the variables.
            saver = tf.train.Saver()
        
        # Create a session
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        config.allow_soft_placement = True
        config.log_device_placement = False
        sess = tf.Session(config=config)

        sess.run(tf.global_variables_initializer())

        
        
        # Add summary writers
        merged = tf.summary.merge_all()
        train_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'train'), sess.graph)
        test_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'test'), sess.graph)

        # Init variables
        print("Total number of weights for the model: ",np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()]))
        ops = {'pointclouds_pl': pointclouds_pl,
               'labels_pl':labels_pl,
               'is_training_pl': is_training_pl,
               'max_pool':max_pool,
               'pred': pred,
               'alpha': alpha,
               'mu': mu,
               'stack_dist':stack_dist,
               'class_loss': class_loss,
               'kmeans_loss': kmeans_loss,
               'train_op': train_op,
               'train_op_full': train_op_full,
               'merged': merged,
               'step': batch,
               'learning_rate':learning_rate
        }




        for epoch in range(MAX_EPOCH):
            log_string('**** EPOCH %03d ****' % (epoch))
            sys.stdout.flush()
            
            is_full_training = epoch > MAX_PRETRAIN
            max_pool = train_one_epoch(sess, ops, train_writer,is_full_training)
            if epoch == MAX_PRETRAIN:
                centers  = KMeans(n_clusters=FLAGS.n_clusters).fit(np.squeeze(max_pool))
                centers = centers.cluster_centers_
                sess.run(tf.assign(mu,centers))
                
            
            eval_one_epoch(sess, ops, test_writer,is_full_training)
            if is_full_training:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'cluster.ckpt'))
            else:
                save_path = saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))


            log_string("Model saved in file: %s" % save_path)

def get_batch(data,label,  start_idx, end_idx):
    batch_label = label[start_idx:end_idx]
    batch_data = data[start_idx:end_idx,:,:]
    return batch_data, batch_label

def cluster_acc(y_true, y_pred):
    """
    Calculate clustering accuracy. Require scikit-learn installed
    """
    y_true = y_true.astype(np.int64)
    D = max(y_pred.max(), y_true.max()) + 1
    w = np.zeros((D, D), dtype=np.int64)
    for i in range(y_pred.size):
        w[y_pred[i], y_true[i]] += 1
    ind = linear_sum_assignment(w.max() - w)
    ind = np.asarray(ind)
    ind = np.transpose(ind)
    return sum([w[i, j] for i, j in ind]) * 1.0 / y_pred.size
        

  def train_one_epoch(sess, ops, train_writer,is_full_training):
    """ ops: dict mapping from string to tf ops """
    is_training = True
    
    
    train_idxs = np.arange(0, len(TRAIN_FILES))
    
    acc =  loss_sum = 0
    y_pool = []
    for fn in range(len(TRAIN_FILES)):
        #log_string('----' + str(fn) + '-----')
        current_file = os.path.join(H5_DIR,TRAIN_FILES[train_idxs[fn]])
        current_data, current_label, current_cluster = provider.load_h5_data_label_seg(current_file)
        
        current_label = np.squeeze(current_label)
        
        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        #num_batches = 5
        log_string(str(datetime.now()))        

        
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx+1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label,start_idx, end_idx)
            cur_batch_size = end_idx-start_idx
                

            #print(batch_weight) 
            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['labels_pl']: batch_label,
                         ops['is_training_pl']: is_training,
                         ops['alpha']: 2*(EPOCH_CNT-MAX_PRETRAIN+1),
                         
                        
            }
            if is_full_training:
                summary, step, _, loss_val,dist,lr = sess.run([ops['merged'], ops['step'],
                                                            ops['train_op_full'], ops['kmeans_loss'],
                                                            ops['stack_dist'],ops['learning_rate']
                                                           ], 
                                                              feed_dict=feed_dict)

            
                batch_cluster = np.array([np.where(r==1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)


                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc+=cluster_acc(batch_cluster,cluster_assign)
            else:
                summary, step, _, loss_val,max_pool,lr = sess.run([ops['merged'], ops['step'],
                                                                   ops['train_op'], ops['class_loss'],
                                                                   ops['max_pool'],ops['learning_rate']],
                                                               
                                                               feed_dict=feed_dict)
                
                
                
                if len(y_pool)==0:
                    y_pool=np.squeeze(max_pool)                
                else:
                    y_pool=np.concatenate((y_pool,np.squeeze(max_pool)),axis=0)
                
            loss_sum += np.mean(loss_val)

            train_writer.add_summary(summary, step)
    log_string('learning rate: %f' % (lr))
    log_string('train mean loss: %f' % (loss_sum / float(num_batches)))
    log_string('train clustering accuracy: %f' % (acc/ float(num_batches)))
    return y_pool

def eval_one_epoch(sess, ops, test_writer,is_full_training):
    """ ops: dict mapping from string to tf ops """
    global EPOCH_CNT
    is_training = False
    test_idxs = np.arange(0, len(TEST_FILES))
    # Test on all data: last batch might be smaller than BATCH_SIZE
    loss_sum = acc =0
    acc_kmeans = 0


    for fn in range(len(TEST_FILES)):
        #log_string('----' + str(fn) + '-----')
        current_file = os.path.join(H5_DIR,TEST_FILES[test_idxs[fn]])
        current_data, current_label, current_cluster = provider.load_h5_data_label_seg(current_file)
        current_label = np.squeeze(current_label)

            
            
        file_size = current_data.shape[0]
        num_batches = file_size // BATCH_SIZE
        #num_batches = 5
        for batch_idx in range(num_batches):
            start_idx = batch_idx * BATCH_SIZE
            end_idx = (batch_idx+1) * BATCH_SIZE
            batch_data, batch_label = get_batch(current_data, current_label,start_idx, end_idx)
            cur_batch_size = end_idx-start_idx
            
            feed_dict = {ops['pointclouds_pl']: batch_data,
                         ops['is_training_pl']: is_training,
                         ops['labels_pl']: batch_label,
                         ops['alpha']: 2*(EPOCH_CNT-MAX_PRETRAIN+1),

            }

            if is_full_training:
                summary, step, loss_val, max_pool,dist,mu= sess.run([ops['merged'], ops['step'],
                                                                     ops['kmeans_loss'],
                                                                     ops['max_pool'],ops['stack_dist'],
                                                                     ops['mu']
                                                                 ],
                                                                    feed_dict=feed_dict)
                if batch_idx==0:
                    log_string("mu: {}".format(mu))     
                batch_cluster = np.array([np.where(r==1)[0][0] for r in current_cluster[start_idx:end_idx]])
                cluster_assign = np.zeros((cur_batch_size), dtype=int)
                for i in range(cur_batch_size):
                    index_closest_cluster = np.argmin(dist[:, i])
                    cluster_assign[i] = index_closest_cluster

                acc+=cluster_acc(batch_cluster,cluster_assign)
                

            else:
                summary, step, loss_val= sess.run([ops['merged'], ops['step'],
                                                   ops['class_loss']
                                               ],
                                                  feed_dict=feed_dict)


            test_writer.add_summary(summary, step)
            


            loss_sum += np.mean(loss_val)
        
    total_loss = loss_sum*1.0 / float(num_batches)
    log_string('test mean loss: %f' % (total_loss))
    log_string('testing clustering accuracy: %f' % (acc / float(num_batches)))

    EPOCH_CNT += 1

# if __name__ == "__main__":
# #    log_string('pid: %s'%(str(os.getpid())))
#     train()
# #    LOG_FOUT.close()

import org.apache.spark.sql._
import org.apache.spark.sql.functions._

import org.graphframes._

import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.graphframes._

/*
** A function counting the number of multiple edges.
*/
def count_mult_edges(input_graph: GraphFrame) = {
  input_graph.find("(a)-[e1]->(b); (a)-[e2]->(b)").filter("e1 != e2").count
}

count_mult_edges: (input_graph: org.graphframes.GraphFrame)Long

/*
** A function counting the number of 3-cycles.
*/
def count_3_cycle(input_graph: GraphFrame): Long = {
  input_graph.find("(a)-[]->(b); (b)-[]->(c); (c)-[]->(a)").count
}

count_3_cycle: (input_graph: org.graphframes.GraphFrame)Long

/*
** A function counting the number of loop edges.
*/
def count_loop(input_graph: GraphFrame) = {
  input_graph.find("(a)-[]->(a)").count
}

count_loop: (input_graph: org.graphframes.GraphFrame)Long

/*
** A function counting the number of bidirected edges.
*/
def count_bidir_edges(input_graph: GraphFrame) = {
  input_graph.find("(a)-[]->(b); (b)-[]->(a)").filter("a.id != b.id").count
}

count_bidir_edges: (input_graph: org.graphframes.GraphFrame)Long

/*
** Two functions counting the number of 2- and 4-dimensional faces of a
** simplicial graphical simplex. It is assumed the the input_graph does
** not contain any loops.
*/

def count_K2(input_graph: GraphFrame): Long = { // Should be the same as the number of edges.
  input_graph.find("(a)-[]->(b)").count
}

def count_K3(input_graph: GraphFrame): Long = {
  input_graph.find("(a)-[]->(b); (a)-[]->(c); (b)-[]->(c)").count
}

def count_K4(input_graph: GraphFrame): Long = {
  input_graph.find("(a)-[]->(b); (a)-[]->(c); (a)-[]->(d); (b)-[]->(c); (b)-[]->(d); (c)-[]->(d)").count
}

count_K2: (input_graph: org.graphframes.GraphFrame)Long
count_K3: (input_graph: org.graphframes.GraphFrame)Long
count_K4: (input_graph: org.graphframes.GraphFrame)Long

/*
** The file edges.csv contains lines on the form "1,5,edge" indicating
** we have an edge from vertex 1 to vertex 5. As we do not care about the
** third entry (i just says "edge") we choose just the two first entries.
*/

val edges = spark.read.format("csv").option("sep",",").load("/FileStore/shared_uploads/petterre@kth.se/edges.csv").withColumnRenamed("_c0", "src").withColumnRenamed("_c1", "dst").select("src", "dst")

edges.count

edges: org.apache.spark.sql.DataFrame = [src: string, dst: string]
res0: Long = 7822274

/*
** For simplicity we read all the nodes from the column "scr". Notice
** that we use the fact that every node is the source of at least one
** edge. Thus we can get the vertices like this.
*/

val vertices = edges.select("src").groupBy("src").count().select("src").sort(desc("src")).withColumnRenamed("src", "id")

vertices.count

vertices: org.apache.spark.sql.DataFrame = [id: string]
res2: Long = 31346

/*
** Now we can create out graph of interest.
*/

val rat_brain_graph = GraphFrame(vertices, edges)

rat_brain_graph: org.graphframes.GraphFrame = GraphFrame(v:[id: string], e:[src: string, dst: string])

/*
** As mentioned before, we do not want loops in this graph. Thus we
** call our above function and check whether we have any.
*/
count_loop(rat_brain_graph) // 0

res4: Long = 0

/*
** A very relevant question is if this graph is connected. If it is not
** it would be more efficient to just look at the induvidual components.
** That is not the case.
*/

spark.sparkContext.setCheckpointDir("/FileStore/shared_uploads/petterre@kth.se/")

val rat_brain_graph_connected_components = rat_brain_graph.connectedComponents.run()
rat_brain_graph_connected_components.select("component").describe().show()

+-------+---------+
|summary|component|
+-------+---------+
|  count|    31346|
|   mean|      0.0|
| stddev|      0.0|
|    min|        0|
|    max|        0|
+-------+---------+

rat_brain_graph_connected_components: org.apache.spark.sql.DataFrame = [id: string, component: bigint]

/*
** Let us continue to look at some stastistics for this graph. We can
** count the number of 3-cycles.
*/

count_3_cycle(rat_brain_graph) //25 630 728

res7: Long = 25630728

/*
** Let us continue to look at some stastistics for this graph. We can
** count the number of multiple edges.
*/

count_mult_edges(rat_brain_graph) // 0

res8: Long = 0

/*
** Let us continue to look at some stastistics for this graph. We can
** count the number of bidirected edges.
*/

count_bidir_edges(rat_brain_graph) //165 220

res9: Long = 165220

/*
** First we look after K2. As that is the graph a->b, we expect this to
** be equal to the number of edges, otherwise something is wrong.
*/
count_K2(rat_brain_graph) //7 822 274

res11: Long = 7822274

/*
** Now we find K3. As we will see, this takes a lot of time, longer than
** we want it to. This is because 'find' does a general search algorithm.
** See below for a discussion.
*/

count_K3(rat_brain_graph) //35 976 731

res12: Long = 35976731

//count_K4(rat_brain_graph)

This is a direct implementation.

/*
** A implementation of counting cells. It is not parallelized but could be
** if we start collecting the results in a better way.
*/

def count_cells(old_child_set: DataFrame, f_vector: List[Int], edges: DataFrame, cut_of: Int, dim: Int): List[Int] = {
  // Make a new f-vector that is mutable
  var f_vector_new = f_vector;
  // For each new node
  for (vert <- old_child_set.collect()){
    // Update the f-vector
    f_vector_new = f_vector_new.updated(dim, f_vector_new(dim) +1);
    // If we have not reached our cut off limit
    if (cut_of > dim){
      // Find the children of "vert" and find the intersection.
      // val child_set = old_child_set.intersect(edges.filter(edges("src") === vert(0)).select("dst"));
      // Call this function recursively
      f_vector_new = count_cells(old_child_set.intersect(edges.filter(edges("src") === vert(0)).select("dst")), f_vector_new, edges, cut_of, dim+1);
    }
  }
  // Return the f-vector
  f_vector_new
}

count_cells: (old_child_set: org.apache.spark.sql.DataFrame, f_vector: List[Int], edges: org.apache.spark.sql.DataFrame, cut_of: Int, dim: Int)List[Int]

This is a parallelized version. Notice that the parallelization is very naive and it starts to many threads.

/*
** Helper function to count_cells_par. Does a component-wise addition.
** Badly written.
*/

def component_addition(a: List[Int], b:List[Int]): List[Int] = {
  // Do it the dumb way
  List(a(0)+b(0),a(1)+b(1),a(2)+b(2),a(3)+b(3),a(4)+b(4),a(5)+b(5),a(6)+b(6),a(7)+b(7),a(8)+b(8),a(9)+b(9))
}

component_addition: (a: List[Int], b: List[Int])List[Int]

/*
** A parallel (?) implementation of counting cells.
*/

def count_cells_par(old_child_set: DataFrame, edges: DataFrame, cut_of: Int, dim: Int): List[Int] = {
  if ((cut_of > dim) && (old_child_set.count > 0)){
    return old_child_set.collect().par.map(vert => count_cells_par(old_child_set.intersect(edges.filter(edges("src") === vert(0)).select("dst")), edges, cut_of, dim+1)/* vert_to_f-vector*/).reduce(component_addition(_,_)).updated(dim, 1)
  }
  else{
    return List(0,0,0,0,0,0,0,0,0,0).updated(dim, 1);
  }
}

count_cells_par: (old_child_set: org.apache.spark.sql.DataFrame, edges: org.apache.spark.sql.DataFrame, cut_of: Int, dim: Int)List[Int]

/*
** A parallel (?) implementation of counting cells.
*/

def count_cells_opt(old_child_set: DataFrame, edges: DataFrame, cut_of: Int, dim: Int): List[Int] = {
  if ((cut_of > dim) && (old_child_set.count > 0)){
    return old_child_set.collect().par.map(vert => count_cells_opt_helper(old_child_set.intersect(edges.filter(edges("src") === vert(0)).select("dst")), edges, cut_of, dim+1)/* vert_to_f-vector*/).reduce(component_addition(_,_)).updated(dim, 1)
  }
  else{
    return List(0,0,0,0,0,0,0,0,0,0).updated(dim, 1);
  }
}

def count_cells_opt_helper(old_child_set: DataFrame, edges: DataFrame, cut_of: Int, dim: Int): List[Int] = {
  if ((cut_of > dim) && (old_child_set.count > 0)){
    // The next line should not parallelize the process.
    return old_child_set.collect().map(vert => count_cells_par(old_child_set.intersect(edges.filter(edges("src") === vert(0)).select("dst")), edges, cut_of, dim+1)/* vert_to_f-vector*/).reduce(component_addition(_,_)).updated(dim, 1)
  }
  else{
    return List(0,0,0,0,0,0,0,0,0,0).updated(dim, 1);
  }
}

count_cells_opt: (old_child_set: org.apache.spark.sql.DataFrame, edges: org.apache.spark.sql.DataFrame, cut_of: Int, dim: Int)List[Int]
count_cells_opt_helper: (old_child_set: org.apache.spark.sql.DataFrame, edges: org.apache.spark.sql.DataFrame, cut_of: Int, dim: Int)List[Int]

/*
** Makes it easier to call "count_cells" on a GraphFrame object. Notice
** that we can get wrong results if the input graph has loops. We recommend
** running "count_loops" to see if that is the case.
*/

def f_vector_of_graphframe(graph: GraphFrame, cut_of: Int): List[Int] = {
  var f_vector = List(0,0,0,0,0,0,0,0,0,0);
  if (cut_of > 9){
    print("Too big cut_of")
    f_vector
  }
  count_cells(graph.vertices.select("id"), f_vector, graph.edges.select("src", "dst"), cut_of, 0);
}

f_vector_of_graphframe: (graph: org.graphframes.GraphFrame, cut_of: Int)List[Int]

/*
** Makes it easier to call "count_cells_par" on a GraphFrame object. Notice
** that we can get wrong results if the input graph has loops. We recommend
** running "count_loops" to see if that is the case.
*/

def f_vector_of_graphframe_par(graph: GraphFrame, cut_of: Int): List[Int] = {
  var temp_int = 0;
  if (cut_of > 9){
    print("Too big cut_of")
    temp_int = 9;
  }
  else{
    temp_int = cut_of;
  }
  count_cells_par(graph.vertices.select("id"), graph.edges.select("src", "dst"), temp_int, 0);
}

f_vector_of_graphframe_par: (graph: org.graphframes.GraphFrame, cut_of: Int)List[Int]

/*
** Makes it easier to call "count_cells_opt" on a GraphFrame object. Notice
** that we can get wrong results if the input graph has loops. We recommend
** running "count_loops" to see if that is the case.
*/

def f_vector_of_graphframe_opt(graph: GraphFrame, cut_of: Int): List[Int] = {
  var f_vector = List(0,0,0,0,0,0,0,0,0,0);
  if (cut_of > 9){
    print("Too big cut_of")
    f_vector
  }
  count_cells_opt(graph.vertices.select("id"), graph.edges.select("src", "dst"), cut_of, 0);
}

f_vector_of_graphframe_opt: (graph: org.graphframes.GraphFrame, cut_of: Int)List[Int]

/*
** First we will define a small graph that is manageable by hand. This
** graph encodes for a simplicial complex with f-vector (6, 11, 7, 1).
*/
// Vertex DataFrame
val v = sqlContext.createDataFrame(List(
  ("a", 1),
  ("b", 2),
  ("c", 3),
  ("d", 4),
  ("e", 5),
  ("f", 6)
)).toDF("id", "no")

// Edge DataFrame
val e = sqlContext.createDataFrame(List(
  ("a", "b"),
  ("a", "c"),
  ("b", "c"),
  ("b", "d"),
  ("d", "c"),
  ("b", "e"),
  ("c", "e"),
  ("d", "e"),
  ("a", "f"),
  ("c", "a"),
  ("c", "f")
)).toDF("src", "dst")

val g = GraphFrame(v, e);

v: org.apache.spark.sql.DataFrame = [id: string, no: int]
e: org.apache.spark.sql.DataFrame = [src: string, dst: string]
g: org.graphframes.GraphFrame = GraphFrame(v:[id: string, no: int], e:[src: string, dst: string])

f_vector_of_graphframe(g, 5)

res18: List[Int] = List(6, 11, 7, 1, 0, 0, 0, 0, 0, 0)

f_vector_of_graphframe_par(g, 5)

res19: List[Int] = List(1, 6, 11, 7, 1, 0, 0, 0, 0, 0)

f_vector_of_graphframe_opt(g, 5)

res20: List[Int] = List(1, 6, 11, 7, 1, 0, 0, 0, 0, 0)

println(g.vertices.count)
println(count_K2(g))
println(count_K3(g))
println(count_K4(g))

6
11
7
1

//f_vector_of_graphframe_opt(rat_brain_graph, cut_of = 2)

# Load packages

import numpy as np

M = np.load('/dbfs/FileStore/shared_uploads/adlindhe@kth.se/M.npy')
np.savetxt("/dbfs/FileStore/shared_uploads/adlindhe@kth.se/M.csv", M, delimiter=",")

/*
** Example on how we can read the data.
*/
val Ms = spark.read.format("csv").option("sep",",").option("MaxColumns",40000).load("/FileStore/shared_uploads/adlindhe@kth.se/M.csv")

# Loop over the whole matrix. This takes time (~ 1h).

edges_file = open("/dbfs/FileStore/shared_uploads/petterre@kth.se/edges.csv", "w")

for i in range(len(M)):
  for j in range(len(M[i])):
    for k in range(M[i][j]):
      edges_file.write(str(i) + "," + str(j) + ",edge\n")

edges_file.close()

head /dbfs/FileStore/shared_uploads/petterre@kth.se/edges.csv

Ms.cache // cache the DataFrame this time
Ms.count // now after this action, the next count will be fast...

display(Ms)

import numpy as np

def matrix_to_string(input_matrix, input_function_name, loops = True):
  ret_string = "def " + input_function_name + "(input_graph: GraphFrame) = {\n\tval string_" +input_function_name+ " = \""
  filter_string = ""
  pos_edges = ""
  neg_edges = ""
  alphabet = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']
  
  for i in range(len(input_matrix)):
    for j in range(len(input_matrix[i])):
      if i == j:
        if loops and input_matrix[i][j] != 0:
          pos_edges += "(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
        elif loops:
          neg_edges += "!(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
      elif input_matrix[i][j] != 0:
        pos_edges += "(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
      else:
        neg_edges += "!(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
      if i > j:
        filter_string += ".filter(\"" + alphabet[j] + ".id != " + alphabet[i] + ".id\")"
  ret_string += pos_edges + neg_edges
  ret_string = ret_string[0:-2]
  ret_string += "\"\n\tinput_graph.find(g_1)"
  ret_string += filter_string
  ret_string += ".count\n}\n"
  return ret_string
  
  
  

def matrix_to_string_signed(input_matrix, input_function_name):
  ret_string = "def " + input_function_name + "(input_graph: GraphFrame) = {\n\tval string_" +input_function_name+ " = \""
  filter_string = ""
  pos_edges = ""
  neg_edges = ""
  alphabet = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']
  
  for i in range(len(input_matrix)):
    for j in range(len(input_matrix[i])):
      if input_matrix[i][j] == 1:
        pos_edges += "(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
      elif input_matrix[i][j] == -1:
        neg_edges += "!(" + alphabet[i] + ")-[]->(" + alphabet[j] + "); "
      if i > j:
        filter_string += ".filter(\"" + alphabet[j] + ".id != " + alphabet[i] + ".id\")"
  ret_string += pos_edges + neg_edges
  ret_string = ret_string[0:-2]
  ret_string += "\"\n\tinput_graph.find(g_1)"
  ret_string += filter_string
  ret_string += ".count\n}\n"
  return ret_string

print(matrix_to_string([[0,0,0,1],[0,0,0,1],[0,0,0,1],[0,0,0,0]], "count_claw", loops = False))

def count_claw(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(d); (b)-[]->(d); (c)-[]->(d); !(a)-[]->(b); !(a)-[]->(c); !(b)-[]->(a); !(b)-[]->(c); !(c)-[]->(a); !(c)-[]->(b); !(d)-[]->(a); !(d)-[]->(b); !(d)-[]->(c)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").filter("a.id != d.id").filter("b.id != d.id").filter("c.id != d.id").count
}

print(matrix_to_string([[0,0,1],[0,0,1],[0,0,0]], "count_v_struc", loops = False))

def count_v_struc(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(c); (b)-[]->(c); !(a)-[]->(b); !(b)-[]->(a); !(c)-[]->(a); !(c)-[]->(b)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").count
}

print(matrix_to_string([[1]], "count_loop"))

def count_loop(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(a)"
	input_graph.find(g_1).count
}

def adj_matrix_complete_graph(size):
  adj_matrix = np.zeros((size,size))
  for i in range(len(adj_matrix)):
    for j in range(i+1, len(adj_matrix[i])):
      adj_matrix[i][j] = 1
  return adj_matrix
  

for i in range(3,7+1):
  print(matrix_to_string_signed(adj_matrix_complete_graph(i), "count_K"+str(i)))

def count_K3(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(b); (a)-[]->(c); (b)-[]->(c)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").count
}

def count_K4(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(b); (a)-[]->(c); (a)-[]->(d); (b)-[]->(c); (b)-[]->(d); (c)-[]->(d)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").filter("a.id != d.id").filter("b.id != d.id").filter("c.id != d.id").count
}

def count_K5(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(b); (a)-[]->(c); (a)-[]->(d); (a)-[]->(e); (b)-[]->(c); (b)-[]->(d); (b)-[]->(e); (c)-[]->(d); (c)-[]->(e); (d)-[]->(e)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").filter("a.id != d.id").filter("b.id != d.id").filter("c.id != d.id").filter("a.id != e.id").filter("b.id != e.id").filter("c.id != e.id").filter("d.id != e.id").count
}

def count_K6(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(b); (a)-[]->(c); (a)-[]->(d); (a)-[]->(e); (a)-[]->(f); (b)-[]->(c); (b)-[]->(d); (b)-[]->(e); (b)-[]->(f); (c)-[]->(d); (c)-[]->(e); (c)-[]->(f); (d)-[]->(e); (d)-[]->(f); (e)-[]->(f)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").filter("a.id != d.id").filter("b.id != d.id").filter("c.id != d.id").filter("a.id != e.id").filter("b.id != e.id").filter("c.id != e.id").filter("d.id != e.id").filter("a.id != f.id").filter("b.id != f.id").filter("c.id != f.id").filter("d.id != f.id").filter("e.id != f.id").count
}

def count_K7(input_graph: GraphFrame) = {
	val g_1 = "(a)-[]->(b); (a)-[]->(c); (a)-[]->(d); (a)-[]->(e); (a)-[]->(f); (a)-[]->(g); (b)-[]->(c); (b)-[]->(d); (b)-[]->(e); (b)-[]->(f); (b)-[]->(g); (c)-[]->(d); (c)-[]->(e); (c)-[]->(f); (c)-[]->(g); (d)-[]->(e); (d)-[]->(f); (d)-[]->(g); (e)-[]->(f); (e)-[]->(g); (f)-[]->(g)"
	input_graph.find(g_1).filter("a.id != b.id").filter("a.id != c.id").filter("b.id != c.id").filter("a.id != d.id").filter("b.id != d.id").filter("c.id != d.id").filter("a.id != e.id").filter("b.id != e.id").filter("c.id != e.id").filter("d.id != e.id").filter("a.id != f.id").filter("b.id != f.id").filter("c.id != f.id").filter("d.id != f.id").filter("e.id != f.id").filter("a.id != g.id").filter("b.id != g.id").filter("c.id != g.id").filter("d.id != g.id").filter("e.id != g.id").filter("f.id != g.id").count
}

from random import randrange
import random
from pathlib import Path
# External libs added to cluster
from pyspark.mllib.random import RandomRDDs
from pyspark.ml.feature import StringIndexer, OneHotEncoder, StandardScaler, VectorAssembler
from pyspark.ml import Pipeline
from pyspark.rdd import PipelinedRDD
from toolz.itertoolz import partition_all
from toolz.itertoolz import cons
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from matplotlib import pyplot as plt

def create_gaussian_RDD(means, variances, num_observations, class_proportions, train_test_split=False):
  """Create toy Gaussian classification data
  Let C := number of clusters/classes and P := number of data features

  Args:
    means (np.array[float]): mean vector of shape (C, P)
    variances (np.array[float]): vector of variances, shape (C, P)
    num_observations (scalar[int]): the total number of observations in the final data set
    class_proportions (np.array[float]): vector of class proportions, length C
    train_test_split: whether to split the data into train/test sets or not

  Returns:
    Gaussian data, RDD of tuples (list(features), int(label))
  """

  assert means.shape[0] == variances.shape[0]
  assert means.shape[1] == variances.shape[1]
  assert class_proportions.sum() == 1

  num_classes = means.shape[0]
  num_features = means.shape[1]

  data_rdd = sc.emptyRDD()
  for k in range(num_classes):

    # Generate standard normal data
    class_size = int(num_observations * class_proportions[k])
    class_rdd = RandomRDDs.normalVectorRDD(sc, numRows=class_size, numCols=num_features, numPartitions=1) #, seed=123)

    # Map to true distribution
    class_rdd_transformed = class_rdd.map(lambda v: means[k, :] + (variances[k, :]**0.5) * v)

    # Add labels
    class_rdd_w_label = class_rdd_transformed.map(lambda v: (v, k))

    data_rdd = data_rdd.union(class_rdd_w_label)

  # We will shuffle and repartition the data
  num_partitions = 10
  shuffled_rdd =  data_rdd.sortBy(lambda v: randrange(num_observations)).repartition(num_partitions)
  final_rdd = shuffled_rdd.map(tuple).map(lambda v: (list(v[0]), int(v[1])))

  if train_test_split:
    train_rdd, test_rdd = final_rdd.randomSplit(weights=[0.8, 0.2], seed=12)
    final_rdd = (train_rdd, test_rdd)

  return final_rdd                                   

def load_firewall_data(train_test_split=False,file_location="/FileStore/shared_uploads/amanda.olmin@liu.se/fire_wall_data.csv"):
  """Load and preprocess firewall data
  Args:
     file_location: file location from which to load the data
     train_test_split: whether to split the data into train/test sets or not

  Returns:
     Firewall data, RDD of tuples (list(features), int(label))
  """

  # File location and type
  # file_location = "/FileStore/shared_uploads/amanda.olmin@liu.se/fire_wall_data.csv"
  file_type = "csv"

  # CSV options
  infer_schema = "true"
  first_row_is_header = "true"
  delimiter = ","

  # Load the data from file
  df = spark.read.format(file_type) \
      .option("inferSchema", infer_schema) \
      .option("header", first_row_is_header) \
      .option("sep", delimiter) \
      .load(file_location)

  # Preprocess data
  col_num = ["Source Port", "Destination Port", "NAT Source Port", "NAT Destination Port", "Bytes", "Bytes Sent", "Bytes Received", "Packets", "Elapsed Time (sec)", "pkts_sent", "pkts_received"]

  # Index qualitative variable
  indexer = StringIndexer(inputCol = "Action", outputCol = "label")

  # Scale numerical features
  va = VectorAssembler(inputCols = col_num, outputCol = "numerical_features")
  scaler = StandardScaler(inputCol = "numerical_features", outputCol = "features")

  # Apply pipeline
  pipeline = Pipeline(stages=[indexer, va, scaler])
  final_df = pipeline.fit(df).transform(df).select("features", "label")

  # Convert to RDD
  final_rdd = final_df.rdd.map(tuple).map(lambda v: (list(v[0]), int(v[1])))

  if train_test_split:
    train_rdd, test_rdd = final_rdd.randomSplit(weights=[0.8, 0.2], seed=12)
    final_rdd = (train_rdd, test_rdd)

  return final_rdd

def get_partitioned_rdd(input_rdd, partition_size=1000):
  """Partition RDD

  Args:
    input_rdd: RDD to be partitioned

  Returns:
    Partitioned RDD
  """

  return input_rdd.mapPartitions(lambda partition: partition_all(partition_size, partition))

#Feedforward network for classification
class MLP(nn.Module):

  def __init__(self,shape):
    #shape: number of neurons in each layer (including the input and output layers)
    super(MLP,self).__init__()

    self.units=nn.ModuleList()
    for i in range(len(shape)-1):
      self.units.append(nn.Linear(shape[i],shape[i+1]))

    self._shape=shape
    self._nlayers=len(shape)

  def forward(self,x):

    y=x

    for i,layer in enumerate(self.units):
      if i<self._nlayers-2:
        y=nn.functional.tanh(layer(y))
      else:
        y=nn.functional.softmax(layer(y),dim=1)

    return y

  #constructing an instance of this class based on a state dictionary (network parameters)
  @staticmethod
  def from_state_dict(state_dict):
    net_shape = MLP.shape_from_state_dict(state_dict)
    net=MLP(net_shape)
    net.load_state_dict(state_dict)
    return net

  @staticmethod
  def shape_from_state_dict(state_dict):
    """Infer MLP layer shapes from state_dict"""
    iter_ = iter(state_dict.items())
    _, input_size = next(iter_)
    bias_tensors = filter(lambda key_val: key_val[0].find("bias") != -1, iter_)
    shapes = map(lambda key_val: key_val[1].size(0), bias_tensors)
    return list(cons(input_size.size(1), shapes))

#utility class for pytorch data loader
class DataSet(torch.utils.data.Dataset):
  def __init__(self, x, y):
    self.x = x
    self.y = y

  def __len__(self):
    return self.x.shape[0]

  def __getitem__(self, ind):
    x = self.x[ind]
    y = self.y[ind]

    return x, y

#The main training function (is run on worker nodes)
def Train(net_params,x,y):
  #net_params: initial parameters of the feedforward network (state dictionary)
  #x,y: training data (pytorch tensors)

  n_epochs=100
  batchsize=10

  net=MLP.from_state_dict(net_params)

  train_data = DataSet(x, y)
  dataloader = torch.utils.data.DataLoader(train_data, batch_size=batchsize)

  opt=optim.Adam(net.parameters())
  loss=nn.CrossEntropyLoss()

  for i in range(n_epochs):
    for batch in dataloader:

      opt.zero_grad()

      xb,yb=batch

      yhat=net(xb)
      err=loss(yhat,yb)
      err.backward()

      opt.step()

  err=loss(net(x),y)
  lossval=float(err.detach().numpy())

  #returns parameters of the trained network and loss
  return (net.state_dict(),lossval)

#Get the output of a feedforward network given an input tensor
def Predict(net_params, x):
  #net_params: parameters (state dictionary) of the network
  #x: input (pytorch tensor)
  net = MLP.from_state_dict(net_params)
  net.eval()
  return net(x)

#Reshaping and converting the tuples stored in a dataset RDD into input and target tensors
def Totensor(d):
  #d: the dataset (list of tuples)

  x=[v[0] for v in d]
  y=[v[1] for v in d]
  x=torch.tensor(x,dtype=torch.float)
  y=torch.tensor(y,dtype=torch.long)

  return (x,y)

def make_prediction(state_dict, x):
  print(state_dict)
  return Predict(state_dict, x)

def train_ensemble(n_models, inputdims, nclasses, max_layers, min_neurons, max_neurons, data_iterator):
  """Constructing and training a distributed ensemble of feedforward networks

    Args:
      n_models: number of the ensemble memebrs
      inputdims: number of features dimesnions
      nclasses: number of the classes
      max_layers: maximum allowed number of hidden layers for the networks
      min_neurons,max_neurons: the valid range for the number of neurons in each hidden layer
      data_iterator: a Python iterator over the parts of the training data (one part per each member of the ensemble)

    Returns: a list of state dictionaries of the trained networks
  """

  # initialization
  model_data=[] # pairs of model parameters and their training data
  for i in range(n_models):
    # pick random number of hidden layers and neurons for each network
    nhidden=random.randint(1, max_layers)
    shape=[inputdims]
    for k in range(nhidden):
      shape.append(random.randint(min_neurons, max_neurons))
    shape.append(nclasses)

    net=MLP(shape)

    #fetch the next part of data
    d=next(data_iterator)
    x=d[0]
    y=d[1]

    model_data.append((net.state_dict(),x,y))

  # distribute the array
  model_data_par= sc.parallelize(model_data)
  # execute the train function on the worker nodes
  models_trained = model_data_par.map(lambda t: Train(*t))

  #transfer the trained models and loss values to the driver
  models_trained=models_trained.collect()

  #print the training loss values
  print("training losses:")
  print([v[1] for v in models_trained])

  # return the state dicts
  return [v[0] for v in models_trained]

def save_models_distr(models, dir_, model_names=None):
  dir_ = Path(dir_)
  dir_.mkdir(exist_ok=True, parents=True)

  if model_names is None:
    model_names = [f"m{idx}.pt" for idx in range(0, models.count())]
  assert len(model_names) == models.count()
  model_paths = [dir_ / model_name for model_name in model_names]
  model_paths = sc.parallelize(model_paths)
  models.zip(model_paths).foreach(lambda dict_and_path: torch.save(*dict_and_path))

def save_models(models, dir_, model_names=None):
  dir_ = Path(dir_)
  dir_.mkdir(exist_ok=True, parents=True)

  if model_names is None:
    model_names = [f"m{idx}.pt" for idx in range(0, len(models))]
  assert len(model_names) == len(models)
  model_paths = [dir_ / model_name for model_name in model_names]
  for state_dict, path in zip(models, model_paths):
    torch.save(state_dict, path)

def load_models(model_names, dir_):
  dir_ = Path(dir_)
  model_paths = [dir_ / model_name for model_name in model_names]
  state_dicts = [torch.load(path) for path in model_paths]
  return sc.parallelize(state_dicts)

def ens_preds(models, test_x):
  """Distributed ensemble predictions
  Takes a set of models and test data and makes distributed predictions
  Let N := number of data points and D_x := the dimension of a single datapoint x

  Args:
    models (list[state_dict]): set of models represented as a list (state_dict, shape)
    test_x (torch.Tensor): Tensor of size (N, D_x)

  Returns:
    Distributed iterator over the predictions. E.g. an iterator over probability vectors in the case of a classifier ens.
  """
  pred_iter = _pred_models_iter(models, test_x)
  return pred_iter.map(lambda t: Predict(*t))

def ens_preds_reduced(models, test_x, red_fn):
  """Reduced/aggregated ensemble predictions
  Takes a set of models and test data and makes distributed predictions and reduces them with a provided `red_fn`
  Let N := number of data points and D_x := the dimension of a single datapoint x

  Args:
    models (list[state_dict]): set of models represented as a list (state_dict, shape)
    test_x (torch.Tensor): Tensor of size (N, D_x)
    red_fn function: f: R^D_x x R^D_x --> R^D_x

  Returns:
    Single reduced/aggregated prediction of the whole ensemble
  """
  return ens_preds(models, test_x).reduce(red_fn)

def ens_metrics(models, test_x, test_y, metrics):
  """Distributed ensemble metrics
  Takes a set of models and test data, predicts probability vectors and calculates the provided metrics
  given true labels `test_y`
  Let N := number of data points and D_x := the dimension of a single datapoint x

  Args:
    models (list[state_dict]): set of models represented as a list (state_dict, shape)
    test_x (torch.Tensor): Tensor of size (N, D_x)
    test_y (torch.Tensor): Tensor of size (N). NB: hard labels
    metrics (list[functions]): List of functions where each funcion f: R^(N x D_x) x R^(N) --> T, where T is a generic output type.
  """
  return ens_preds(models, test_x).map(lambda prob_vecs: [metric(prob_vecs, test_y) for metric in metrics])


def _pred_models_iter(models, test_x):
  """Helper function to generate a distributed iterator over models and test data
  NB: the same `test_x` is given to all elements in the iterator

  Args:
    models (list[state_dict]): set of models represented as a list (state_dict, shape)
    test_x (torch.Tensor): Tensor of size (N, D_x)
  """
  if isinstance(models, PipelinedRDD):
    return models.map(lambda model: (model, test_x))
  elif isinstance(models, list):
    models_and_data = [(params, test_x) for params in models]
    return sc.parallelize(models_and_data)
  else:
    raise TypeError("'models' must be an RDD or a list")

def avg_accuracy(prob_vecs, labels):
  """Example metrics function: average accuracy
  Let N := number of data points and C := the number of classes

  Args:
    prob_vecs (torch.Tensor): Tensor of size (N, C)
    labels (torch.Tensor): Tensor of size (N), hard labels, with classes corresponding to indices 0, ..., C-1

  Returns:
    torch.Tensor: Tensor of size (N), average accuracy over all datapoints.
  """
  hard_preds = torch.argmax(prob_vecs, 1)
  return (hard_preds == labels).float().mean()

def entropy(prob_vecs):
  return - (prob_vecs * torch.log(prob_vecs)).sum(1)

def avg_entropy(prob_vec_1, prob_vec_2):
  e_1 = entropy(prob_vec_1)
  e_2 = entropy(prob_vec_2)
  return (e_1 + e_2)

means = np.array([(0, 0), (1,0), (1, 1)])
variances = 0.1 * np.ones((3, 2))
num_observations = 5000
class_proportions = np.array([1/3, 1/3, 1/3])
data_train, data_test = create_gaussian_RDD(means, variances, num_observations, class_proportions, train_test_split=True)

data_iterator=get_partitioned_rdd(data_train).map(Totensor).toLocalIterator()
n_models=5 # ensemble size
inputdims=2 # features dimensions
nclasses=3 # number of classes
max_layers=2
min_neurons=2
max_neurons=5

models_trained = train_ensemble(n_models, inputdims, nclasses, max_layers, min_neurons, max_neurons, data_iterator)
saved_models_dir = Path("saved_models/gaussian")
save_models(models_trained, saved_models_dir)

training losses:
[0.6597320437431335, 0.6314507126808167, 0.6506103277206421, 0.6424266695976257, 0.657072901725769]

test_xx, test_yy = Totensor(data_test.collect())
model_names = [f"m{idx}.pt" for idx in range(n_models)]
models = load_models(model_names, saved_models_dir).collect()
avg_prob_vecs = ens_preds_reduced(models, test_xx, lambda x, y: (x+y)/2) # (A single) Average prob. vec for all data points.
avg_acc = ens_metrics(models, test_xx, test_yy, [avg_accuracy]).collect() # Average acc. for each ens. over all data points

print(f"Average accuracy for each ensemble member: {[acc[0].item() for acc in avg_acc]}")
print(f"Average accuracy for the whole ensemble: {avg_accuracy(avg_prob_vecs, test_yy).item()}")

Average accuracy for each ensemble member: [0.9240759015083313, 0.9240759015083313, 0.9160839319229126, 0.9110888838768005, 0.9220778942108154]
Average accuracy for the whole ensemble: 0.9210789203643799

preds = avg_prob_vecs.detach().numpy()
hard_preds = avg_prob_vecs.argmax(1).detach().numpy()
every_nth = 5
train_xx, train_yy = Totensor(data_train.collect())

(fig, (ax_1, ax_2)) = plt.subplots(1, 2)
# For the train data we use the true labels to simulate a completely certain prediction.
color_map = {0: [1, 0 ,0], 1: [0, 1, 0], 2: [0, 0, 1]}
ax_1.scatter(train_xx[:, 0], train_xx[:, 1], c=[color_map[class_.item()] for class_ in train_yy], label="Train")
ax_2.scatter(test_xx[::every_nth, 0], test_xx[::every_nth, 1], c=preds[::every_nth], label="Test")
ax_1.set_title("Train")
ax_2.set_title("Test")
plt.show()

data_train, data_test = load_firewall_data(True)
data_iterator=get_partitioned_rdd(data_train).map(Totensor).toLocalIterator()

n_models=10
models_trained=train_ensemble(n_models,
                              inputdims=11,
                              nclasses=4,
                              max_layers=4,
                              min_neurons=5,
                              max_neurons=15,
                              data_iterator=data_iterator)
saved_models_dir = Path("saved_models/firewall")
save_models(models_trained, saved_models_dir)

training losses:
[0.7487057447433472, 0.750811755657196, 0.7578539252281189, 0.7506877183914185, 0.7623474597930908, 0.7521470785140991, 0.7517384886741638, 0.7481321692466736, 0.750828742980957, 0.7547193765640259]

def gen_ood_data(test_x, num_samples):
  num_test_samples, dim_x = test_x.size()
  random_mean = np.random.rand(dim_x).reshape(1, dim_x)
  random_cov = np.random.rand(dim_x).reshape(1, dim_x) * 10
  ood_x, _ = Totensor(create_gaussian_RDD(random_mean, random_cov, num_test_samples, np.array([1.0]), train_test_split=False).collect())
  return ood_x


data = data_test.collect()
batch_size = -1
batch = data[0:batch_size]
test_xx, test_yy = Totensor(batch)
ood_x = gen_ood_data(test_xx, batch_size)

models_p = load_models(model_names, saved_models_dir).collect()

# We can either calculate the average entropy of the ensemble members
avg_entropy_id = ens_preds(models_p, test_xx).map(entropy).reduce(lambda x, y: (x+y)/2).detach().numpy()
avg_entropy_ood = ens_preds(models_p, ood_x).map(entropy).reduce(lambda x, y: (x+y)/2).detach().numpy()

# ... or we the entropy of the average ensemble prediction.
entropy_avg_id = entropy(ens_preds_reduced(models_p, test_xx, lambda x, y: (x+y)/2)).detach().numpy()
entropy_avg_ood = entropy(ens_preds_reduced(models_p, ood_x, lambda x, y: (x+y)/2)).detach().numpy()

# Set entropy measure
entropy_id = avg_entropy_id
entropy_ood = avg_entropy_ood

def entropy_hist(id_, ood, n_bins, upper_x_bound):
  (fig, (ax_1, ax_2)) = plt.subplots(2, 1)
  _plot_hist(ax_1, id_, n_bins, "ID", "b", upper_x_bound)
  _plot_hist(ax_2, ood, n_bins, "OOD", "r", upper_x_bound)
  fig.suptitle("Entropy histogram")
  ax_2.set_xlabel("entropy")
  plt.show()

def _plot_hist(ax, counts, n_bins, label, color, upper_x_bound):
    ax.hist(counts, bins=n_bins, label=label, color=color, density=True)
    ax.set_xbound(lower = 0.0, upper = upper_x_bound)
    ax.set_ylabel("rel freq")
    ax.legend()

n_bins = 100
entropy_bound = 0.15
entropy_hist(entropy_id, entropy_ood, n_bins, entropy_bound)

def is_ood(entropies, cut_off_entropy):
  return entropies > cut_off_entropy

def fpr_and_tpr(id_, ood, res):
  max_entropy = max(id_.max(), ood.max())
  # max_entropy = id_.max()
  thresholds = np.arange(0.0, max_entropy, max_entropy / res)
  roc = np.array([(fpr(id_, th), tpr(ood, th)) for th in thresholds])
  roc = roc[roc[:,0].argsort()]
  fprs, tprs = (roc[:, 0], roc[:, 1])
  return fprs, tprs

def fpr(id_, th):
  id_pred = is_ood(id_, th)
  fp = id_pred.sum()
  tn = id_pred.shape[0] - fp
  return fp / (tn + fp)

def tpr(ood, th):
  ood_pred = is_ood(ood, th)
  tp = ood_pred.sum()
  fn = ood_pred.shape[0] - tp
  return tp / (tp + fn)

fpr, tpr = fpr_and_tpr(avg_entropy_id, avg_entropy_ood, res = 100)
(fig, ax) = plt.subplots()

ax.plot(fpr, tpr)
ax.set_xlabel("FPR")
ax.set_ylabel("TPR")
ax.set_title("ROC")

print(f"AUROC: {np.trapz(tpr, fpr)}")

AUROC: 0.6480554090292237

// Paths to datasets of different regions.
val paths: List[String] = List("dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_oceania.fasta",
                               "dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_northamerica.fasta",
                               "dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_southamerica.fasta",
                               "dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_europe.fasta",
                               "dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_africa.fasta",
                               "dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_asia.fasta")

paths: List[String] = List(dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_oceania.fasta, dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_northamerica.fasta, dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_southamerica.fasta, dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_europe.fasta, dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_africa.fasta, dbfs:/FileStore/shared_uploads/hugower@kth.se/sequences_asia.fasta)

import scala.util.matching.Regex

// regex pattern to take region name, label, from complete path name (Must be changed accordingly if path follows a different structure)
val pattern: Regex = "/[a-zA-Z]+_([a-zA-Z]+)\\.".r 

def read_datasets(paths:List[String]): List[RDD[(String,String)]] = {
  if (paths.size < 1) { // return an empty RDD
    return List.fill(0) (sc.emptyRDD)
  }
  else {
    pattern.findFirstMatchIn(paths.head) match { // extract the label based on the pattern defined above
      case Some(x) => {
        val label:String = x.group(1)  // create the label based on the path name
        return (sc.textFile(paths.head).filter(_ != "").map(_.trim()).map(s => (s,label)))::read_datasets(paths.tail) // read the file in path and attach the data with its label to RDD list
      }
      case None => throw new RuntimeException("no label found")
    }
  }
}

import scala.util.matching.Regex
pattern: scala.util.matching.Regex = /[a-zA-Z]+_([a-zA-Z]+)\.
read_datasets: (paths: List[String])List[org.apache.spark.rdd.RDD[(String, String)]]

// read data and set the delimiter as ">" which seperates each sample in fasta format
sc.hadoopConfiguration.set("textinputformat.record.delimiter",">")
val datasets = read_datasets(paths)
  

datasets: List[org.apache.spark.rdd.RDD[(String, String)]] = List(MapPartitionsRDD[202189] at map at command-3103574048361361:13, MapPartitionsRDD[202196] at map at command-3103574048361361:13, MapPartitionsRDD[202205] at map at command-3103574048361361:13, MapPartitionsRDD[202210] at map at command-3103574048361361:13, MapPartitionsRDD[202215] at map at command-3103574048361361:13, MapPartitionsRDD[202220] at map at command-3103574048361361:13)

datasets.length

res0: Int = 6

datasets(0).take(1)

// combine the datasets into one and cache for optimization
val data = datasets.reduce( (a,b) => a++b).cache()

data: org.apache.spark.rdd.RDD[(String, String)] = UnionRDD[202273] at $plus$plus at command-3103574048361373:1

data.take(1)

// get the headers for each sample (the first line of each sample is a header)
val headers = data.map( {case (genome,label) => (genome.split("\n").head.split('|'),label)})
headers.count

headers: org.apache.spark.rdd.RDD[(Array[String], String)] = MapPartitionsRDD[35] at map at command-3103574048360661:1
res2: Long = 31550

headers.take(5)

res3: Array[(Array[String], String)] = Array((Array("MW320729.1 ", Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/AUS/VIC16982/2020, complete genome),oceania), (Array("MW320730.1 ", Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/AUS/VIC17307/2020, complete genome),oceania), (Array("MW320731.1 ", Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/AUS/VIC17193/2020, complete genome),oceania), (Array("MW320733.1 ", Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/AUS/VIC16732/2020, complete genome),oceania), (Array("MW320735.1 ", Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/AUS/VIC16821/2020, complete genome),oceania))

// remove the headers and only get genome sequences of samples.
val samples = data.map( {case (genome,label) => (genome.split("\n").tail.mkString(""), label)}).cache()
samples.count

samples: org.apache.spark.rdd.RDD[(String, String)] = MapPartitionsRDD[202309] at map at command-3103574048360662:2
res0: Long = 31550

// get the genome lengths per sample (this is just to check if there are extreme cases so we would remove those)
val genome_length_per_s = samples.map({case (genome,label) => genome.length()})

genome_length_per_s: org.apache.spark.rdd.RDD[Int] = MapPartitionsRDD[14298] at map at command-3103574048360664:1

// check the statistics if there is any significant variation
genome_length_per_s.stats

res6: org.apache.spark.util.StatCounter = (count: 31550, mean: 29812.288621, stdev: 81.069114, max: 30018.000000, min: 28645.000000)

// A tail recursive overlapping subsequence function 
// ex1: input: ("abcd", 2, true) -> output: "ab bc cd": 
// ex2: input: ("abcd", 2, false) -> output: "ab cd"
def subsequence_str( sequence:String, k:Int, overlapping:Boolean ): String = {
  def helper(seq:String, acc:String): String = {
    if (seq.length < k ) {
      return acc
    }
    else {
      val sub = seq.substring(0,k)
      if(overlapping) helper(seq.tail, acc + sub + " ")
      else helper(seq.substring(k), acc + sub + " ")
    }
  }
  return helper(sequence, "")
}

subsequence_str: (sequence: String, k: Int, overlapping: Boolean)String

// Extract the subsequences, kmers, for each sample
val k_mers = samples.map( {case (genome,label) => (subsequence_str(genome, 3, false),label)} ).cache()

k_mers: org.apache.spark.rdd.RDD[(String, String)] = MapPartitionsRDD[202801] at map at command-3103574048360668:1

k_mers.take(1)

// index kmers
val kmers_df = k_mers.zipWithIndex.map({case ((a,b),c) => (a,b,c)}).toDF("genome", "label", "id").cache()

kmers_df: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]

kmers_df.take(1)

// split train and test data
val split = kmers_df.randomSplit(Array(0.7, 0.3), seed=42)

split: Array[org.apache.spark.sql.Dataset[org.apache.spark.sql.Row]] = Array([genome: string, label: string ... 1 more field], [genome: string, label: string ... 1 more field])

val train = split(0).cache()
train.take(1)

train.count

res6: Long = 22155

val test = split(1).cache()
test.take(1)

test.count

res9: Long = 9395

// save the results for the next notebook
dbutils.fs.rm("/FileStore/shared_uploads/caylak@kth.se/data_test_nonoverlapping", recurse=true) // remove existing folder
test.write.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_test_nonoverlapping")

dbutils.fs.rm("/FileStore/shared_uploads/caylak@kth.se/data_train_nonoverlapping", recurse=true) // remove existing folder
train.write.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_train_nonoverlapping")

val k_mers_df_train = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_train_nonoverlapping").cache()
val k_mers_df_test = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_test_nonoverlapping").cache()

k_mers_df_train: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]
k_mers_df_test: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]

import org.apache.spark.ml.feature.RegexTokenizer

// Set params for RegexTokenizer
val tokenizer = new RegexTokenizer()
.setPattern("[\\W_]+") // break by white space character(s)
.setInputCol("genome") // name of the input column
.setOutputCol("tokens") // name of the output column

// Tokenize train and test documents
val tokenized_df_train = tokenizer.transform(k_mers_df_train)
val tokenized_df_test = tokenizer.transform(k_mers_df_test)

import org.apache.spark.ml.feature.RegexTokenizer
tokenizer: org.apache.spark.ml.feature.RegexTokenizer = RegexTokenizer: uid=regexTok_b3a99fc4c607, minTokenLength=1, gaps=true, pattern=[\W_]+, toLowercase=true
tokenized_df_train: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 2 more fields]
tokenized_df_test: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 2 more fields]

display(tokenized_df_train.select("tokens"))

display(tokenized_df_test.select("tokens"))

import org.apache.spark.ml.feature.CountVectorizerModel
// create a dictionary array from all possible k-mers
val k = 3
val fixed_vocab = List.fill((k))(List("a","t","g","c")).flatten.combinations(k).flatMap(_.permutations).toArray.map(_.mkString("")) // https://stackoverflow.com/questions/38406959/creating-all-permutations-of-a-list-with-a-limited-range-in-scala
val fixed_vectorizer = new CountVectorizerModel(fixed_vocab)
.setInputCol("tokens")
.setOutputCol("features")

import org.apache.spark.ml.feature.CountVectorizerModel
k: Int = 3
fixed_vocab: Array[String] = Array(aaa, aat, ata, taa, aag, aga, gaa, aac, aca, caa, att, tat, tta, atg, agt, tag, tga, gat, gta, atc, act, tac, tca, cat, cta, agg, gag, gga, agc, acg, gac, gca, cag, cga, acc, cac, cca, ttt, ttg, tgt, gtt, ttc, tct, ctt, tgg, gtg, ggt, tgc, tcg, gtc, gct, ctg, cgt, tcc, ctc, cct, ggg, ggc, gcg, cgg, gcc, cgc, ccg, ccc)
fixed_vectorizer: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVecModel_4bd2f56ddf2e, vocabularySize=64

import org.apache.spark.ml.feature.CountVectorizer

// Create a dictionary of kmers
val vectorizer = new CountVectorizer()
      .setInputCol("tokens")
      .setOutputCol("features")
      .setMinDF(10)  // a term must appear at least in 10 documents to be included in the vocabulary.
      .fit(tokenized_df_train) // create the vocabulary based on the train data

import org.apache.spark.ml.feature.CountVectorizer
vectorizer: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVec_cf83465b3abd, vocabularySize=241

// Vocabulary of k-mers which contains some weird nucleotides
val vocabList = vectorizer.vocabulary

vocabList: Array[String] = Array(ttt, tgt, aaa, tta, aca, ttg, taa, att, aat, ctt, caa, tga, gtt, atg, act, aga, tat, tac, aac, tgg, tgc, aag, tca, cta, ttc, tct, gtg, agt, gaa, cat, gct, ctg, cac, gta, ata, tag, gat, ggt, cag, acc, gca, cca, atc, agg, gac, cct, agc, gag, gga, ctc, gtc, ggc, tcc, gcc, acg, cgt, ggg, ccc, tcg, cgc, cga, gcg, cgg, ccg, nnn, nna, naa, ntt, tnn, cnn, gnn, ann, nnt, nng, nnc, agn, ttn, aan, acn, tan, nat, tcn, ngt, nct, can, gtn, ctn, nta, atn, ana, tgn, nca, ggn, nga, tna, nac, ntg, gcn, gan, tgk, ngc, ccn, ncc, ngg, tnt, ntc, nag, agk, yta, cnt, ktt, aya, gkt, kta, gnt, nan, ytt, ktg, gkc, tty, ayt, tay, yaa, acy, gsc, aay, tgy, ggk, ant, tyt, yac, yat, tya, ang, anc, cay, tkt, cng, cak, rcc, cna, cgn, aty, akt, ggw, gyt, tng, raa, cyt, acw, ytg, aak, yca, ntn, gna, gay, cty, kat, kct, tkg, gnc, ngn, yag, tnc, kca, ayc, tyg, gka, ygt, aka, cnc, cya, ayg, ttk, gng, tth, maa, ncn, yga, tka, ama, aar, ytc, gtk, kag, cch, ncg, ctk, kaa, gty, yct, ara, rtg, ckt, tar, gya, tkc, tak, tgr, ccy, akg, kac, crc, grt, ggr, trt, gcy, tyc, ygg, gak, wga, ygc, cgk, gcr, kgt, wtc, tck, cwt, waa, tcy, vcc, tma, atr, agy, rgc, rac, tgs, kgc, gam, atk, cyc, haa, agr, tha, rgt, gwg, tra, cra, gtr, gkg, nam)

vocabList.size

res4: Int = 241

// Create vector of token counts
val countVectors_train = vectorizer.transform(tokenized_df_train).select("id", "features")
val countVectors_test = vectorizer.transform(tokenized_df_test).select("id", "features")

countVectors_train: org.apache.spark.sql.DataFrame = [id: bigint, features: vector]
countVectors_test: org.apache.spark.sql.DataFrame = [id: bigint, features: vector]

tokenized_df_train.take(1)

countVectors_train.take(5)

countVectors_test.take(5)

import org.apache.spark.ml.linalg.{Vector => MLVector}
import org.apache.spark.mllib.{linalg => mllib}
import org.apache.spark.ml.{linalg => ml}
// convert each sample from ml to mllib vectors (because this causes problems in classifier step)
val lda_countVector_train = countVectors_train.map { case Row(id: Long, countVector: MLVector) => (id, mllib.Vectors.fromML(countVector)) }.cache()
val lda_countVector_test = countVectors_test.map { case Row(id: Long, countVector: MLVector) => (id, mllib.Vectors.fromML(countVector)) }.cache()

import org.apache.spark.ml.linalg.{Vector=>MLVector}
import org.apache.spark.mllib.{linalg=>mllib}
import org.apache.spark.ml.{linalg=>ml}
lda_countVector_train: org.apache.spark.sql.Dataset[(Long, org.apache.spark.mllib.linalg.Vector)] = [_1: bigint, _2: vector]
lda_countVector_test: org.apache.spark.sql.Dataset[(Long, org.apache.spark.mllib.linalg.Vector)] = [_1: bigint, _2: vector]

// format: Array(id, (VocabSize, Array(indexedTokens), Array(Token Frequency)))
lda_countVector_test.take(1)

res10: Array[(Long, org.apache.spark.mllib.linalg.Vector)] = Array((13340,(241,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],[334.0,211.0,327.0,282.0,288.0,277.0,95.0,278.0,315.0,259.0,245.0,70.0,310.0,311.0,291.0,129.0,213.0,181.0,189.0,99.0,87.0,221.0,189.0,183.0,136.0,217.0,181.0,152.0,251.0,136.0,265.0,150.0,108.0,174.0,183.0,89.0,213.0,245.0,163.0,99.0,158.0,138.0,107.0,78.0,142.0,146.0,53.0,116.0,98.0,93.0,101.0,90.0,56.0,73.0,42.0,67.0,34.0,31.0,39.0,29.0,28.0,37.0,20.0,19.0])))

// The number of topics
val num_topics = 20

num_topics: Int = 20

import org.apache.spark.mllib.clustering.{LDA, EMLDAOptimizer,DistributedLDAModel}

val lda = new LDA()
.setOptimizer(new EMLDAOptimizer())
.setK(num_topics)
.setMaxIterations(20000)
.setDocConcentration(-1) // use default values
.setTopicConcentration(-1) // use default values

import org.apache.spark.mllib.clustering.{LDA, EMLDAOptimizer, DistributedLDAModel}
lda: org.apache.spark.mllib.clustering.LDA = org.apache.spark.mllib.clustering.LDA@53a22046

// Run the LDA based on the model described  
val lda_countVector_train_mllib = lda_countVector_train.rdd
val lda_countVector_test_mllib = lda_countVector_test.rdd
val ldaModel = lda.run(lda_countVector_train_mllib)

lda_countVector_train_mllib: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[399] at rdd at command-685894176420037:2
lda_countVector_test_mllib: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[404] at rdd at command-685894176420037:3
ldaModel: org.apache.spark.mllib.clustering.LDAModel = org.apache.spark.mllib.clustering.DistributedLDAModel@4e5ac90e

// Cast to distributed LDA model (which is possible through EMLDAOptimizer in the model) so we can get topic distributions
val distLDAModel = ldaModel.asInstanceOf[DistributedLDAModel]

distLDAModel: org.apache.spark.mllib.clustering.DistributedLDAModel = org.apache.spark.mllib.clustering.DistributedLDAModel@4e5ac90e

val topicIndices = distLDAModel.describeTopics(maxTermsPerTopic = 10)

// https://spark.apache.org/docs/1.5.0/api/scala/index.html#org.apache.spark.mllib.clustering.DistributedLDAModel
// Get the topic distributions for each train document which we will use as features in the classification step
val topicDistributions_train = distLDAModel.topicDistributions.cache()

topicDistributions_train: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[828598] at map at LDAModel.scala:768

lda_countVector_test_mllib.take(1)

res11: Array[(Long, org.apache.spark.mllib.linalg.Vector)] = Array((13340,(241,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],[334.0,211.0,327.0,282.0,288.0,277.0,95.0,278.0,315.0,259.0,245.0,70.0,310.0,311.0,291.0,129.0,213.0,181.0,189.0,99.0,87.0,221.0,189.0,183.0,136.0,217.0,181.0,152.0,251.0,136.0,265.0,150.0,108.0,174.0,183.0,89.0,213.0,245.0,163.0,99.0,158.0,138.0,107.0,78.0,142.0,146.0,53.0,116.0,98.0,93.0,101.0,90.0,56.0,73.0,42.0,67.0,34.0,31.0,39.0,29.0,28.0,37.0,20.0,19.0])))

// Get the topic distributions for each test document which we will use as features in the classification step
val topicDistributions_test = distLDAModel.toLocal.topicDistributions(lda_countVector_test_mllib).cache()

topicDistributions_test: org.apache.spark.rdd.RDD[(Long, org.apache.spark.mllib.linalg.Vector)] = MapPartitionsRDD[828874] at map at LDAModel.scala:373

assert (topicDistributions_train.take(1)(0)._2.size == num_topics)

topicDistributions_train.take(1)

res13: Array[(Long, org.apache.spark.mllib.linalg.Vector)] = Array((31198,[0.050970660127433086,0.03854914539182694,0.04233208760127245,0.05520192993035042,0.04529229810049971,0.044746575004622195,0.04738680077580458,0.04917176116296172,0.028216072817023714,0.06235014298552372,0.07282634237929085,0.05367477019497316,0.02834755979614659,0.007634148695007966,0.0747900052332216,0.07432157320344387,0.05619372014685311,0.05917612984447815,0.059889692006826215,0.048928584602440005]))

topicDistributions_test.take(1)

res14: Array[(Long, org.apache.spark.mllib.linalg.Vector)] = Array((13340,[0.05125119795748651,0.05590356021965471,0.057337790514388406,0.053942804029345516,0.03184697426952284,0.054473799789446706,0.055602712254395316,0.07875351104584705,0.05294681251741617,0.03326797507323466,0.04996448701691076,0.06303737755619679,0.07611049117795103,0.009242706964001621,0.017554379171811085,0.026326662949499827,0.040104268572631795,0.09709248658013904,0.03851877943786023,0.05672122290225994]))

import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
import org.apache.spark.ml.linalg.{Vectors => NewVectors}

val n_topicDistributions_train = topicDistributions_train.map({case (a,b) =>(a,b.asML)})
val n_topicDistributions_test = topicDistributions_test.map({case (a,b) =>(a,b.asML)})

import org.apache.spark.mllib.linalg.{Vectors=>OldVectors}
import org.apache.spark.ml.linalg.{Vectors=>NewVectors}
n_topicDistributions_train: org.apache.spark.rdd.RDD[(Long, org.apache.spark.ml.linalg.Vector)] = MapPartitionsRDD[828876] at map at command-685894176420046:4
n_topicDistributions_test: org.apache.spark.rdd.RDD[(Long, org.apache.spark.ml.linalg.Vector)] = MapPartitionsRDD[828877] at map at command-685894176420046:5

// save the topic distributions for train and test with partitioning for the next notebook
dbutils.fs.rm("/FileStore/shared_uploads/caylak@kth.se/topic_dist_train_t20_i20k_no_cv", recurse=true) // remove existing folder
n_topicDistributions_train.toDF.write.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/topic_dist_train_t20_i20k_no_cv")

dbutils.fs.rm("/FileStore/shared_uploads/caylak@kth.se/topic_dist_test_t20_i20k_no_cv", recurse=true) // remove existing folder
n_topicDistributions_test.toDF.write.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/topic_dist_test_t20_i20k_no_cv")

// Get the top word distributions for each topic
val topics = topicIndices.map { case (terms, termWeights) =>
  terms.map(vocabList(_)).zip(termWeights)
}
println(s"$num_topics topics:")
topics.zipWithIndex.foreach { case (topic, i) =>
  println(s"TOPIC $i")
  topic.foreach { case (term, weight) => println(s"$term\t$weight") }
  println(s"==========")
}

//Zip topic terms with topic IDs
val termArray = topics.zipWithIndex

// Transform data into the form (term, probability, topicId)
val termRDD = sc.parallelize(termArray)
val termRDD2 =termRDD.flatMap( (x: (Array[(String, Double)], Int)) => {
  val arrayOfTuple = x._1
  val topicId = x._2
  arrayOfTuple.map(el => (el._1, el._2, topicId))
})

termRDD: org.apache.spark.rdd.RDD[(Array[(String, Double)], Int)] = ParallelCollectionRDD[829330] at parallelize at command-685894176420051:2
termRDD2: org.apache.spark.rdd.RDD[(String, Double, Int)] = MapPartitionsRDD[829331] at flatMap at command-685894176420051:3

// Create DF with proper column names
val termDF = termRDD2.toDF.withColumnRenamed("_1", "term").withColumnRenamed("_2", "probability").withColumnRenamed("_3", "topicId")

termDF: org.apache.spark.sql.DataFrame = [term: string, probability: double ... 1 more field]

Create JSON data
val rawJson = termDF.toJSON.collect().mkString(",\n")

displayHTML(s"""
<!DOCTYPE html>
<meta charset="utf-8">
<style>

circle {
  fill: rgb(31, 119, 180);
  fill-opacity: 0.5;
  stroke: rgb(31, 119, 180);
  stroke-width: 1px;
}

.leaf circle {
  fill: #ff7f0e;
  fill-opacity: 1;
}

text {
  font: 14px sans-serif;
}

</style>
<body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script>

var json = {
 "name": "data",
 "children": [
  {
     "name": "topics",
     "children": [
      ${rawJson}
     ]
    }
   ]
};

var r = 1000,
    format = d3.format(",d"),
    fill = d3.scale.category20c();

var bubble = d3.layout.pack()
    .sort(null)
    .size([r, r])
    .padding(1.5);

var vis = d3.select("body").append("svg")
    .attr("width", r)
    .attr("height", r)
    .attr("class", "bubble");


var node = vis.selectAll("g.node")
    .data(bubble.nodes(classes(json))
    .filter(function(d) { return !d.children; }))
    .enter().append("g")
    .attr("class", "node")
    .attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
    color = d3.scale.category20();

  node.append("title")
      .text(function(d) { return d.className + ": " + format(d.value); });

  node.append("circle")
      .attr("r", function(d) { return d.r; })
      .style("fill", function(d) {return color(d.topicName);});

var text = node.append("text")
    .attr("text-anchor", "middle")
    .attr("dy", ".3em")
    .text(function(d) { return d.className.substring(0, d.r / 3)});

  text.append("tspan")
      .attr("dy", "1.2em")
      .attr("x", 0)
      .text(function(d) {return Math.ceil(d.value * 10000) /10000; });

// Returns a flattened hierarchy containing all leaf nodes under the root.
function classes(root) {
  var classes = [];

  function recurse(term, node) {
    if (node.children) node.children.forEach(function(child) { recurse(node.term, child); });
    else classes.push({topicName: node.topicId, className: node.term, value: node.probability});
  }

  recurse(null, root);
  return {children: classes};
}
</script>
""")

val k_mers_df_train = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_train").cache()
val k_mers_df_test = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_test").cache()

k_mers_df_train: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]
k_mers_df_test: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]

val k_mers_df_train = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_train_nonoverlapping").cache()
val k_mers_df_test = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_test_nonoverlapping").cache()

k_mers_df_train: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]
k_mers_df_test: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [genome: string, label: string ... 1 more field]

import org.apache.spark.ml.feature.RegexTokenizer

// Set params for RegexTokenizer
val tokenizer = new RegexTokenizer()
.setPattern("[\\W_]+") // break by white space character(s)  - try to remove emails and other patterns
.setInputCol("genome") // name of the input column
.setOutputCol("tokens") // name of the output column

// Tokenize document
val tokenized_df_train = tokenizer.transform(k_mers_df_train)
val tokenized_df_test = tokenizer.transform(k_mers_df_test)

import org.apache.spark.ml.feature.RegexTokenizer
tokenizer: org.apache.spark.ml.feature.RegexTokenizer = RegexTokenizer: uid=regexTok_5df744efa843, minTokenLength=1, gaps=true, pattern=[\W_]+, toLowercase=true
tokenized_df_train: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 2 more fields]
tokenized_df_test: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 2 more fields]

display(tokenized_df_train.select("tokens"))

import org.apache.spark.ml.feature.CountVectorizer
val vectorizer = new CountVectorizer()
      .setInputCol("tokens")
      .setOutputCol("features")
      .setMinDF(10)
      .fit(tokenized_df_train)

import org.apache.spark.ml.feature.CountVectorizer
vectorizer: org.apache.spark.ml.feature.CountVectorizerModel = CountVectorizerModel: uid=cntVec_9226a16a835f, vocabularySize=241

val vocabList = vectorizer.vocabulary

vocabList: Array[String] = Array(ttt, tgt, aaa, tta, aca, ttg, taa, att, aat, ctt, caa, tga, gtt, atg, act, aga, tat, tac, aac, tgg, tgc, aag, tca, cta, ttc, tct, gtg, agt, gaa, cat, gct, ctg, cac, gta, ata, tag, gat, ggt, cag, acc, gca, cca, atc, agg, gac, cct, agc, gag, gga, ctc, gtc, ggc, tcc, gcc, acg, cgt, ggg, ccc, tcg, cgc, cga, gcg, cgg, ccg, nnn, nna, naa, ntt, tnn, cnn, gnn, ann, nnt, nng, nnc, agn, ttn, aan, acn, tan, nat, tcn, ngt, nct, can, gtn, ctn, nta, atn, tgn, ana, nca, ggn, nga, tna, nac, ntg, gcn, gan, tgk, ngc, ccn, ncc, ngg, tnt, ntc, nag, agk, yta, cnt, ktt, aya, gkt, kta, gnt, nan, ytt, ktg, gkc, tty, ayt, tay, yaa, acy, gsc, aay, tgy, ggk, ant, tyt, yac, tya, yat, anc, ang, cay, tkt, cak, cna, cng, rcc, akt, ggw, aty, cgn, gyt, tng, acw, aak, cyt, ytg, raa, yca, ntn, gay, gna, kat, kct, gnc, ngn, cty, tkg, tnc, yag, ayc, kca, aka, tyg, gka, ygt, cnc, cya, ttk, ayg, tka, gng, maa, tth, yga, ncn, ama, aar, kag, ytc, gtk, cch, ncg, kaa, ctk, gty, yct, ara, rtg, ckt, tar, gya, kac, tgr, crc, ccy, tkc, tak, akg, tyc, grt, gcy, trt, ggr, ygg, gak, gcr, kgt, ygc, cgk, wga, wtc, tma, tck, atr, waa, vcc, cwt, tcy, tgs, rgc, rac, agy, kgc, gam, haa, agr, rgt, tha, cyc, atk, gtr, tra, nam, gkg, gwg, cra)

vocabList.size

res17: Int = 241

// Create vector of token counts
val countVectors_train = vectorizer.transform(tokenized_df_train).select("id", "features")
val countVectors_test = vectorizer.transform(tokenized_df_test).select("id", "features")

countVectors_train: org.apache.spark.sql.DataFrame = [id: bigint, features: vector]
countVectors_test: org.apache.spark.sql.DataFrame = [id: bigint, features: vector]

import org.apache.spark.ml.linalg.{Vector => MLVector}
import org.apache.spark.mllib.{linalg => mllib}
import org.apache.spark.ml.{linalg => ml}

val lda_countVector_train = countVectors_train.map { case Row(id: Long, countVector: MLVector) => (id, mllib.Vectors.fromML(countVector)) }.cache()
val lda_countVector_test = countVectors_test.map { case Row(id: Long, countVector: MLVector) => (id, mllib.Vectors.fromML(countVector)) }.cache()

import org.apache.spark.ml.linalg.{Vector=>MLVector}
import org.apache.spark.mllib.{linalg=>mllib}
import org.apache.spark.ml.{linalg=>ml}
lda_countVector_train: org.apache.spark.sql.Dataset[(Long, org.apache.spark.mllib.linalg.Vector)] = [_1: bigint, _2: vector]
lda_countVector_test: org.apache.spark.sql.Dataset[(Long, org.apache.spark.mllib.linalg.Vector)] = [_1: bigint, _2: vector]

import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
import org.apache.spark.ml.linalg.{Vectors => NewVectors}


val lda_countVector_train_1 = lda_countVector_train.map({case (a,b) =>(a,b.asML)})
val lda_countVector_test_1 = lda_countVector_test.map({case (a,b) =>(a,b.asML)})

import org.apache.spark.mllib.linalg.{Vectors=>OldVectors}
import org.apache.spark.ml.linalg.{Vectors=>NewVectors}
lda_countVector_train_1: org.apache.spark.sql.Dataset[(Long, org.apache.spark.ml.linalg.Vector)] = [_1: bigint, _2: vector]
lda_countVector_test_1: org.apache.spark.sql.Dataset[(Long, org.apache.spark.ml.linalg.Vector)] = [_1: bigint, _2: vector]

val trainDF = lda_countVector_train_1.toDF()
val testDF = lda_countVector_test_1.toDF()

trainDF: org.apache.spark.sql.DataFrame = [_1: bigint, _2: vector]
testDF: org.apache.spark.sql.DataFrame = [_1: bigint, _2: vector]

spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1)

val mergedTrainingData = trainDF.join(k_mers_df_train,trainDF("_1") === k_mers_df_train("id"),"inner").withColumnRenamed("_2","features").drop("_1")
val mergedTestData = testDF.join(k_mers_df_test,testDF("_1") === k_mers_df_test("id"),"inner").withColumnRenamed("_2","features").drop("_1")

mergedTrainingData: org.apache.spark.sql.DataFrame = [features: vector, genome: string ... 2 more fields]
mergedTestData: org.apache.spark.sql.DataFrame = [features: vector, genome: string ... 2 more fields]

import org.apache.spark.ml.feature.{StringIndexer,VectorAssembler}
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.classification.RandomForestClassifier
import org.apache.spark.ml.linalg.Vector

val transformers = Array(
              new StringIndexer().setInputCol("label").setOutputCol("label_id"))

// Train a RandomForest model.
val rf = new RandomForestClassifier() 
              .setLabelCol("label_id")
              .setFeaturesCol("features")
              .setNumTrees(500)
              .setFeatureSubsetStrategy("auto")
              .setImpurity("gini")
              .setMaxDepth(20)
              .setMaxBins(32)
              .setSeed(12345)

val model = new Pipeline().setStages(transformers :+ rf).fit(mergedTrainingData)

import org.apache.spark.ml.feature.{StringIndexer, VectorAssembler}
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.classification.RandomForestClassifier
import org.apache.spark.ml.linalg.Vector
transformers: Array[org.apache.spark.ml.feature.StringIndexer] = Array(strIdx_e8cf65204547)
rf: org.apache.spark.ml.classification.RandomForestClassifier = rfc_0a45a46b7366
model: org.apache.spark.ml.PipelineModel = pipeline_14ad91398432

import org.apache.spark.mllib.evaluation.MulticlassMetrics

def evaluateModel(model: org.apache.spark.ml.PipelineModel, df: org.apache.spark.sql.DataFrame){
  val predictionsOnData = model.transform(df)
  val predictionAndLabelsRdd = predictionsOnData.select("prediction", "label_id").as[(Double,Double)].rdd

  val metricsMulti = new MulticlassMetrics(predictionAndLabelsRdd)
  val accuracy = metricsMulti.accuracy
  
  val fm0 = metricsMulti.fMeasure(0)
  val fm1 = metricsMulti.fMeasure(1)
  val fm2 = metricsMulti.fMeasure(2)
  val fm3 = metricsMulti.fMeasure(3)
  val fm4 = metricsMulti.fMeasure(4)
  val fm5 = metricsMulti.fMeasure(5)
  
  println("Confusion matrix:")
  println(metricsMulti.confusionMatrix)
  
  println("Summary Statistics")
  println(s"Accuracy = $accuracy")
  
  println(s"fm0 = $fm0")
  println(s"fm1 = $fm1")
  println(s"fm2 = $fm2")
  println(s"fm3 = $fm3")
  println(s"fm4 = $fm4")
  println(s"fm5 = $fm5")

}

evaluateModel(model, mergedTrainingData)

Confusion matrix:
12776.0  0.0     0.0     0.0    0.0    0.0   
70.0     6955.0  3.0     0.0    0.0    0.0   
139.0    1.0     1002.0  1.0    0.0    0.0   
133.0    0.0     4.0     756.0  0.0    0.0   
36.0     0.0     0.0     0.0    175.0  0.0   
33.0     0.0     0.0     0.0    0.0    71.0  
Summary Statistics
Accuracy = 0.981042654028436
fm0 = 0.9841697800716405
fm1 = 0.99470823798627
fm2 = 0.9312267657992566
fm3 = 0.9163636363636364
fm4 = 0.9067357512953368
fm5 = 0.8114285714285714

evaluateModel(model, mergedTestData)

Confusion matrix:
5450.0  8.0     6.0    4.0    0.0   0.0   
140.0   2743.0  8.0    0.0    0.0   0.0   
189.0   7.0     314.0  6.0    0.0   0.0   
116.0   1.0     7.0    269.0  0.0   0.0   
44.0    2.0     1.0    3.0    46.0  0.0   
9.0     0.0     0.0    0.0    0.0   22.0  
Summary Statistics
Accuracy = 0.9413517828632251
fm0 = 0.9548002803083393
fm1 = 0.9706298655343242
fm2 = 0.7370892018779341
fm3 = 0.7970370370370371
fm4 = 0.647887323943662
fm5 = 0.8301886792452831

//Load LDA model or the topic distributions
// data format: org.apache.spark.sql.DataFrame = [genome:string, label:string, id:long]
val k_mers_df_train = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_train_nonoverlapping")
val k_mers_df_test = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/data_test_nonoverlapping")

// data format: org.apache.spark.sql.DataFrame = [_1: bigint, _2: vector] the vector part contains the topic distributions
val trainingData = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/topic_dist_train_t20_i20k_no_cv")
val testData = spark.read.parquet("dbfs:/FileStore/shared_uploads/caylak@kth.se/topic_dist_test_t20_i20k_no_cv")

k_mers_df_train: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 1 more field]
k_mers_df_test: org.apache.spark.sql.DataFrame = [genome: string, label: string ... 1 more field]
trainingData: org.apache.spark.sql.DataFrame = [_1: bigint, _2: vector]
testData: org.apache.spark.sql.DataFrame = [_1: bigint, _2: vector]

//Merge data sources to get labels
spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1)
val mergedTrainingData = trainingData.join(k_mers_df_train,trainingData("_1") === k_mers_df_train("id"),"inner").withColumnRenamed("_2","features").drop("_1")
val mergedTestData = testData.join(k_mers_df_test,testData("_1") === k_mers_df_test("id"),"inner").withColumnRenamed("_2","features").drop("_1")

mergedTrainingData: org.apache.spark.sql.DataFrame = [features: vector, genome: string ... 2 more fields]
mergedTestData: org.apache.spark.sql.DataFrame = [features: vector, genome: string ... 2 more fields]

mergedTrainingData.show(5)

+--------------------+--------------------+-------+----+
|            features|              genome|  label|  id|
+--------------------+--------------------+-------+----+
|[0.05124403025197...|CTT GTA GAT CTG T...|oceania|  26|
|[0.04866065493877...|CTC TTG TAG ATC T...|oceania|  29|
|[0.04856023665158...|ACT TTC GAT CTC T...|oceania| 474|
|[0.05114514533282...|CTT GTA GAT CTG T...|oceania| 964|
|[0.04856846330546...|ACT TTC GAT CTC T...|oceania|1677|
+--------------------+--------------------+-------+----+
only showing top 5 rows

mergedTestData.show(5)

+--------------------+--------------------+-------+----+
|            features|              genome|  label|  id|
+--------------------+--------------------+-------+----+
|[0.04856691976914...|ACT TTC GAT CTC T...|oceania|2250|
|[0.04775129705622...|ACT TTC GAT CTC T...|oceania|3091|
|[0.04856692420144...|ACT TTC GAT CTC T...|oceania|7279|
|[0.04881282010425...|ACT TTC GAT CTC T...|oceania|8075|
|[0.05110516016513...|CTT GTA GAT CTG T...|oceania|9458|
+--------------------+--------------------+-------+----+
only showing top 5 rows

import org.apache.spark.sql.functions._
import org.apache.spark.ml._


//Split the feature vector into seprate columns
val vecToArray = udf( (xs: linalg.Vector) => xs.toArray )
val dfArr = mergedTrainingData.withColumn("featuresArr" , vecToArray($"features") )
val elements = Array("f1", "f2", "f3", "f4", "f5", "f6","f7", "f8", "f9","f10")
val sqlExpr = elements.zipWithIndex.map{ case (alias, idx) => col("featuresArr").getItem(idx).as(alias) }
val df_feats = dfArr.select((col("label") +: sqlExpr) : _*)

import org.apache.spark.sql.functions._
import org.apache.spark.ml._
vecToArray: org.apache.spark.sql.expressions.UserDefinedFunction = SparkUserDefinedFunction($Lambda$9301/1088827103@108f1ec1,ArrayType(DoubleType,false),List(Some(class[value[0]: vector])),None,true,true)
dfArr: org.apache.spark.sql.DataFrame = [features: vector, genome: string ... 3 more fields]
elements: Array[String] = Array(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10)
sqlExpr: Array[org.apache.spark.sql.Column] = Array(featuresArr[0] AS `f1`, featuresArr[1] AS `f2`, featuresArr[2] AS `f3`, featuresArr[3] AS `f4`, featuresArr[4] AS `f5`, featuresArr[5] AS `f6`, featuresArr[6] AS `f7`, featuresArr[7] AS `f8`, featuresArr[8] AS `f9`, featuresArr[9] AS `f10`)
df_feats: org.apache.spark.sql.DataFrame = [label: string, f1: double ... 9 more fields]

df_feats.describe().show()

+-------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+
|summary|       label|                  f1|                  f2|                  f3|                  f4|                  f5|                  f6|                  f7|                  f8|                  f9|                 f10|
+-------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+
|  count|       22155|               22155|               22155|               22155|               22155|               22155|               22155|               22155|               22155|               22155|               22155|
|   mean|        null| 0.04931437716125177| 0.05103219325119624| 0.05043777410343692| 0.04985720003755815| 0.05109828486193199|0.049969419011346175|0.049612622552246397| 0.05172608119915934| 0.05519918725594238|0.050902505322165705|
| stddev|        null| 0.00376757883941906|0.012025788212709033|0.008575417672641356|0.007216341887817029|   0.017036551424972|0.006744772303600655|0.006141190111960429|0.015925101732795318|0.028248525286887288|0.011750518257958392|
|    min|      africa|7.597652484573448E-5|7.931226826328689E-5|7.472127009799751E-5|8.488001247712824E-5|5.737788781311747E-5|7.510235436716417E-5| 8.01478429209781E-5|1.102376409869290...|7.060888276954563E-5|5.730971677795215...|
|    max|southamerica| 0.07096255125893133| 0.08483680076143339| 0.07401129608174387| 0.09129137902172121| 0.07569705042586143| 0.09892558758161725| 0.12033929830179531| 0.08811599095688616| 0.09753703174079206| 0.08152012057643862|
+-------+------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+

import org.apache.spark.sql.functions.rand
display(df_feats.sample(true,0.5).orderBy(rand()))

import org.apache.spark.ml.feature.{StringIndexer,VectorAssembler}
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.classification.RandomForestClassifier
import org.apache.spark.ml.linalg.Vector

val transformers = Array(
              new StringIndexer().setInputCol("label").setOutputCol("label_id"))

// Train a RandomForest model.
val rf = new RandomForestClassifier()
              .setLabelCol("label_id")
              .setFeaturesCol("features")
              .setNumTrees(500)
              .setFeatureSubsetStrategy("auto")
              .setImpurity("gini")
              .setMaxDepth(20)
              .setMaxBins(32)
              .setSeed(12345)

val model = new Pipeline().setStages(transformers :+ rf).fit(mergedTrainingData)

import org.apache.spark.ml.feature.{StringIndexer, VectorAssembler}
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.classification.RandomForestClassifier
import org.apache.spark.ml.linalg.Vector
transformers: Array[org.apache.spark.ml.feature.StringIndexer] = Array(strIdx_5c4c8beb7d03)
rf: org.apache.spark.ml.classification.RandomForestClassifier = rfc_c92cc57e8554
model: org.apache.spark.ml.PipelineModel = pipeline_fd80c4a01f49

import org.apache.spark.mllib.evaluation.MulticlassMetrics

def evaluateModel(model: org.apache.spark.ml.PipelineModel, df: org.apache.spark.sql.DataFrame){
  val predictionsOnData = model.transform(df)
  val predictionAndLabelsRdd = predictionsOnData.select("prediction", "label_id").as[(Double,Double)].rdd

  val metricsMulti = new MulticlassMetrics(predictionAndLabelsRdd)

  val accuracy = metricsMulti.accuracy
  val fm0 = metricsMulti.fMeasure(0)
  val fm1 = metricsMulti.fMeasure(1)
  val fm2 = metricsMulti.fMeasure(2)
  val fm3 = metricsMulti.fMeasure(3)
  val fm4 = metricsMulti.fMeasure(4)
  val fm5 = metricsMulti.fMeasure(5)

  println("Confusion matrix:")
  println(metricsMulti.confusionMatrix)

  println("Summary Statistics")
  println(s"Accuracy = $accuracy")

  println(s"fm0 = $fm0")
  println(s"fm1 = $fm1")
  println(s"fm2 = $fm2")
  println(s"fm3 = $fm3")
  println(s"fm4 = $fm4")
  println(s"fm5 = $fm5")

}

import org.apache.spark.mllib.evaluation.MulticlassMetrics
evaluateModel: (model: org.apache.spark.ml.PipelineModel, df: org.apache.spark.sql.DataFrame)Unit

evaluateModel(model, mergedTrainingData)

Confusion matrix:
12688.0  80.0    7.0     1.0    0.0    0.0   
137.0    6891.0  0.0     0.0    0.0    0.0   
53.0     18.0    1071.0  1.0    0.0    0.0   
47.0     3.0     0.0     843.0  0.0    0.0   
16.0     2.0     0.0     1.0    192.0  0.0   
19.0     0.0     0.0     0.0    0.0    85.0  
Summary Statistics
Accuracy = 0.9826224328593997
fm0 = 0.9860118122474355
fm1 = 0.9828840393667095
fm2 = 0.9644304367402071
fm3 = 0.9695227142035653
fm4 = 0.9528535980148882
fm5 = 0.8994708994708994

evaluateModel(model, mergedTestData)

Confusion matrix:
5373.0  89.0    3.0   3.0    0.0  0.0   
368.0   2522.0  0.0   1.0    0.0  0.0   
478.0   15.0    20.0  3.0    0.0  0.0   
193.0   8.0     0.0   192.0  0.0  0.0   
89.0    5.0     0.0   1.0    1.0  0.0   
14.0    0.0     0.0   0.0    0.0  17.0  
Summary Statistics
Accuracy = 0.8648217136774881
fm0 = 0.896770424768422
fm1 = 0.9121157323688969
fm2 = 0.07421150278293136
fm3 = 0.6475548060708264
fm4 = 0.020618556701030924
fm5 = 0.7083333333333333

val emoticonsList = List(
  "😀", "😁", "😂",	"😃", "😄",	"😅", "😆",	"😇", "😈",	"😉", "😊",	"😋", "😌", "😍", "😎", "😏",
  "😐",	"😑", "😒",	"😓", "😔", "😕", "😖", "😗", "😘", "😙", "😚", "😛", "😜", "😝", "😞", "😟",
  "😠",	"😡", "😢", "😣", "😤", "😥", "😦", "😧", "😨", "😩", "😪", "😫", "😬", "😭", "😮", "😯",
  "😰",	"😱", "😲", "😳", "😴", "😵", "😶", "😷", "😸", "😹", "😺", "😻", "😼", "😽", "😾", "😿",
  "🙀",	"🙁", "🙂", "🙃", "🙄", "🙅", "🙆", "🙇", "🙈", "🙉", "🙊", "🙋", "🙌", "🙍", "🙎", "🙏"
)

val emoticonsMap = emoticonsList.zipWithIndex.toMap
val nbrEmoticons = emoticonsList.length

val fullDF = sqlContext.read.parquet("/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/{2020,2021,continuous_22_12}/*/*/*/*/*")
println(fullDF.count)

2013137
fullDF: org.apache.spark.sql.DataFrame = [CurrentTweetDate: timestamp, CurrentTwID: bigint ... 7 more fields]

def emoticonPairToIndex (a : Int, b : Int) : Int = { // helper function for indexing
  val i = if (a < b) a else b // makes sure j >= i
  val j = if (a < b) b else a
  return i*nbrEmoticons - (i * (i+1))/2 + j
}

def createEmoticonIterator (s : String) : scala.util.matching.Regex.MatchIterator = { // helper function for iterating through the emoticons in a string s
  return s"""[${emoticonsList.mkString}]""".r.findAllIn(s)
}

def createEmoticonMatrix (s : String) : Array[Int] = { // The pair (i, j) will be at location i*(nbrEmoticons - (i+1)/2) + j in this array (this is compatible with later scipy functions)
  var m = Array.fill((nbrEmoticons*nbrEmoticons + nbrEmoticons)/2)(0) // there are 80 emoticons and thus 80^2 / 2 + 80 / 2 emoticon pairs including pairs of the same emoticon
  val emoticonIterator = createEmoticonIterator(s)
  // sets m to 1 for each index corresponding to a pair of emoticons present in the string s (very hacky code...)
  emoticonIterator.zipWithIndex.foreach(em_pair => // iterate over emoticons in s
                                        (createEmoticonIterator(s).drop(em_pair._2)).foreach( // iterate over remaining emoticons in s
                                          second_em =>
                                            (m(emoticonPairToIndex(
                                                emoticonsMap(em_pair._1),
                                                emoticonsMap(second_em))
                                              )
                                            = 1) // set m to 1 for each emoticon pair found in s
                                         )
                                       )
  return m
}

emoticonPairToIndex: (a: Int, b: Int)Int
createEmoticonIterator: (s: String)util.matching.Regex.MatchIterator
createEmoticonMatrix: (s: String)Array[Int]

val emoticonsMatrix = fullDF.select("CurrentTweet")
                            .filter($"CurrentTweet".rlike(emoticonsList.mkString("|"))) // filters tweets with emoticons
                            .map(row => createEmoticonMatrix(row.mkString)) // creates an "adjacency matrix" for each tweet
                            .reduce((_, _).zipped.map(_ + _)) // sums the matrices elementwise

emoticonsList.zipWithIndex.foreach({case (e, i) => println(e + ", " + Integer.toString(emoticonsMatrix(emoticonPairToIndex(i, i))) + " occurences")})

def jaccardDistance (e1 : Int, e2 : Int) : Double = { // specify the emojis in terms of their indices in the list
  return 1.0 - 1.0 * emoticonsMatrix(emoticonPairToIndex(e1, e2)) /
    (emoticonsMatrix(emoticonPairToIndex(e1, e1)) +  emoticonsMatrix(emoticonPairToIndex(e2, e2)) - emoticonsMatrix(emoticonPairToIndex(e1, e2)))
}

jaccardDistance: (e1: Int, e2: Int)Double

var jaccardMatrix = Array.fill(emoticonsMatrix.length)(1.0)
(0 until nbrEmoticons).foreach(i => (i until nbrEmoticons).foreach(j => (jaccardMatrix(emoticonPairToIndex(i, j)) = jaccardDistance(i, j))))

//scala.tools.nsc.io.File("/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/emoticonsList.txt").writeAll(emoticonsList.mkString("\n"))
//scala.tools.nsc.io.File("/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/jMatrix.txt").writeAll(jaccardMatrix.mkString("\n"))

%pip install pycountry

Python interpreter will be restarted.
Collecting pycountry
  Downloading pycountry-20.7.3.tar.gz (10.1 MB)
Building wheels for collected packages: pycountry
  Building wheel for pycountry (setup.py): started
  Building wheel for pycountry (setup.py): finished with status 'done'
  Created wheel for pycountry: filename=pycountry-20.7.3-py2.py3-none-any.whl size=10746863 sha256=870cc02de7d6d11499effd59f2b73acdbffc04146079c4a4f9a841ca30f2b587
  Stored in directory: /root/.cache/pip/wheels/57/e8/3f/120ccc1ff7541c108bc5d656e2a14c39da0d824653b62284c6
Successfully built pycountry
Installing collected packages: pycountry
Successfully installed pycountry-20.7.3
Python interpreter will be restarted.

import json
import os
from matplotlib import font_manager as fm, pyplot as plt, rcParams
import numpy as np
import pandas as pd
import pycountry
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.manifold import locally_linear_embedding, TSNE
from sklearn.neighbors import NearestNeighbors

jMatrix = np.loadtxt("/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/jMatrix.txt")

emoticonsList = []

with open("/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/emoticonsList.txt", 'r') as filehandle:
    for line in filehandle:
        e = line.strip() #remove line break
        emoticonsList.append(e)

nbrEmoticons = len(emoticonsList)
print(emoticonsList)

['😀', '😁', '😂', '😃', '😄', '😅', '😆', '😇', '😈', '😉', '😊', '😋', '😌', '😍', '😎', '😏', '😐', '😑', '😒', '😓', '😔', '😕', '😖', '😗', '😘', '😙', '😚', '😛', '😜', '😝', '😞', '😟', '😠', '😡', '😢', '😣', '😤', '😥', '😦', '😧', '😨', '😩', '😪', '😫', '😬', '😭', '😮', '😯', '😰', '😱', '😲', '😳', '😴', '😵', '😶', '😷', '😸', '😹', '😺', '😻', '😼', '😽', '😾', '😿', '🙀', '🙁', '🙂', '🙃', '🙄', '🙅', '🙆', '🙇', '🙈', '🙉', '🙊', '🙋', '🙌', '🙍', '🙎', '🙏']

def emoticonPairToIndex(a, b): # same helper function as already defined in scala previously
  i = min(a, b) # makes sure j >= i
  j = max(a, b)
  return i * nbrEmoticons - (i * (i+1))//2 + j

fullDistanceMatrix = np.zeros([nbrEmoticons, nbrEmoticons])
for r in range(nbrEmoticons):
  for c in range(nbrEmoticons):
    fullDistanceMatrix[r, c] = jMatrix[emoticonPairToIndex(r, c)]

def scatterEmojis(emoticonsEmbedded):
  # This function plots a scatter plot of emoticons.
  # emoticonsEmbedded should be an 80x2 array
  # containing 2D coordinates for each of the
  # 80 emoticons in the unicode emoticon block (in the correct order).

  # standardize the embedding for nicer plotting:
  emoticonsEmbedded = emoticonsEmbedded - np.mean(emoticonsEmbedded)
  emoticonsEmbedded = emoticonsEmbedded/np.std(emoticonsEmbedded)

  # for proper emoji rendering change the font
  fpath = os.path.join(rcParams["datapath"], "/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/unifont_upper-13.0.05.ttf")
  prop = fm.FontProperties(fname=fpath, size=50)

  fig = plt.figure(figsize=(14, 14))
  for i, label in enumerate(emoticonsList):
      plt.text(emoticonsEmbedded[i, 0], emoticonsEmbedded[i, 1], label, fontproperties=prop)
  plt.setp(plt.gca(), frame_on=False, xticks=(), yticks=())

nbrNeighbors = 8
emoticonsNeighbors = NearestNeighbors(n_neighbors=nbrNeighbors, metric="precomputed").fit(fullDistanceMatrix)
emoticonsEmbedded, err = locally_linear_embedding(emoticonsNeighbors, n_neighbors=nbrNeighbors, n_components=2)

scatterEmojis(emoticonsEmbedded)

emoticonsEmbedded = TSNE(n_components=2, perplexity=20.0, early_exaggeration=12.0, learning_rate=2.0, n_iter=10000,
                         metric='precomputed', angle=0.01).fit_transform(fullDistanceMatrix)

scatterEmojis(emoticonsEmbedded)

# remove diagonal from jMatrix, as this is expected by the scipy linkage function:
diagonalIndices = [emoticonPairToIndex(i, i) for i in range(nbrEmoticons)]
jMatrixUpper = jMatrix[[i for i in range((nbrEmoticons**2 + nbrEmoticons)//2) if not i in diagonalIndices]]
assert len(jMatrixUpper) == len(jMatrix) - nbrEmoticons, "the upper matrix should have exactly 80 elements fewer than the upper+diagonal"

# creating a linkage matrix
Z = linkage(jMatrixUpper, 'complete', optimal_ordering=True)

# plotting a dendrogram
fig = plt.figure(figsize=(40, 8))
dn = dendrogram(Z, labels=emoticonsList, leaf_rotation=0, color_threshold=1.)
ax = plt.gca()

# for proper emoji rendering change the font
fpath = os.path.join(rcParams["datapath"], "/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/unifont_upper-13.0.05.ttf")
prop = fm.FontProperties(fname=fpath, size=28)
x_labels = ax.get_xmajorticklabels()
for x in x_labels:
    x.set_fontproperties(prop)

ax.set_ylim([.85, 1.01])

monkeyEmoticons = dn["leaves"][76:79]
prayerEmoticons = dn["leaves"][0:6]
shockedAndKissesEmoticons = dn["leaves"][38:46]
happyEmoticons = dn["leaves"][9:29]
notHappyEmoticons = dn["leaves"][46:76]
catEmoticons = dn["leaves"][29:38]
emoticonsDict = {"monkey" : monkeyEmoticons,
                 "prayer" : prayerEmoticons,
                 "SK" : shockedAndKissesEmoticons,
                "happy" : happyEmoticons,
                "notHappy" : notHappyEmoticons,
                "cat" : catEmoticons}
print(emoticonsDict)

{'monkey': [72, 74, 73], 'prayer': [76, 79, 10, 71, 75, 55], 'SK': [54, 44, 50, 46, 47, 23, 25, 26], 'happy': [29, 27, 28, 14, 9, 15, 8, 12, 11, 13, 24, 7, 0, 3, 4, 1, 6, 5, 66, 67], 'notHappy': [53, 39, 38, 40, 48, 49, 51, 2, 45, 41, 20, 34, 30, 37, 19, 42, 52, 43, 22, 35, 31, 65, 21, 16, 17, 18, 68, 36, 33, 32], 'cat': [57, 56, 61, 58, 59, 63, 62, 64, 60]}

#with open('/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/emoticonClusters.json', 'w+') as f:
#    json.dump(emoticonsDict, f)

import org.json4s.jackson.JsonMethods.parse
val jsonString = scala.io.Source.fromFile("/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/emoticonClusters.json").mkString
val emoticonClusters = parse(jsonString).values.asInstanceOf[Map[String, List[BigInt]]]
emoticonClusters.foreach({case (key, list) => println(key + ": " + list.map(i => emoticonsList(i.toInt)).mkString)})

prayer: 🙌🙏😊🙇🙋😷
monkey: 🙈🙊🙉
happy: 😝😛😜😎😉😏😈😌😋😍😘😇😀😃😄😁😆😅🙂🙃
SK: 😶😬😲😮😯😗😙😚
cat: 😹😸😽😺😻😿😾🙀😼
notHappy: 😵😧😦😨😰😱😳😂😭😩😔😢😞😥😓😪😴😫😖😣😟🙁😕😐😑😒🙄😤😡😠
import org.json4s.jackson.JsonMethods.parse
jsonString: String = {"monkey": [72, 74, 73], "prayer": [76, 79, 10, 71, 75, 55], "SK": [54, 44, 50, 46, 47, 23, 25, 26], "happy": [29, 27, 28, 14, 9, 15, 8, 12, 11, 13, 24, 7, 0, 3, 4, 1, 6, 5, 66, 67], "notHappy": [53, 39, 38, 40, 48, 49, 51, 2, 45, 41, 20, 34, 30, 37, 19, 42, 52, 43, 22, 35, 31, 65, 21, 16, 17, 18, 68, 36, 33, 32], "cat": [57, 56, 61, 58, 59, 63, 62, 64, 60]}
emoticonClusters: Map[String,List[BigInt]] = Map(prayer -> List(76, 79, 10, 71, 75, 55), monkey -> List(72, 74, 73), happy -> List(29, 27, 28, 14, 9, 15, 8, 12, 11, 13, 24, 7, 0, 3, 4, 1, 6, 5, 66, 67), SK -> List(54, 44, 50, 46, 47, 23, 25, 26), cat -> List(57, 56, 61, 58, 59, 63, 62, 64, 60), notHappy -> List(53, 39, 38, 40, 48, 49, 51, 2, 45, 41, 20, 34, 30, 37, 19, 42, 52, 43, 22, 35, 31, 65, 21, 16, 17, 18, 68, 36, 33, 32))

val emoticonDF = fullDF.filter($"CurrentTweet".rlike(emoticonsList.mkString("|"))) // filter tweets with emoticons
                       .select(($"countryCode" :: // select the countryCode column
                                $"CurrentTweetDate" :: // and the timestamp
                                (for {(name, cluster) <- emoticonClusters.toList} yield  // also create a new column for each emoticon cluster indicating if the tweet contains an emoticon of that cluster
                                 $"CurrentTweet".rlike(cluster.map(i => emoticonsList(i.toInt)).mkString("|"))
                                                .alias(name))) // rename new column
                                                : _*) // expand list

emoticonDF.show(3)

+-----------+-------------------+------+------+-----+-----+-----+--------+
|countryCode|   CurrentTweetDate|prayer|monkey|happy|   SK|  cat|notHappy|
+-----------+-------------------+------+------+-----+-----+-----+--------+
|         EG|2020-12-31 15:59:54| false| false|false|false|false|    true|
|         SA|2020-12-31 15:59:54|  true| false|false|false|false|   false|
|         DO|2020-12-31 15:59:54| false| false| true|false|false|   false|
+-----------+-------------------+------+------+-----+-----+-----+--------+
only showing top 3 rows

emoticonDF: org.apache.spark.sql.DataFrame = [countryCode: string, CurrentTweetDate: timestamp ... 6 more fields]

// save to file
// emoticonDF.write.format("parquet").mode("overwrite").save("/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/processedEmoticonClusterParquets/emoticonCluster.parquet")

val emoticonCCDF = emoticonDF.groupBy($"countryCode")
                             .count
emoticonCCDF.show(3)

+-----------+-----+
|countryCode|count|
+-----------+-----+
|         DZ|  157|
|         MM|   44|
|         TC|    5|
+-----------+-----+
only showing top 3 rows

emoticonCCDF: org.apache.spark.sql.DataFrame = [countryCode: string, count: bigint]

def createPropClusterDF (cluster : org.apache.spark.sql.Column) : org.apache.spark.sql.DataFrame = {  
  // This function filters the emoticonDF by a cluster-column and then
  // creates a dataframe with a row per country and columns for the countryCode and proportion
  // of tweets from that country that fall into the cluster as well as the count of tweets
  // falling into the cluster.
  val nbrClusterTweets = emoticonDF.filter(cluster).count
  val clusterDF = emoticonDF.filter(cluster)
                            .groupBy($"countryCode")
                            .count
  val propDF = emoticonCCDF.alias("total")
                           .join(clusterDF.alias("cluster"), "countryCode")
                           .select($"countryCode", $"cluster.count".alias("count"), ($"cluster.count" / $"total.count").alias("proportion"))
  return propDF
}

createPropClusterDF: (cluster: org.apache.spark.sql.Column)org.apache.spark.sql.DataFrame

val clusterColumn = $"notHappy"
val propClusterDF = createPropClusterDF(clusterColumn)
propClusterDF.show(3)

+-----------+-----+------------------+
|countryCode|count|        proportion|
+-----------+-----+------------------+
|         DZ|   80|0.5095541401273885|
|         MM|   14|0.3181818181818182|
|         CI|  164|0.6507936507936508|
+-----------+-----+------------------+
only showing top 3 rows

clusterColumn: org.apache.spark.sql.ColumnName = notHappy
propClusterDF: org.apache.spark.sql.DataFrame = [countryCode: string, count: bigint ... 1 more field]

def createPropClusterDFAndCreateTmpView (clusterName : String) = {
  // function for creating proportion dataframes for each cluster and making them available for later python code
  val propClusterDF = createPropClusterDF(org.apache.spark.sql.functions.col(clusterName))
  // make df available to python/sql etc
  propClusterDF.createOrReplaceTempView(clusterName)
}

createPropClusterDFAndCreateTmpView: (clusterName: String)Unit

// create proportion dataframes for each cluster and make them available for later python code
emoticonClusters.keys.foreach(createPropClusterDFAndCreateTmpView _)

def add_iso_a3_col(df_cc):
  cc_dict = {}
  for country in pycountry.countries:
    cc_dict[country.alpha_2] = country.alpha_3

    df_cc["iso_a3"] = df_cc["countryCode"].map(cc_dict)
  return df_cc

def cc_df_from_spark_to_pandas_and_process(df_cc, columnOfInterest):
  #df_cc should be a dataframe with a column "countryCode" and a column columnOfInterest which has some interesting numerical data
  df_cc = df_cc.toPandas()
  add_iso_a3_col(df_cc)
  df_cc = df_cc[["iso_a3", columnOfInterest]]  #reorder to have iso_a3 as first column (required in order to use the map view in display), and select the useful columns
  return df_cc

from pyspark.sql.functions import col
def createProps(clusterName):
  df = sql("select * from " + clusterName)
  return cc_df_from_spark_to_pandas_and_process(df.filter(col("count")/col("proportion") >= 100), "proportion") # filter so that only countries with at least 100 tweets in a given country are used

display(createProps("happy"))

display(createProps("notHappy"))

display(createProps("monkey"))

display(createProps("cat"))

display(createProps("SK"))

display(createProps("prayer"))

sudo apt-get -y install libproj-dev
sudo apt-get -y install libgeos++-dev

Reading package lists...
Building dependency tree...
Reading state information...
libproj-dev is already the newest version (4.9.3-2).
The following packages were automatically installed and are no longer required:
  libcap2-bin libpam-cap zulu-repo
Use 'sudo apt autoremove' to remove them.
0 upgraded, 0 newly installed, 0 to remove and 17 not upgraded.
Reading package lists...
Building dependency tree...
Reading state information...
libgeos++-dev is already the newest version (3.6.2-1build2).
The following packages were automatically installed and are no longer required:
  libcap2-bin libpam-cap zulu-repo
Use 'sudo apt autoremove' to remove them.
0 upgraded, 0 newly installed, 0 to remove and 17 not upgraded.

%pip install plotly
%pip install pycountry
%pip install geopandas
%pip install geoplot
%pip install imageio

Python interpreter will be restarted.
Collecting plotly
  Downloading plotly-4.14.3-py2.py3-none-any.whl (13.2 MB)
Requirement already satisfied: six in /databricks/python3/lib/python3.7/site-packages (from plotly) (1.14.0)
Collecting retrying>=1.3.3
  Downloading retrying-1.3.3.tar.gz (10 kB)
Building wheels for collected packages: retrying
  Building wheel for retrying (setup.py): started
  Building wheel for retrying (setup.py): finished with status 'done'
  Created wheel for retrying: filename=retrying-1.3.3-py3-none-any.whl size=11430 sha256=d44981ab5b644b464561fdd8ffbda5b3b476a52efe985dab72bd1549d6a6d1f0
  Stored in directory: /root/.cache/pip/wheels/f9/8d/8d/f6af3f7f9eea3553bc2fe6d53e4b287dad18b06a861ac56ddf
Successfully built retrying
Installing collected packages: retrying, plotly
Successfully installed plotly-4.14.3 retrying-1.3.3
Python interpreter will be restarted.
Python interpreter will be restarted.
Collecting pycountry
  Downloading pycountry-20.7.3.tar.gz (10.1 MB)
Building wheels for collected packages: pycountry
  Building wheel for pycountry (setup.py): started
  Building wheel for pycountry (setup.py): finished with status 'done'
  Created wheel for pycountry: filename=pycountry-20.7.3-py2.py3-none-any.whl size=10746863 sha256=2b60c51268a303df86bea421d372ba1ed4001f8ffa211a0d9e98601fa0ab1ed5
  Stored in directory: /root/.cache/pip/wheels/57/e8/3f/120ccc1ff7541c108bc5d656e2a14c39da0d824653b62284c6
Successfully built pycountry
Installing collected packages: pycountry
Successfully installed pycountry-20.7.3
Python interpreter will be restarted.
Python interpreter will be restarted.
Collecting geopandas
  Downloading geopandas-0.8.2-py2.py3-none-any.whl (962 kB)
Requirement already satisfied: pandas>=0.23.0 in /databricks/python3/lib/python3.7/site-packages (from geopandas) (1.0.1)
Collecting pyproj>=2.2.0
  Downloading pyproj-3.0.0.post1-cp37-cp37m-manylinux2010_x86_64.whl (6.4 MB)
Collecting fiona
  Downloading Fiona-1.8.18-cp37-cp37m-manylinux1_x86_64.whl (14.8 MB)
Collecting shapely
  Downloading Shapely-1.7.1-cp37-cp37m-manylinux1_x86_64.whl (1.0 MB)
Requirement already satisfied: python-dateutil>=2.6.1 in /databricks/python3/lib/python3.7/site-packages (from pandas>=0.23.0->geopandas) (2.8.1)
Requirement already satisfied: numpy>=1.13.3 in /databricks/python3/lib/python3.7/site-packages (from pandas>=0.23.0->geopandas) (1.18.1)
Requirement already satisfied: pytz>=2017.2 in /databricks/python3/lib/python3.7/site-packages (from pandas>=0.23.0->geopandas) (2019.3)
Requirement already satisfied: certifi in /databricks/python3/lib/python3.7/site-packages (from pyproj>=2.2.0->geopandas) (2020.6.20)
Requirement already satisfied: six>=1.5 in /databricks/python3/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas>=0.23.0->geopandas) (1.14.0)
Collecting munch
  Downloading munch-2.5.0-py2.py3-none-any.whl (10 kB)
Collecting click-plugins>=1.0
  Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)
Collecting attrs>=17
  Downloading attrs-20.3.0-py2.py3-none-any.whl (49 kB)
Collecting cligj>=0.5
  Downloading cligj-0.7.1-py3-none-any.whl (7.1 kB)
Collecting click<8,>=4.0
  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)
Installing collected packages: click, munch, cligj, click-plugins, attrs, shapely, pyproj, fiona, geopandas
Successfully installed attrs-20.3.0 click-7.1.2 click-plugins-1.1.1 cligj-0.7.1 fiona-1.8.18 geopandas-0.8.2 munch-2.5.0 pyproj-3.0.0.post1 shapely-1.7.1
Python interpreter will be restarted.
Python interpreter will be restarted.
Collecting geoplot
  Downloading geoplot-0.4.1-py3-none-any.whl (28 kB)
Requirement already satisfied: seaborn in /databricks/python3/lib/python3.7/site-packages (from geoplot) (0.10.0)
Requirement already satisfied: matplotlib in /databricks/python3/lib/python3.7/site-packages (from geoplot) (3.1.3)
Collecting mapclassify>=2.1
  Downloading mapclassify-2.4.2-py3-none-any.whl (38 kB)
Collecting contextily>=1.0.0
  Downloading contextily-1.0.1-py3-none-any.whl (23 kB)
Collecting cartopy
  Downloading Cartopy-0.18.0.tar.gz (14.4 MB)
Requirement already satisfied: pandas in /databricks/python3/lib/python3.7/site-packages (from geoplot) (1.0.1)
Requirement already satisfied: geopandas in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from geoplot) (0.8.2)
Collecting descartes
  Downloading descartes-1.1.0-py3-none-any.whl (5.8 kB)
Collecting mercantile
  Downloading mercantile-1.1.6-py3-none-any.whl (13 kB)
Collecting pillow
  Downloading Pillow-8.1.0-cp37-cp37m-manylinux1_x86_64.whl (2.2 MB)
Collecting geopy
  Downloading geopy-2.1.0-py3-none-any.whl (112 kB)
Requirement already satisfied: joblib in /databricks/python3/lib/python3.7/site-packages (from contextily>=1.0.0->geoplot) (0.14.1)
Collecting rasterio
  Downloading rasterio-1.2.0-cp37-cp37m-manylinux1_x86_64.whl (19.1 MB)
Requirement already satisfied: requests in /databricks/python3/lib/python3.7/site-packages (from contextily>=1.0.0->geoplot) (2.22.0)
Requirement already satisfied: scipy>=1.0 in /databricks/python3/lib/python3.7/site-packages (from mapclassify>=2.1->geoplot) (1.4.1)
Requirement already satisfied: numpy>=1.3 in /databricks/python3/lib/python3.7/site-packages (from mapclassify>=2.1->geoplot) (1.18.1)
Requirement already satisfied: scikit-learn in /databricks/python3/lib/python3.7/site-packages (from mapclassify>=2.1->geoplot) (0.22.1)
Collecting networkx
  Downloading networkx-2.5-py3-none-any.whl (1.6 MB)
Requirement already satisfied: python-dateutil>=2.6.1 in /databricks/python3/lib/python3.7/site-packages (from pandas->geoplot) (2.8.1)
Requirement already satisfied: pytz>=2017.2 in /databricks/python3/lib/python3.7/site-packages (from pandas->geoplot) (2019.3)
Requirement already satisfied: six>=1.5 in /databricks/python3/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas->geoplot) (1.14.0)
Requirement already satisfied: shapely>=1.5.6 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from cartopy->geoplot) (1.7.1)
Collecting pyshp>=1.1.4
  Downloading pyshp-2.1.3.tar.gz (219 kB)
Requirement already satisfied: setuptools>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from cartopy->geoplot) (45.2.0)
Requirement already satisfied: pyproj>=2.2.0 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from geopandas->geoplot) (3.0.0.post1)
Requirement already satisfied: fiona in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from geopandas->geoplot) (1.8.18)
Requirement already satisfied: certifi in /databricks/python3/lib/python3.7/site-packages (from pyproj>=2.2.0->geopandas->geoplot) (2020.6.20)
Requirement already satisfied: munch in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from fiona->geopandas->geoplot) (2.5.0)
Requirement already satisfied: click-plugins>=1.0 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from fiona->geopandas->geoplot) (1.1.1)
Requirement already satisfied: attrs>=17 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from fiona->geopandas->geoplot) (20.3.0)
Requirement already satisfied: cligj>=0.5 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from fiona->geopandas->geoplot) (0.7.1)
Requirement already satisfied: click<8,>=4.0 in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from fiona->geopandas->geoplot) (7.1.2)
Collecting geographiclib<2,>=1.49
  Downloading geographiclib-1.50-py3-none-any.whl (38 kB)
Requirement already satisfied: cycler>=0.10 in /databricks/python3/lib/python3.7/site-packages (from matplotlib->geoplot) (0.10.0)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /databricks/python3/lib/python3.7/site-packages (from matplotlib->geoplot) (2.4.6)
Requirement already satisfied: kiwisolver>=1.0.1 in /databricks/python3/lib/python3.7/site-packages (from matplotlib->geoplot) (1.1.0)
Requirement already satisfied: decorator>=4.3.0 in /databricks/python3/lib/python3.7/site-packages (from networkx->mapclassify>=2.1->geoplot) (4.4.1)
Collecting affine
  Downloading affine-2.3.0-py2.py3-none-any.whl (15 kB)
Collecting snuggs>=1.4.1
  Downloading snuggs-1.4.7-py3-none-any.whl (5.4 kB)
Requirement already satisfied: idna<2.9,>=2.5 in /databricks/python3/lib/python3.7/site-packages (from requests->contextily>=1.0.0->geoplot) (2.8)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /databricks/python3/lib/python3.7/site-packages (from requests->contextily>=1.0.0->geoplot) (1.25.8)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/lib/python3/dist-packages (from requests->contextily>=1.0.0->geoplot) (3.0.4)
Building wheels for collected packages: cartopy, pyshp
  Building wheel for cartopy (setup.py): started
  Building wheel for cartopy (setup.py): finished with status 'done'
  Created wheel for cartopy: filename=Cartopy-0.18.0-cp37-cp37m-linux_x86_64.whl size=15127828 sha256=a7843bd96a5753ca15c4c1952b65a93a5ae9378f67d5c17cf1398436b77dfb80
  Stored in directory: /root/.cache/pip/wheels/0b/a9/54/172056df34478378e0636d30cd4d1a868de00e37254649bf1a
  Building wheel for pyshp (setup.py): started
  Building wheel for pyshp (setup.py): finished with status 'done'
  Created wheel for pyshp: filename=pyshp-2.1.3-py3-none-any.whl size=37262 sha256=f31c01d2d2d597d89e643a155abbf55b7de6db2c096a5aa21a9d267d548ed9b3
  Stored in directory: /root/.cache/pip/wheels/43/f8/87/53c8cd41545ba20e536ea29a8fcb5431b5f477ca50d5dffbbe
Successfully built cartopy pyshp
Installing collected packages: snuggs, geographiclib, affine, rasterio, pyshp, pillow, networkx, mercantile, geopy, mapclassify, descartes, contextily, cartopy, geoplot
Successfully installed affine-2.3.0 cartopy-0.18.0 contextily-1.0.1 descartes-1.1.0 geographiclib-1.50 geoplot-0.4.1 geopy-2.1.0 mapclassify-2.4.2 mercantile-1.1.6 networkx-2.5 pillow-8.1.0 pyshp-2.1.3 rasterio-1.2.0 snuggs-1.4.7
Python interpreter will be restarted.
Python interpreter will be restarted.
Collecting imageio
  Downloading imageio-2.9.0-py3-none-any.whl (3.3 MB)
Requirement already satisfied: numpy in /databricks/python3/lib/python3.7/site-packages (from imageio) (1.18.1)
Requirement already satisfied: pillow in /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages (from imageio) (8.1.0)
Installing collected packages: imageio
Successfully installed imageio-2.9.0
Python interpreter will be restarted.

git clone https://github.com/mthh/cartogram_geopandas.git
cd cartogram_geopandas/
python setup.py install

Cloning into 'cartogram_geopandas'...
Compiling cycartogram.pyx because it changed.
[1/1] Cythonizing cycartogram.pyx
running install
running build
running build_py
creating build
creating build/lib.linux-x86_64-3.7
copying cartogram_geopandas.py -> build/lib.linux-x86_64-3.7
running build_ext
building 'cycartogram' extension
creating build/temp.linux-x86_64-3.7
x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.7m -I/local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/include/python3.7m -c cycartogram.c -o build/temp.linux-x86_64-3.7/cycartogram.o
cycartogram.c: In function ‘__pyx_f_11cycartogram_transform_geom’:
cycartogram.c:2084:39: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
       for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) {
                                       ^
cycartogram.c:2134:41: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
         for (__pyx_t_21 = 0; __pyx_t_21 < __pyx_t_20; __pyx_t_21+=1) {
                                         ^
cycartogram.c: In function ‘__pyx_pf_11cycartogram_9Cartogram_4cartogram’:
cycartogram.c:3835:37: warning: comparison between signed and unsigned integer expressions [-Wsign-compare]
     for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_12; __pyx_t_13+=1) {
                                     ^
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/cycartogram.o -o build/lib.linux-x86_64-3.7/cycartogram.cpython-37m-x86_64-linux-gnu.so
running install_lib
copying build/lib.linux-x86_64-3.7/cycartogram.cpython-37m-x86_64-linux-gnu.so -> /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages
copying build/lib.linux-x86_64-3.7/cartogram_geopandas.py -> /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages
byte-compiling /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages/cartogram_geopandas.py to cartogram_geopandas.cpython-37.pyc
running install_egg_info
Writing /local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages/cartogram_geopandas-0.0.0c.egg-info
/databricks/python/lib/python3.7/site-packages/Cython/Compiler/Main.py:369: FutureWarning: Cython directive 'language_level' not set, using 2 for now (Py2). This will change in a later release! File: /databricks/driver/cartogram_geopandas/cycartogram.pyx
  tree = Parsing.p_module(s, pxd, full_module_name)

"./06_appendix_tweet_carto_functions"

path = "/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/{2020,2021}/*/*/*/*/*"
df = load_twitter_geo_data(path)
#Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour.
df = df[(df['day'] != 22) & (df['day'] != 2 )].reset_index() #Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour. Data collection ended in the middle of Jan second so to only have full day we disregard Jan 2.
pre_proc_path = "/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/tmp/processedDF.csv"
df.to_csv(pre_proc_path)

# Load the data
pre_proc_path = "/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/tmp/processedDF.csv"
df = pd.read_csv(pre_proc_path)
display(df.head(5))

display(df.query("countryCode=='GB'"))

display(df.query("countryCode=='SA'"))

dateList = ["2020-12-23", "2020-12-24", "2020-12-25", "2020-12-26", "2020-12-27", "2020-12-28", "2020-12-29", "2020-12-30", "2020-12-31", "2021-01-01"]
for d in dateList:
  N = len(df.query("(countryCode=='SA') and (date=='%s')"%d))
  print("%d tweets were collected in Saudi Arabia on %s"%(N, d))

1519 tweets were collected in Saudi Arabia on 2020-12-23
1437 tweets were collected in Saudi Arabia on 2020-12-24
1556 tweets were collected in Saudi Arabia on 2020-12-25
1871 tweets were collected in Saudi Arabia on 2020-12-26
1529 tweets were collected in Saudi Arabia on 2020-12-27
3123 tweets were collected in Saudi Arabia on 2020-12-28
1700 tweets were collected in Saudi Arabia on 2020-12-29
1643 tweets were collected in Saudi Arabia on 2020-12-30
3127 tweets were collected in Saudi Arabia on 2020-12-31
1486 tweets were collected in Saudi Arabia on 2021-01-01

# Group by country code
df_cc = country_code_grouping(df)
df_cc = add_iso_a3_col(df_cc)
df_cc = df_cc[["iso_a3", "count"]] #reorder to have iso_a3 as first column (required in order to use the map view in display). Also we don't need countryCode and index columns.

# Inspect result
display(df_cc)

# Create the geopandas dataframe
df_world = create_geo_df(df_cc)
# Inspect result
display(df_world.head())

df = pd.read_csv(pre_proc_path)
df = df.query("(day==31) and (hour==23)")
# Group by country code
df_cc = country_code_grouping(df)
df_cc = add_iso_a3_col(df_cc)
df_cc = df_cc[["iso_a3", "count"]] #reorder to have iso_a3 as first column (required in order to use the map view in display). Also we don't need countryCode and index columns.
# Create the geopandas dataframe
df_world = create_geo_df(df_cc)


#fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(30,15))

# Make a choropleth plot
# df_world.plot(column='numTweets', cmap='viridis', ax=axes[0])

# The make_cartogram function can not handle a tweetcount of zero, so a not so elegant solution is to clip the tweet count at 1.
# The alternative (to remove countries without tweets) is not elegant either (and causes problems when we look at the time evolution, since countries will be popping in and out of existence).
df_world["numTweets"] = df_world["numTweets"].clip(1, max(df_world["numTweets"]))

df_cartogram = make_cartogram(df_world, 'numTweets', 5, inplace=False)
# df_cartogram.plot(column='numTweets', cmap='viridis', ax=axes[1])


# plt.show()

/local_disk0/.ephemeral_nfs/envs/pythonEnv-318e9509-84da-4944-ac59-77216123051e/lib/python3.7/site-packages/cartogram_geopandas.py:48: UserWarning: Geometry is in a geographic CRS. Results from 'area' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.

  return crtgm.make()

pre_proc_path = "/dbfs/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/tmp/processedDF.csv"
df = pd.read_csv(pre_proc_path)
key = "hour"
timeOfDayList = list(range(0, 24))
nIter = 5
cartogram_key = "numTweets"
for day in range(23,32):
  out_path = "/dbfs/FileStore/group10/cartogram/2020_Dec_%d_hour_"%day
  legend = "2020 December %d: "%day
  animate_cartogram(df.query("(day==%d) and (year==2020)"%day), key, timeOfDayList, out_path, nIter, cartogram_key, legend)
for day in range(1,2):
  out_path = "/dbfs/FileStore/group10/cartogram/2021_Jan_%d_hour_"%day
  legend = "2021 January %d: "%day
  animate_cartogram(df.query("(day==%d) and (year==2021)"%day), key, timeOfDayList, out_path, nIter, cartogram_key, legend)

images=[] # array for storing the png frames for the gif
timeOfDayList = list(range(0, 24))
#Append images to image list
for day in range(23,32):
  for hour in timeOfDayList:
    out_path = "/dbfs/FileStore/group10/cartogram/2020_Dec_%d_hour_%d.png"%(day, hour)
    images.append(imageio.imread(out_path))
for day in range(1,2):
  for hour in timeOfDayList:
    out_path = "/dbfs/FileStore/group10/cartogram/2021_Jan_%d_hour_%d.png"%(day, hour)
    images.append(imageio.imread(out_path))
#create gif from the image list
imageio.mimsave("/dbfs/FileStore/group10/cartogram/many_days_cartogram.gif", images, duration=0.2)

path = "/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/processedEmoticonClusterParquets/emoticonCluster.parquet"
cluster_names = ["all", "happy", "notHappy", "cat", "monkey", "SK", "prayer"]
df = load_twitter_geo_data_sentiment(path)
df = df[(df['day'] != 22) & (df['day'] != 2 )] #Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour. Data collection ended in the middle of Jan second so to only have full day we disregard Jan 2.

# Let's try combining the happy and sad columns to one "sentiment" column.
df["sentiment"] = (1 + df["happy"] - df["notHappy"])/2

display(df[["happy", "notHappy", "sentiment"]])

df_cc = country_code_grouping_sentiment(df)
df_cc = add_iso_a3_col(df_cc)
df_cc = df_cc[["iso_a3", "count", "sentiment"]] #reorder to have iso_a3 as first column (required in order to use the map view in display). Also we don't need countryCode and index columns.
df_cc
# Create the geopandas dataframe
df_world = create_geo_df_sentiment(df_cc)

vmin = min(df_world.query("numTweets>=20")["sentiment"])
vmax = max(df_world.query("numTweets>=20")["sentiment"])
cmap = matplotlib.colors.LinearSegmentedColormap.from_list("", ["red","yellow","green"])
cmap.set_under(color='gray', alpha=0.5)

#We filter out countries with very few emojii tweets
df_world.loc[df_world["numTweets"] < 100, 'sentiment'] = vmin -1
# df_world = df_world.query("numTweets>10").reset_index()

# Make a choropleth plot
# df_world.plot(column='sentiment', cmap=cmap, legend=True, vmin=vmin, vmax=vmax, figsize=(20,8))
# plt.title("Sentiment by country", fontsize=24)
# plt.xlabel("Longitude $^\circ$", fontsize=20)
# plt.ylabel("Latitude $^\circ$", fontsize=20)
# plt.show()

display(df_world.query("numTweets>=100").sort_values("sentiment", ascending=False)[["name", "continent", "numTweets", "sentiment"]])

#Load data
path = "/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/processedEmoticonClusterParquets/emoticonCluster.parquet"
cluster_names = ["all", "happy", "notHappy", "cat", "monkey", "SK", "prayer"]
df = load_twitter_geo_data_sentiment(path)
df = df[(df['day'] != 22) & (df['day'] != 2 )] #Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour. Data collection ended in the middle of Jan second so to only have full day we disregard Jan 2.

# Combine the happy and sad columns into one "sentiment" column.
df["sentiment"] = (1 + df["happy"] - df["notHappy"])/2

# Arguments for the function animate_cartogram_sentiment(...)
legendList = ["2020-12-23", "2020-12-24", "2020-12-25", "2020-12-26", "2020-12-27", "2020-12-28", "2020-12-29", "2020-12-30", "2020-12-31", "2021-01-01"]
key = "day"
nIter = 5
minSamples = 30
cartogram_key = "numTweets"
dayList = [23, 24, 25, 26, 27, 28, 29, 30, 31, 1]
cmap = matplotlib.colors.LinearSegmentedColormap.from_list("", ["red","yellow","green"])
cmap.set_under(color='gray', alpha=0.5)
cmap.set_bad(color='gray', alpha=0.5)

# Find upper and lower range for the color dimension.
# We want to utilize the full dynamic range of the z-axis.
vmin = 0.45
vmax = 0.55
for day in dayList:
  df_filtered = df.query("day==%d"%day).reset_index()
  df_cc = country_code_grouping_sentiment(df_filtered)
  df_cc = df_cc.query("count>%d"%minSamples)
  lower = min(df_cc["sentiment"])
  upper = max(df_cc["sentiment"])
  if lower<vmin:
    vmin = lower
  if upper>vmax:
    vmax = upper

out_path = "/dbfs/FileStore/group10/cartogram/sentiment"
animate_cartogram_sentiment(df.reset_index(), key, dayList, out_path, nIter, cartogram_key, minSamples, cmap, vmin, vmax, legendList)

path = "/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus/processedEmoticonClusterParquets/emoticonCluster.parquet"
cluster_names = ["all", "happy", "notHappy", "cat", "monkey", "SK", "prayer"]

# # This cell might take 15-20 minutes to run

# for cluster_name in cluster_names:
#    if cluster_name == "all":
#      df = load_twitter_geo_data(path)
#    else:
#      df = load_twitter_geo_data_with_filter(path, cluster_name)

#    df = df[df['day'] != 22].reset_index() #Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour.
#    df = df[(df['day'] != 22) & (df['day'] != 2 )].reset_index() #Data collection was continuous during the 22nd December whereas for the remaining days we only streamed for 3 minutes per hour. Data collection ended in the middle of Jan second so to only have full day we disregard Jan 2.

#    # add column in order to be able to display ratio of total tweets
#    df['numTweets'] = df.groupby('countryCode')['countryCode'].transform('count')
#    df["proportion"] = 1 / df["numTweets"]

#    # filter out countries with very few tweets
#    df = df[df.numTweets >= 100]
#    # create cartogram
#    key = "hour"
#    timeOfDayList = list(range(0, 24))
#    out_path = "/dbfs/FileStore/group10/cartogram_" + cluster_name
#    nIter = 30
#    cartogram_key = "proportion"

#    animate_cartogram_extra(df[["index", "countryCode", "proportion", "hour"]], key, timeOfDayList, out_path, nIter, cartogram_key, default_value=0, scale_factor=3, vmin=0.0, vmax=0.15)

clusters_to_plot = ["happy", "notHappy", "SK", "cat", "monkey", "prayer"]
html_str = "\n".join([f"""
  <figure>
  <img src="/files/group10/cartogram_{cluster_name}.gif" style="width:60%">
  <figcaption>The "{cluster_name}" cluster.</figcaption>
  </figure>
  <hr style="height:3px;border:none;color:#333;background-color:#333;" />
  """ for cluster_name in clusters_to_plot])

displayHTML(html_str)

// parameter for number of minutes of streaming (can be used in Jobs feature)
dbutils.widgets.text("nbr_minutes", "3", label = "Minutes of streaming (int)")
val nbr_minutes = dbutils.widgets.get("nbr_minutes").toInt

nbr_minutes: Int = 2

"./07_a_appendix_extendedTwitterUtils2run"

"./07_b_appendix_TTTDFfunctions"

// needs upgraded databricks subscription, works on project shard

var usr = dbutils.notebook.getContext.tags("user")
var keys_notebook_location = "/Users/" + usr + "/KeysAndTokens"
dbutils.notebook.run(keys_notebook_location, 100)

Warning: No value returned from the notebook run. To return a value from a notebook, use dbutils.notebook.exit(value)
usr: String = bokman@chalmers.se
keys_notebook_location: String = /Users/bokman@chalmers.se/KeysAndTokens
res18: String = null

import com.google.gson.Gson 
import org.apache.spark.sql.functions._
//import org.apache.spark.sql.types._

val outputDirectoryRoot = "/datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus" // output directory


val batchInterval = 1 // in minutes
val timeoutJobLength =  batchInterval * 5

var newContextCreated = false
var numTweetsCollected = 0L // track number of tweets collected

// This is the function that creates the SteamingContext and sets up the Spark Streaming job.
def streamFuncWithProcessing(): StreamingContext = {
  // Create a Spark Streaming Context.
  val ssc = new StreamingContext(sc, Minutes(batchInterval))
  // Create the OAuth Twitter credentials 
  val auth = Some(new OAuthAuthorization(new ConfigurationBuilder().build()))
  
  // Create filter
  val locationsQuery = new FilterQuery().locations(Array(-180.0, -90.0), Array(180.0, 90.0)) // all locations
  
  // Create a Twitter Stream for the input source.  
  val twitterStream = ExtendedTwitterUtils.createStream(ssc, auth, Some(locationsQuery))
  // Transform the discrete RDDs into JSON
  val twitterStreamJson = twitterStream.map(x => { val gson = new Gson();
                                                 val xJson = gson.toJson(x)
                                                 xJson
                                               }) 
  // take care
  val partitionsEachInterval = 1 // This tells the number of partitions in each RDD of tweets in the DStream.
  
  // get some time fields from current `.Date()`, use the same for each batch in the job
  val year = (new java.text.SimpleDateFormat("yyyy")).format(new java.util.Date())
  val month = (new java.text.SimpleDateFormat("MM")).format(new java.util.Date())
  val day = (new java.text.SimpleDateFormat("dd")).format(new java.util.Date())
  val hour = (new java.text.SimpleDateFormat("HH")).format(new java.util.Date())
  
  // what we want done with each discrete RDD tuple: (rdd, time)
  twitterStreamJson.foreachRDD((rdd, time) => { // for each filtered RDD in the DStream
      val count = rdd.count() //We count because the following operations can only be applied to non-empty RDD's
      if (count > 0) {
        val outputRDD = rdd.repartition(partitionsEachInterval) // repartition as desired
        // to write to parquet directly in append mode in one directory per 'time'------------       
        val outputDF = outputRDD.toDF("tweetAsJsonString")
        val processedDF = tweetsDF2TTTDF(tweetsJsonStringDF2TweetsDF(outputDF)).filter($"countryCode" =!= lit(""))

        
        // Writing the full processed df (We probably don't need it, but useful for exploring the data initially)
        processedDF.write.mode(SaveMode.Append)
                .parquet(outputDirectoryRoot + "/" + year + "/" + month + "/" + day + "/" + hour + "/" + time.milliseconds) 
        
        // end of writing as parquet file-------------------------------------
        numTweetsCollected += count // update with the latest count
      }
  })
  newContextCreated = true
  ssc
}

import com.google.gson.Gson
import org.apache.spark.sql.functions._
outputDirectoryRoot: String = /datasets/ScaDaMaLe/twitter/student-project-10_group-Geosmus
batchInterval: Int = 1
timeoutJobLength: Int = 5
newContextCreated: Boolean = false
numTweetsCollected: Long = 0
streamFuncWithProcessing: ()org.apache.spark.streaming.StreamingContext

// Now just use the function to create a Spark Streaming Context
val ssc = StreamingContext.getActiveOrCreate(streamFuncWithProcessing)

ssc: org.apache.spark.streaming.StreamingContext = org.apache.spark.streaming.StreamingContext@212f8bef

// you only need one of these to start
ssc.start()
// ssc.awaitTerminationOrTimeout(30000) //time in milliseconds

// Note, this is not fool-proof...
Thread.sleep(nbr_minutes*60*1000) //time in milliseconds

ssc.stop(stopSparkContext = false)

numTweetsCollected // number of tweets collected so far

import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pycountry
import geopandas
from cartogram_geopandas import make_cartogram
import imageio

def load_twitter_geo_data(path):
  df = spark.read.parquet(path)
  df = df.select('countryCode', "CurrentTweetDate")
  df = df.toPandas()
  
  # Add some new datetime derived columns.
  df["date"] = df["CurrentTweetDate"].dt.date
  df["year"] = df["CurrentTweetDate"].dt.year
  df["month"] = df["CurrentTweetDate"].dt.month
  df["day"] = df["CurrentTweetDate"].dt.day
  df["dayofweek"] = df["CurrentTweetDate"].dt.dayofweek
  df["hour"] = df["CurrentTweetDate"].dt.hour
  df["minute"] = df["CurrentTweetDate"].dt.minute
  df["second"] = df["CurrentTweetDate"].dt.second
  return df

def load_twitter_geo_data_with_filter(path, filter_str):
  df = spark.read.parquet(path)
  df = df.filter(filter_str).select('countryCode', "CurrentTweetDate")
  df = df.toPandas()
  
  # Add some new datetime derived columns.
  df["year"] = df["CurrentTweetDate"].dt.year
  df["month"] = df["CurrentTweetDate"].dt.month
  df["day"] = df["CurrentTweetDate"].dt.day
  df["dayofweek"] = df["CurrentTweetDate"].dt.dayofweek
  df["hour"] = df["CurrentTweetDate"].dt.hour
  df["minute"] = df["CurrentTweetDate"].dt.minute
  df["second"] = df["CurrentTweetDate"].dt.second
  return df

def country_code_grouping(df):
  df['count'] = df.groupby('countryCode')['countryCode'].transform('count') #The count inside the transform function calls pandas count function
  df_cc = df.drop_duplicates(subset=['countryCode'])
  df_cc = df_cc.filter(['countryCode', 'count']).reset_index()
  return df_cc

def country_code_grouping_extra(df, key):
  #df['count'] = df.groupby('countryCode')['countryCode'].transform('count') #The count inside the transform function calls pandas count function
  df_cc = df[["countryCode", key]].groupby('countryCode').sum().reset_index() #df.drop_duplicates(subset=['countryCode'])
  return df_cc

def add_iso_a3_col(df_cc):
  cc_dict = {}
  for country in pycountry.countries:
    cc_dict[country.alpha_2] = country.alpha_3

    df_cc["iso_a3"] = df_cc["countryCode"].map(cc_dict)
  return df_cc

def create_geo_df(df_cc):
  df_world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
  # natural earth has missing iso_a3 names for France, Norway, Somalia, Kosovo and Northen Cypruys..
  # See the following issue: https://github.com/geopandas/geopandas/issues/1041
  # The following lines manually fixes it for all but Northern Cyprus, which does not have an iso_a3 code.
  df_world.loc[df_world['name'] == 'France', 'iso_a3'] = 'FRA'
  df_world.loc[df_world['name'] == 'Norway', 'iso_a3'] = 'NOR'
  df_world.loc[df_world['name'] == 'Somaliland', 'iso_a3'] = 'SOM'
  df_world.loc[df_world['name'] == 'Kosovo', 'iso_a3'] = 'RKS'

  numTweetDict = {}
  for countryCode in df_world["iso_a3"]:
      numTweetDict[countryCode] = 0
  for index, row in df_cc.iterrows():
      numTweetDict[row["iso_a3"]] = row["count"]

  df_world["numTweets"] = df_world["iso_a3"].map(numTweetDict)
  
  # Could be useful to throw away antarctica and antarctic isles.
  # df_world = df_world.query("(continent != 'Antarctica') or (continent != 'Seven seas (open ocean)')") 

  # Redundant
  # df_world_proj = df_world.to_crs({'init': 'EPSG:4326'})
  # df_world["area"] = df_world_proj['geometry'].area
  # df_world["tweetDensity"] = df_world["numTweets"]/df_world["area"]
  # df_world["tweetPerCapita"] = df_world["numTweets"]/df_world["pop_est"]
  return df_world


def create_geo_df_extra(df_cc, data_of_interest="count", default_value=0):
  df_world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
  # natural earth has missing iso_a3 names for France, Norway, Somalia, Kosovo and Northen Cypruys..
  # See the following issue: https://github.com/geopandas/geopandas/issues/1041
  # The following lines manually fixes it for all but Northern Cyprus, which does not have an iso_a3 code.
  df_world.loc[df_world['name'] == 'France', 'iso_a3'] = 'FRA'
  df_world.loc[df_world['name'] == 'Norway', 'iso_a3'] = 'NOR'
  df_world.loc[df_world['name'] == 'Somaliland', 'iso_a3'] = 'SOM'
  df_world.loc[df_world['name'] == 'Kosovo', 'iso_a3'] = 'RKS'

  dataTweetDict = {}
  for countryCode in df_world["iso_a3"]:
      dataTweetDict[countryCode] = default_value
  for index, row in df_cc.iterrows():
      dataTweetDict[row["iso_a3"]] = row[data_of_interest]

  df_world[data_of_interest] = df_world["iso_a3"].map(dataTweetDict)
  
  # Could be useful to throw away antarctica and antarctic isles.
  # df_world = df_world.query("(continent != 'Antarctica') or (continent != 'Seven seas (open ocean)')") 

  # Redundant
  # df_world_proj = df_world.to_crs({'init': 'EPSG:4326'})
  # df_world["area"] = df_world_proj['geometry'].area
  # df_world["tweetDensity"] = df_world["numTweets"]/df_world["area"]
  # df_world["tweetPerCapita"] = df_world["numTweets"]/df_world["pop_est"]
  return df_world

def animate_cartogram(df, filterKey, filterList, out_path, nIter, cartogram_key, legend=""):
  vmax = max(df.groupby([filterKey, "countryCode"]).count()["index"]) # Get maximum count within a single country in a single hour. We will use this to fix the colorbar.
  images=[] # array for storing the png frames for the gif
  for i in filterList:
    # Load the data and add ISOa3 codes.
    df_filtered = df.query("%s==%d"%(filterKey, i)).reset_index()
    df_cc = country_code_grouping(df_filtered)
    df_cc = add_iso_a3_col(df_cc)

    # Create the geopandas dataframe
    df_world = create_geo_df(df_cc)
    #Create cartogram
    # The make_cartogram function can not handle a tweetcount of zero, so a not so elegant solution is to clip the tweet count at 1.
    # The alternative (to remove countries without tweets) is not elegant either, and causes problems when we look at the time evolution, since countries will be popping in and out of existence.
    df_world2 = df_world.copy(deep=True)
    df_world2["numTweets"] = df_world2["numTweets"].clip(lower=1)
    df_cartogram = make_cartogram(df_world2, cartogram_key, nIter, inplace=False)
    plot = df_cartogram.plot(column=cartogram_key, cmap='viridis', figsize=(20, 8), legend=True, vmin=0, vmax=vmax) 

    # Plot a vertical line indicating midnight. 360degrees/24hours = 15 degrees/hour
    if i<12:
      t_midnight = -15*i #15deg per hour
      t_noon = t_midnight + 180
    else:
      t_midnight = 180 - (i-12)*15
      t_noon = t_midnight - 180

    plt.axvline(x=t_midnight, ymin=-90, ymax=90, ls="--", c="black")
    plt.axvline(x=t_noon, ymin=-90, ymax=90, ls="--", c="yellow")
    plt.title(legend + "Time of day (GMT): %02d"%i, fontsize=24)
    plt.xlabel("Longitude $^\circ$", fontsize=20)
    plt.ylabel("Latitude $^\circ$", fontsize=20)
    plt.ylim(-90,90)
    plt.xlim(-180,180)

    #Save cartogram as a png
    fig = plot.get_figure()
    fig.savefig(out_path + "%d.png"%i)
    plt.close(fig)
    #Append images to image list
    images.append(imageio.imread(out_path + "%d.png"%i))
  #create gif from the image list
  imageio.mimsave(out_path + ".gif", images, duration=0.5)
  
def animate_cartogram_extra(df, filterKey, filterList, out_path, nIter, cartogram_key, default_value, scale_factor=2, vmin=0.0, vmax=1.0):
  # uses scaling proportional to original area of country
  images=[] # array for storing the png frames for the gif
  
  for i in filterList:
    # Load the data and add ISOa3 codes.
    df_filtered = df.query("%s==%d"%(filterKey, i)).reset_index()
    df_cc = country_code_grouping_extra(df_filtered, cartogram_key)
    df_cc = add_iso_a3_col(df_cc)

    # Create the geopandas dataframe
    df_world = create_geo_df_extra(df_cc, cartogram_key, default_value)
     
    # scale by area
    df_world["__scaled"] = (scale_factor - 1) * df_world[cartogram_key] * pd.to_numeric(df_world['geometry'].area)
      
    # make sure the quantity of interest > 0, add area to every value
    df_world["__scaled"] = pd.to_numeric(df_world['geometry'].area) + df_world["__scaled"]
    
    #Create cartogram
    df_cartogram = make_cartogram(df_world, "__scaled", nIter, inplace=False)
    
    plot = df_cartogram.plot(column=cartogram_key, cmap='viridis', figsize=(20, 8), legend=cartogram_key, vmin=vmin, vmax=vmax)
    
    # Plot a vertical line indicating midnight and one indicating noon. 360degrees/24hours = 15 degrees/hour
    if i<12:
      t_midnight = -15*i #15deg per hour
      t_noon = t_midnight + 180
    else:
      t_midnight = 180 - (i-12)*15
      t_noon = t_midnight - 180

    plt.axvline(x=t_midnight, ymin=-90, ymax=90, ls="--", c="black")
    plt.axvline(x=t_noon, ymin=-90, ymax=90, ls="--", c="yellow")
    
    plt.title("Time of day (GMT): %02d"%i, fontsize=24)
    plt.xlabel("Longitude $^\circ$", fontsize=20)
    plt.ylabel("Latitude $^\circ$", fontsize=20)
    plt.ylim(-90,90)
    plt.xlim(-180,180)
    
    #Save cartogram as a png
    fig = plot.get_figure()
    fig.savefig(out_path + "%d.png"%i)
    plt.close(fig)
    #Append images to image list
    images.append(imageio.imread(out_path + "%d.png"%i))
  #create gif from the image list
  imageio.mimsave(out_path + ".gif", images, duration=0.5)

def load_twitter_geo_data_sentiment(path):
  df = spark.read.parquet(path)
  df = df.select('countryCode', "CurrentTweetDate", "prayer", "monkey", "happy", "SK", "cat", "notHappy")
  df = df.toPandas()
  
  # Add some new datetime derived columns.
  df["date"] = df["CurrentTweetDate"].dt.date
  df["year"] = df["CurrentTweetDate"].dt.year
  df["month"] = df["CurrentTweetDate"].dt.month
  df["day"] = df["CurrentTweetDate"].dt.day
  df["dayofweek"] = df["CurrentTweetDate"].dt.dayofweek
  df["hour"] = df["CurrentTweetDate"].dt.hour
  df["minute"] = df["CurrentTweetDate"].dt.minute
  df["second"] = df["CurrentTweetDate"].dt.second
  return df

def country_code_grouping_sentiment(df):
  df['count'] = df.groupby(['countryCode', 'sentiment'])['countryCode'].transform('count') #The count inside the transform function calls pandas count function
  df["sentiment"] = df.groupby(['countryCode'])["sentiment"].transform("mean")
  df = df.drop_duplicates("countryCode")
  df_cc = df.filter(['countryCode', 'count', "sentiment"]).reset_index()
  return df_cc

def create_geo_df_sentiment(df_cc):
  df_world = geopandas.read_file(geopandas.datasets.get_path('naturalearth_lowres'))
  # natural earth has missing iso_a3 names for France, Norway, Somalia, Kosovo and Northen Cypruys..
  # See the following issue: https://github.com/geopandas/geopandas/issues/1041
  # The following lines manually fixes it for all but Northern Cyprus, which does not have an iso_a3 code.
  df_world.loc[df_world['name'] == 'France', 'iso_a3'] = 'FRA'
  df_world.loc[df_world['name'] == 'Norway', 'iso_a3'] = 'NOR'
  df_world.loc[df_world['name'] == 'Somaliland', 'iso_a3'] = 'SOM'
  df_world.loc[df_world['name'] == 'Kosovo', 'iso_a3'] = 'RKS'

  numTweetDict = {}
  for countryCode in df_world["iso_a3"]:
      numTweetDict[countryCode] = 0
  for index, row in df_cc.iterrows():
      numTweetDict[row["iso_a3"]] = row["count"]
  df_world["numTweets"] = df_world["iso_a3"].map(numTweetDict)

  sentimentDict = {}
  for countryCode in df_world["iso_a3"]:
      sentimentDict[countryCode] = 0
  for index, row in df_cc.iterrows():
      sentimentDict[row["iso_a3"]] = row["sentiment"]
  df_world["sentiment"] = df_world["iso_a3"].map(sentimentDict)
  
  # Could be useful to throw away antarctica and antarctic isles.
  # df_world = df_world.query("(continent != 'Antarctica') or (continent != 'Seven seas (open ocean)')") 

  # Redundant
  # df_world_proj = df_world.to_crs({'init': 'EPSG:4326'})
  # df_world["area"] = df_world_proj['geometry'].area
  # df_world["tweetDensity"] = df_world["numTweets"]/df_world["area"]
  # df_world["tweetPerCapita"] = df_world["numTweets"]/df_world["pop_est"]
  return df_world

def animate_cartogram_sentiment(df, filterKey, filterList, out_path, nIter, cartogram_key, minSamples, cmap, vmin, vmax, legendList):

  images=[] # array for storing the png frames for the gif
  frameCount = 0
  for i in filterList:
    
    # Load the data and add ISOa3 codes.
    df_filtered = df.query("%s==%d"%(filterKey, i)).reset_index()
    df_cc = country_code_grouping_sentiment(df_filtered)
    df_cc = add_iso_a3_col(df_cc)

    # Create the geopandas dataframe
    df_world = create_geo_df_sentiment(df_cc)
    
    #Create cartogram
    # The make_cartogram function can not handle a tweetcount of zero, so a not so elegant solution is to clip the tweet count at 1.
    # The alternative (to remove countries without tweets) is not elegant either, and causes problems when we look at the time evolution, since countries will be popping in and out of existence.
    df_world2 = df_world.copy(deep=True)
    df_world2["numTweets"] = df_world2["numTweets"].clip(lower=1)
    
    # We want to color all countries with less than minSamples tweets grey.
    # The colormap will do this if these countries sentiment score is below vmin.
    df_world2.loc[df_world["numTweets"] < minSamples, 'sentiment'] = vmin -1
    df_cartogram = make_cartogram(df_world2, cartogram_key, nIter, inplace=False)
    plot = df_cartogram.plot(column="sentiment", cmap=cmap, figsize=(20, 8), legend=True, vmin=vmin, vmax=vmax)

    plt.title(legendList[frameCount], fontsize=24)
    plt.xlabel("Longitude $^\circ$", fontsize=20)
    plt.ylabel("Latitude $^\circ$", fontsize=20)
    plt.ylim(-90,90)
    plt.xlim(-180,180)
    
    frameCount += 1
    #Save cartogram as a png
    fig = plot.get_figure()
    fig.savefig(out_path + "%d.png"%i)
    plt.close(fig)
    #Append images to image list
    images.append(imageio.imread(out_path + "%d.png"%i))
  #create gif from the image list
  imageio.mimsave(out_path + ".gif", images, duration=1)

import twitter4j._
import twitter4j.auth.Authorization
import twitter4j.conf.ConfigurationBuilder
import twitter4j.auth.OAuthAuthorization

import org.apache.spark.streaming._
import org.apache.spark.streaming.dstream._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.receiver.Receiver

// modifications inspired by https://github.com/apache/bahir/blob/master/streaming-twitter/src/main/scala/org/apache/spark/streaming/twitter/TwitterInputDStream.scala
class ExtendedTwitterReceiver(
    twitterAuth: Authorization,
    query: Option[FilterQuery],
    storageLevel: StorageLevel
  ) extends Receiver[Status](storageLevel) {

  @volatile private var twitterStream: TwitterStream = _
  @volatile private var stopped = false

  def onStart() {
    try {
      val newTwitterStream = new TwitterStreamFactory().getInstance(twitterAuth)
      newTwitterStream.addListener(new StatusListener {
        def onStatus(status: Status): Unit = {
          store(status)
        }
        // Unimplemented
        def onDeletionNotice(statusDeletionNotice: StatusDeletionNotice) {}
        def onTrackLimitationNotice(i: Int) {}
        def onScrubGeo(l: Long, l1: Long) {}
        def onStallWarning(stallWarning: StallWarning) {}
        def onException(e: Exception) {
          if (!stopped) {
            restart("Error receiving tweets", e)
          }
        }
      })

      // do filtering only when filters are available
      if (query.isDefined) {
        newTwitterStream.filter(query.get)
      } else {
        newTwitterStream.sample()
      }
      setTwitterStream(newTwitterStream)
      println("Twitter receiver started")
      stopped = false
    } catch {
      case e: Exception => restart("Error starting Twitter stream", e)
    }
  }

  def onStop() {
    stopped = true
    setTwitterStream(null)
    println("Twitter receiver stopped")
  }

  private def setTwitterStream(newTwitterStream: TwitterStream) = synchronized {
    if (twitterStream != null) {
      twitterStream.shutdown()
    }
    twitterStream = newTwitterStream
  }
}

class ExtendedTwitterInputDStream(
    ssc_ : StreamingContext,
    twitterAuth: Option[Authorization],
    query: Option[FilterQuery],
    storageLevel: StorageLevel
  ) extends ReceiverInputDStream[Status](ssc_)  {

  private def createOAuthAuthorization(): Authorization = {
    new OAuthAuthorization(new ConfigurationBuilder().build())
  }

  private val authorization = twitterAuth.getOrElse(createOAuthAuthorization())

  override def getReceiver(): Receiver[Status] = {
    new ExtendedTwitterReceiver(authorization, query, storageLevel)
  }
}

import twitter4j.Status
import twitter4j.auth.Authorization
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.{ReceiverInputDStream, DStream}

object ExtendedTwitterUtils {
  def createStream(
      ssc: StreamingContext,
      twitterAuth: Option[Authorization],
      query: Option[FilterQuery] = None,
      storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2
    ): ReceiverInputDStream[Status] = {
    new ExtendedTwitterInputDStream(ssc, twitterAuth, query, storageLevel)
  }
}

println("done running the extendedTwitterUtils2run notebook - ready to stream from twitter")

import org.apache.spark.sql.types.{StructType, StructField, StringType};
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
import org.apache.spark.sql.ColumnName
import org.apache.spark.sql.DataFrame

spark.sql("set spark.sql.legacy.timeParserPolicy=LEGACY")

def fromParquetFile2DF(InputDFAsParquetFilePatternString: String): DataFrame = {
      sqlContext.
        read.parquet(InputDFAsParquetFilePatternString)
}

def tweetsJsonStringDF2TweetsDF(tweetsAsJsonStringInputDF: DataFrame): DataFrame = {
      sqlContext
        .read
        .json(tweetsAsJsonStringInputDF.map({case Row(val1: String) => val1}))
      }

def tweetsIDLong_JsonStringPairDF2TweetsDF(tweetsAsIDLong_JsonStringInputDF: DataFrame): DataFrame = {
      sqlContext
        .read
        .json(tweetsAsIDLong_JsonStringInputDF.map({case Row(val0:Long, val1: String) => val1}))
      }

def tweetsDF2TTTDF(tweetsInputDF: DataFrame): DataFrame = {
 tweetsInputDF.select(
  unix_timestamp($"createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CurrentTweetDate"),
  $"id".as("CurrentTwID"),
  $"lang".as("lang"),
  $"place.countryCode".as("countryCode"),
  //$"geo.coordinates".as("coordinates"),
  //$"geoLocation.latitude".as("lat"),
  //$"geoLocation.longitude".as("lon"),
  //unix_timestamp($"retweetedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInRT"), 
  //$"retweetedStatus.id".as("OriginalTwIDinRT"),  
  //unix_timestamp($"quotedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInQT"), 
  //$"quotedStatus.id".as("OriginalTwIDinQT"), 
  //$"inReplyToStatusId".as("OriginalTwIDinReply"), 
  //$"user.id".as("CPostUserId"),
  //unix_timestamp($"user.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("userCreatedAtDate"),
  //$"retweetedStatus.user.id".as("OPostUserIdinRT"), 
  //$"quotedStatus.user.id".as("OPostUserIdinQT"),
  //$"inReplyToUserId".as("OPostUserIdinReply"),
  //$"user.name".as("CPostUserName"), 
  //$"retweetedStatus.user.name".as("OPostUserNameinRT"), 
  //$"quotedStatus.user.name".as("OPostUserNameinQT"), 
  //$"user.screenName".as("CPostUserSN"), 
  //$"retweetedStatus.user.screenName".as("OPostUserSNinRT"), 
  //$"quotedStatus.user.screenName".as("OPostUserSNinQT"),
  //$"inReplyToScreenName".as("OPostUserSNinReply"),
  $"user.favouritesCount",
  $"user.followersCount",
  $"user.friendsCount",
  //$"user.isVerified",
  $"user.isGeoEnabled",
  $"text".as("CurrentTweet"),
  //$"retweetedStatus.userMentionEntities.id".as("UMentionRTiD"), 
  //$"retweetedStatus.userMentionEntities.screenName".as("UMentionRTsN"), 
  //$"quotedStatus.userMentionEntities.id".as("UMentionQTiD"), 
  //$"quotedStatus.userMentionEntities.screenName".as("UMentionQTsN"), 
  //$"userMentionEntities.id".as("UMentionASiD"), 
  //$"userMentionEntities.screenName".as("UMentionASsN")
 )//.withColumn("TweetType",
    //when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
    //  "Original Tweet")
    //.when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
    //  "Reply Tweet")
   // .when($"OriginalTwIDinRT".isNotNull &&$"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
     // "ReTweet")
    //.when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
    //  "Quoted Tweet")
    //.when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
    //  "Retweet of Quoted Tweet")
    //.when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
    //  "Retweet of Reply Tweet")
    //.when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
    //  "Reply of Quoted Tweet")
    //.when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
    //  "Retweet of Quoted Rely Tweet")
    //  .otherwise("Unclassified"))
//.withColumn("MentionType", 
//    when($"UMentionRTid".isNotNull && $"UMentionQTid".isNotNull, "RetweetAndQuotedMention")
//    .when($"UMentionRTid".isNotNull && $"UMentionQTid".isNull, "RetweetMention")
//    .when($"UMentionRTid".isNull && $"UMentionQTid".isNotNull, "QuotedMention")
//    .when($"UMentionRTid".isNull && $"UMentionQTid".isNull, "AuthoredMention")
//    .otherwise("NoMention"))
//.withColumn("Weight", lit(1L))
}

def tweetsDF2TTTDFWithURLsAndHashtags(tweetsInputDF: DataFrame): DataFrame = {
 tweetsInputDF.select(
  unix_timestamp($"createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CurrentTweetDate"),
  $"id".as("CurrentTwID"),
  $"lang".as("lang"),
  $"geoLocation.latitude".as("lat"),
  $"geoLocation.longitude".as("lon"),
  unix_timestamp($"retweetedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInRT"), 
  $"retweetedStatus.id".as("OriginalTwIDinRT"),  
  unix_timestamp($"quotedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInQT"), 
  $"quotedStatus.id".as("OriginalTwIDinQT"), 
  $"inReplyToStatusId".as("OriginalTwIDinReply"), 
  $"user.id".as("CPostUserId"),
  unix_timestamp($"user.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("userCreatedAtDate"),
  $"retweetedStatus.user.id".as("OPostUserIdinRT"), 
  $"quotedStatus.user.id".as("OPostUserIdinQT"),
  $"inReplyToUserId".as("OPostUserIdinReply"),
  $"user.name".as("CPostUserName"), 
  $"retweetedStatus.user.name".as("OPostUserNameinRT"), 
  $"quotedStatus.user.name".as("OPostUserNameinQT"), 
  $"user.screenName".as("CPostUserSN"), 
  $"retweetedStatus.user.screenName".as("OPostUserSNinRT"), 
  $"quotedStatus.user.screenName".as("OPostUserSNinQT"),
  $"inReplyToScreenName".as("OPostUserSNinReply"),
  $"user.favouritesCount",
  $"user.followersCount",
  $"user.friendsCount",
  $"user.isVerified",
  $"user.isGeoEnabled",
  $"text".as("CurrentTweet"), 
  $"retweetedStatus.userMentionEntities.id".as("UMentionRTiD"), 
  $"retweetedStatus.userMentionEntities.screenName".as("UMentionRTsN"), 
  $"quotedStatus.userMentionEntities.id".as("UMentionQTiD"), 
  $"quotedStatus.userMentionEntities.screenName".as("UMentionQTsN"), 
  $"userMentionEntities.id".as("UMentionASiD"), 
  $"userMentionEntities.screenName".as("UMentionASsN"),
  $"urlEntities.expandedURL".as("URLs"),
  $"hashtagEntities.text".as("hashTags")
 ).withColumn("TweetType",
    when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
      "Original Tweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
      "Reply Tweet")
    .when($"OriginalTwIDinRT".isNotNull &&$"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
      "ReTweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
      "Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
      "Retweet of Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
      "Retweet of Reply Tweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
      "Reply of Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
      "Retweet of Quoted Rely Tweet")
      .otherwise("Unclassified"))
.withColumn("MentionType", 
    when($"UMentionRTid".isNotNull && $"UMentionQTid".isNotNull, "RetweetAndQuotedMention")
    .when($"UMentionRTid".isNotNull && $"UMentionQTid".isNull, "RetweetMention")
    .when($"UMentionRTid".isNull && $"UMentionQTid".isNotNull, "QuotedMention")
    .when($"UMentionRTid".isNull && $"UMentionQTid".isNull, "AuthoredMention")
    .otherwise("NoMention"))
.withColumn("Weight", lit(1L))
}

println("""USAGE: val df = tweetsDF2TTTDF(tweetsJsonStringDF2TweetsDF(fromParquetFile2DF("parquetFileName")))
                  val df = tweetsDF2TTTDF(tweetsIDLong_JsonStringPairDF2TweetsDF(fromParquetFile2DF("parquetFileName")))
                  """)

// try to modify the function tweetsDF2TTTDF so some fields are not necessarily assumed to be available
// there are better ways - https://stackoverflow.com/questions/35904136/how-do-i-detect-if-a-spark-dataframe-has-a-column

def tweetsDF2TTTDFLightWeight(tweetsInputDF: DataFrame): DataFrame = {
 tweetsInputDF.select(
  unix_timestamp($"createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CurrentTweetDate"),
  $"id".as("CurrentTwID"),
  $"lang".as("lang"),
  //$"geoLocation.latitude".as("lat"),
  //$"geoLocation.longitude".as("lon"),
  unix_timestamp($"retweetedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInRT"), 
  $"retweetedStatus.id".as("OriginalTwIDinRT"),  
  unix_timestamp($"quotedStatus.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("CreationDateOfOrgTwInQT"), 
  $"quotedStatus.id".as("OriginalTwIDinQT"), 
  $"inReplyToStatusId".as("OriginalTwIDinReply"), 
  $"user.id".as("CPostUserId"),
  unix_timestamp($"user.createdAt", """MMM dd, yyyy hh:mm:ss a""").cast(TimestampType).as("userCreatedAtDate"),
  $"retweetedStatus.user.id".as("OPostUserIdinRT"), 
  $"quotedStatus.user.id".as("OPostUserIdinQT"),
  $"inReplyToUserId".as("OPostUserIdinReply"),
  $"user.name".as("CPostUserName"), 
  $"retweetedStatus.user.name".as("OPostUserNameinRT"), 
  $"quotedStatus.user.name".as("OPostUserNameinQT"), 
  $"user.screenName".as("CPostUserSN"), 
  $"retweetedStatus.user.screenName".as("OPostUserSNinRT"), 
  $"quotedStatus.user.screenName".as("OPostUserSNinQT"),
  $"inReplyToScreenName".as("OPostUserSNinReply"),
  $"user.favouritesCount",
  $"user.followersCount",
  $"user.friendsCount",
  $"user.isVerified",
  $"user.isGeoEnabled",
  $"text".as("CurrentTweet"), 
  $"retweetedStatus.userMentionEntities.id".as("UMentionRTiD"), 
  $"retweetedStatus.userMentionEntities.screenName".as("UMentionRTsN"), 
  $"quotedStatus.userMentionEntities.id".as("UMentionQTiD"), 
  $"quotedStatus.userMentionEntities.screenName".as("UMentionQTsN"), 
  $"userMentionEntities.id".as("UMentionASiD"), 
  $"userMentionEntities.screenName".as("UMentionASsN")
 ).withColumn("TweetType",
    when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
      "Original Tweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
      "Reply Tweet")
    .when($"OriginalTwIDinRT".isNotNull &&$"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" === -1,
      "ReTweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
      "Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" === -1,
      "Retweet of Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNull && $"OriginalTwIDinReply" > -1,
      "Retweet of Reply Tweet")
    .when($"OriginalTwIDinRT".isNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
      "Reply of Quoted Tweet")
    .when($"OriginalTwIDinRT".isNotNull && $"OriginalTwIDinQT".isNotNull && $"OriginalTwIDinReply" > -1,
      "Retweet of Quoted Rely Tweet")
      .otherwise("Unclassified"))
.withColumn("MentionType", 
    when($"UMentionRTid".isNotNull && $"UMentionQTid".isNotNull, "RetweetAndQuotedMention")
    .when($"UMentionRTid".isNotNull && $"UMentionQTid".isNull, "RetweetMention")
    .when($"UMentionRTid".isNull && $"UMentionQTid".isNotNull, "QuotedMention")
    .when($"UMentionRTid".isNull && $"UMentionQTid".isNull, "AuthoredMention")
    .otherwise("NoMention"))
.withColumn("Weight", lit(1L))
}

// neccessary imports
import org.isarnproject.sketches.java.TDigest
import org.isarnproject.sketches.spark.tdigest._
import scala.util.Random
import scala.util.Random._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.Dataset
import scala.math._
import org.apache.commons.math3.distribution.NormalDistribution

import org.isarnproject.sketches.java.TDigest
import org.isarnproject.sketches.spark.tdigest._
import scala.util.Random
import scala.util.Random._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.Dataset
import scala.math._
import org.apache.commons.math3.distribution.NormalDistribution

// Define 2-Gaussian mixture model
// This code is taken from [041_SketchingWithTDigest]

def myMixtureOf2Normals( normalLocation: Double, abnormalLocation: Double, normalWeight: Double, r: Random) : Double = {
  val sample = if (r.nextDouble <= normalWeight) {r.nextGaussian+normalLocation }
               else {r.nextGaussian + abnormalLocation}
  return sample
   }

myMixtureOf2Normals: (normalLocation: Double, abnormalLocation: Double, normalWeight: Double, r: scala.util.Random)Double

// Define the naive QE function
def itrQuantEstimator[D](data:Dataset[D], target: Int): Double = {
  // find an interval
  var x = 0.0
  var x_old = 0.0
  var qfound=false
  var step = 1.0
  var Na_old = data.filter($"value">x).count()
  var Na = Na_old
  var scale=1.0
  if (Na_old < 10){scale= (-1)}
  step = step*scale  
  var Nitr = 0
  while (qfound == false){
    // updata iteretion count
    Nitr = Nitr + 1
    // update x
    x = x + step
    // update step
    step = step*2
    Na = data.filter($"value">x).count()
    if (Na*scale < target*scale){
      qfound = true}
    else{
      Na_old=Na
      x_old=x
    }
  }

  // set upper and lower limit
  var UL = x_old
  var LL=x
  if (x_old < x){UL = x; LL=x_old }

  // Find the quantile for current batch
  var Int = UL - LL
  qfound = false
  x=LL
  scale=1
  while (qfound == false){
    // updata iteretion count
    Nitr = Nitr + 1
    // update x
    Int = Int/2
    x = x + scale*Int
    Na = data.filter($"value">x).count()
    if (Na == target){
      qfound = true}
    else if (Na < target){
           // decrease x
      scale= -1
    }
    else if(Na > target){
    // increase x
      scale= 1
    }

  }
  return x
}

itrQuantEstimator: [D](data: org.apache.spark.sql.Dataset[D], target: Int)Double

// Estimate quantiles in loop

val N = 100000 // samples per batch

// Naive QE parameters
val Nmk = 10 // integer "N - k"

// t-digest parameters
val targetQ = 1.0 - Nmk.toDouble/N.toDouble
val comp = 0.2 // Compression parameter
val Nc = 25 // Number of bins
val udf_tDigest = TDigestAggregator.udf[Double](comp,Nc)

// filter parameters
val tau = 20.0
val c = 1.0 - exp(-1.0/tau)
var q_tf = 0.0 //filterd t-digets quantile estimate
var q_nf = 0.0 //filterd naive quantile estimate

// loop parameters
val T = 500 // time or number of batches
var resMap = scala.collection.mutable.Map[Int,(Int,Double,Double,Double,Double,Double,Double,Double,Double,Double,Double)]() // create an empty map for storing results
var q_true = 0.0 // true quantile
var Na_true = 0.0 // true number of
var rr = 0.0 // realisation of anomalies
var Na_t1 = 0.0 // number of anomalies using t-digest estimate from first batch
var Na_tf = 0.0 // number of anomalies using filtered t-digest estimate from current batch
var Na_n1 = 0.0 // number of anomalies using naive estimate from first batch
var Na_nf = 0.0 // number of anomalies using filtered naive estimate from current batch
var q_t1 = 0.0 // first QE with t-digets
var q_n1 = 0.0 // first QE with naive QE
var q_t = 0.0 // t-digest quantile estimate
var q_n = 0.0 // naive quantile estimate

// data parameters
var mu1=0.0
var mu2=0.0
var wN=1.0
val seed = 10L
val r = new Random(seed) // create random instace with "seed"


// Start loop
for( t <- 1 to T){
  //get batch of data
  rr=r.nextFloat
  if( rr < 0.95 )
  {wN=1.0} // All data is normal
  else
  {wN=0.99} // 1% of data is anomalous
  mu1=t.toDouble/1000.0
  mu2=mu1 + 2
  val data = sc.parallelize(Vector.fill(N){myMixtureOf2Normals(mu1, mu2, wN, r)}).toDF.as[Double]


  //do t-digest
  val agg = data.agg(udf_tDigest($"value"))
  val td = agg.first.getAs[TDigest](0)
  q_t = td.cdfInverse(targetQ)

  if( t == 1 ){q_t1 = q_t} // save first quantile estimate  
  if( t == 1 ){q_tf = q_t}else{q_tf = q_t*c + exp(-1.0/tau)*resMap(t-1)._5}   // if first batch use no filter weight
  Na_t1 = data.filter($"value">q_t1).count()
  Na_tf = data.filter($"value">q_tf).count()

  //do naive QE
  q_n = itrQuantEstimator(data,Nmk)

  if( t == 1 ){q_n1 = q_n} // save first quantile estimate  
  if( t == 1 ){q_nf = q_n}else{q_nf = q_n*c + exp(-1.0/tau)*resMap(t-1)._9}   // if first batch use no filter weight
  Na_n1 = data.filter($"value">q_n1).count()
  Na_nf = data.filter($"value">q_nf).count()

  //get true quantile and true number of anomalies (ignoring anomalies)
  val normalDataDistribution = new NormalDistribution(mu1, 1);
  q_true = normalDataDistribution.inverseCumulativeProbability(targetQ)

  val abnormalDataDistribution = new NormalDistribution(mu2, 1);
  var cdf_N = normalDataDistribution.cumulativeProbability(q_true)
  var cdf_A = abnormalDataDistribution.cumulativeProbability(q_true)

  Na_true = N*wN*(1.0-cdf_N) + N*(1.0-wN)*(1.0-cdf_A)


  // save results
  resMap += (t -> (t,q_true,Na_true,q_t,q_tf,Na_t1,Na_tf,q_n,q_nf,Na_n1,Na_nf))
  println("Batch Number: "+ t)

}

// Put results into dataframe for presentation
val resL = resMap.toList.map(_._2) // convert to list and extract data
val resS = resL.sortBy(x => x._1) // sort
val res_all = resS.toDF("Time index, n","true quantile","true number of anomalies","QE with t-digest","filtered QE with t-digest","number of anomalies with fix t-digest quantile","number of anomalies with filtered t-digest quantile","Naive QE","filtered naive QE","number of anomalies with fix naive QE","number of anomalies with filtered naive QE") // convert to DF

// Plot estimated and true quantiles
display(res_all)

// Plot number of anomalies and true number of anomalies
display(res_all)

"./02_DataPreprocess"

"./03_ExplosiveAnalysis"

"./04_DataVisualize"

"./05_Clustering"

"./06_DataPredicton_LR"

"./07_DataPredicton_ARIMA"