# Script to generate some miscellaneous Javascript files for later use:
#   - list of all seats, outputting a file that handles changes of name, different
#     seats with same name.
#   - state to which each seat belongs.
#   - list of all candidates.
#   - party codes and associations.

library(readr)
library(dplyr)

options(warn=2)

setwd("/home/david/david/pseph/historical_2pp/misc/")

source("../tpp_estimates/aux_functions.R")

input_folder = "../consolidated"
output_file = "election_data.js"

js_string_array = function(input_strings, var_name) {
  input_strings = sprintf("\"%s\"", input_strings)
  out_line = paste(input_strings, collapse=",")
  # out_line = sprintf("var %s = [%s];", var_name, out_line)
  out_line = sprintf("election_data.%s = [%s];", var_name, out_line)
  return(out_line)
}


js_numeric_array = function(input_numbers) {
  out_line = paste(input_numbers, collapse = ",")
  out_line = sprintf("[%s]", out_line)
  return(out_line)
}

write("var election_data = {};", file=output_file)

# ****************************************************************************
# Firstly, create the seats file.

source("seat_aux_functions.R")

turnout_files = list.files(path=input_folder, pattern="turnout_[0-9]{4}\\.csv")
years = as.numeric(gsub("[^0-9]", "", turnout_files))

all_seats = character(0)

infile_unopposed = sprintf("%s/unopposed_divisions.csv", input_folder)
unopposed.df = read_csv(infile_unopposed)

# First run through: create a list of seats, grouping together the seats that changed
# their name, without mentioning the latter.
for (year in years) {
  infile_turnout = sprintf("%s/turnout_%d.csv", input_folder, year)
  
  turnout.df = read_csv(infile_turnout)
  turnout.df = transmute(turnout.df, Seat = Seat)
  
  year_unopposed.df = filter(unopposed.df, Year == year)
  year_unopposed.df = transmute(year_unopposed.df, Seat = Seat)
  
  turnout.df = rbind(turnout.df, year_unopposed.df)
  
  turnout.df$Seat_unique = ""
  turnout.df$Seat_grouped = ""
  
  for (i in seq_along(turnout.df$Seat)) {
    turnout.df$Seat_unique[i] = check_same_name(turnout.df$Seat[i], year)
    turnout.df$Seat_grouped[i] = check_group_seat(turnout.df$Seat_unique[i])
  }
  
  all_seats = unique(c(all_seats, turnout.df$Seat_grouped))
  
  turnout.df$Year = year
  
  if (year == years[1]) {
    all_turnout.df = turnout.df
  } else {
    all_turnout.df = rbind(all_turnout.df, turnout.df)
  }
}

all_seats = all_seats[!grepl("(Tasmania|South Australia)", all_seats)]

# Get the details of the grouped seats

group_seats_details = get_group_seats()

for (i in seq_along(group_seats_details)) {
  seat_group = group_seats_details[[i]]$seat
  
  group_turnout.df = filter(all_turnout.df, Seat_grouped == seat_group)
  
  group_names.df = summarise(group_by(group_turnout.df, Seat),
                             Start_year = min(Year))
  
  group_seats_details[[i]]$names = group_names.df$Seat
  group_seats_details[[i]]$start_years = group_names.df$Start_year
}

grouped_seats = unlist(lapply(group_seats_details, function(x) { x$seat }))

all_seats = all_seats[order(all_seats)]

all_seats_orig = all_seats
all_seats_index = all_seats

all_seats.df = data.frame(Seat=all_seats_index, stringsAsFactors=FALSE)
write.csv(all_seats.df, file="seats_index.csv", row.names=FALSE)

all_seats = sprintf("\"%s\"", all_seats)


for (i in seq_along(group_seats_details)) {
  grouped_seat = group_seats_details[[i]]$seat
  grouped_seat = sprintf("\"%s\"", grouped_seat)
  
  all_i = which(all_seats == grouped_seat)
  
  this_string = "["
  
  this_names = group_seats_details[[i]]$names
  this_years = group_seats_details[[i]]$start_years
  
  this_order = order(this_years)
  
  
  for (i in seq_along(this_names)) {
    comma = ","
    if (i == max(seq_along(this_names))) {
      comma = ""
    }
    this_string = sprintf("%s[\"%s\",%d]%s",
                          this_string, this_names[this_order[i]],
                          this_years[this_order[i]], comma)
  }
  this_string = sprintf("%s]", this_string)
  
  all_seats[all_i] = this_string
}

# Remove the 1's and 2's from the unique seat names:
single_names_i = which(!(grepl("\\[", all_seats)))

all_seats[single_names_i] = gsub("[1-2]", "", all_seats[single_names_i])

js_string = paste(all_seats, collapse=",")
js_string = sprintf("election_data.seats = [%s];", js_string)

write(js_string, file=output_file, append=TRUE)


# ****************************************************************************
# Next, the seats-states file.
states = c("NSW", "Vic", "Qld", "WA", "SA", "Tas", "ACT", "NT")
output_line = js_string_array(states, "states")
write(output_line, file=output_file, append=TRUE)


states.df = read_csv("seat_states.csv")
states.df$Seat_unique = ""
states.df$Seat_grouped = ""

for (i in seq_along(states.df$Seat)) {
  states.df$Seat_unique[i] = check_same_name(states.df$Seat[i], 1901)
  states.df$Seat_grouped[i] = check_group_seat(states.df$Seat_unique[i])
}

seat_states = numeric(length(all_seats_orig))

# Taking advantage of the fact that no re-incarnated division has changed states:
all_seats_orig = gsub("[1-2]", "", all_seats_orig)

for (i in seq_along(all_seats_orig)) {
  state_i = which(states.df$Seat == all_seats_orig[i])
  this_state = states.df$State[state_i]
  
  # Subtracting 1 here to get a zero-indexed number for more natural use in JS:
  seat_states[i] = which(states == this_state) - 1
}

output_line = paste(seat_states, collapse=",")
output_line = sprintf("election_data.seat_states = [%s];", output_line)
write(output_line, file=output_file, append=TRUE)

# ****************************************************************************
# Now the candidates.

prim_files = list.files(path=input_folder, pattern="prim_[0-9]{4}\\.csv")

all_cands = character(0)

for (prim_file in prim_files) {
  infile_prim = sprintf("%s/%s", input_folder, prim_file)
  prim.df = read_csv(infile_prim)
  
  all_cands = unique(c(all_cands, prim.df$Candidate))
}

all_cands = unique(c(all_cands, unopposed.df$Candidate))

output_line = js_string_array(all_cands, "cands")
write(output_line, file=output_file, append=TRUE)


# ****************************************************************************
# Now the parties.

infile_party_codes = "party_codes.csv"
# There is a party code "NA", so change the NA string when reading the CSV:
party_codes.df = read_csv(infile_party_codes, na="-999999999999")

all_parties = party_codes.df$Party

#names_line = js_string_array(party_codes.df$Name, "party_names")
names_line = "election_data.party_names = ["
for (i in seq_along(party_codes.df$Party)) {
  comma = ","
  if (i == 1) {
    comma = ""
  }
  
  if ((party_codes.df$Group_title[i] == "") && (party_codes.df$Greater_group_title[i] == "")) {
    this_string = sprintf("\"%s\"",party_codes.df$Name[i])
  } else {
    
    if (party_codes.df$Greater_group_title[i] == "") {
      party_codes.df$Greater_group_title[i] = party_codes.df$Group_title[i]
    }
    
    this_string = sprintf("[\"%s\",\"%s\",\"%s\"]",
                          party_codes.df$Name[i],
                          party_codes.df$Group_title[i],
                          party_codes.df$Greater_group_title[i])
  }
  
  names_line = sprintf("%s%s%s", names_line, comma, this_string)
}
names_line = sprintf("%s];", names_line)


codes_line = js_string_array(party_codes.df$Display_code, "party_codes")
urls_line = js_string_array(party_codes.df$URL, "party_urls")

wp_line = paste(party_codes.df$Wikipedia, collapse=",")
wp_line = sprintf("election_data.party_wp = [%s];", wp_line)

party_groups = seq_along(party_codes.df$Display_code) - 1
need_replace_i = which(party_codes.df$Group_name != "")

for (i in need_replace_i) {
  this_group = party_codes.df$Group_name[i]
  replace_i = which(party_codes.df$Party == this_group) - 1
  party_groups[i] = replace_i
}

party_greater_groups = party_groups
need_replace_i = which(party_codes.df$Greater_group != "")
for (i in need_replace_i) {
  this_group = party_codes.df$Greater_group[i]
  replace_i = which(party_codes.df$Party == this_group) - 1
  party_greater_groups[i] = replace_i
}

groups_line = paste(party_groups, collapse=",")
groups_line = sprintf("election_data.party_groups = [%s];", groups_line)

greater_groups_line = paste(party_greater_groups, collapse=",")
greater_groups_line = sprintf("election_data.party_greater_groups = [%s];", greater_groups_line)

party_colours.df = read_csv("party_colours.csv")
win_colours_line = "election_data.party_win_colours = ["
scale_colours_line = "election_data.party_scale_colours = ["

max_scale_colours = unique(party_colours.df$Colour_max_scale)

for (i in seq_along(party_codes.df$Party)) {
  comma = ","
  if (i == 1) {
    comma = ""
  }
  
  if (party_codes.df$Party[i] %in% party_colours.df$Party) {
    colour_i = which(party_colours.df$Party == party_codes.df$Party[i])
    scale_colour_i = which(max_scale_colours == party_colours.df$Colour_max_scale[colour_i]) - 1
    
    this_string = sprintf("\"%s\"", party_colours.df$Colour_win[colour_i])
    this_string_scale = sprintf("%d", scale_colour_i)
    
  } else {
    this_string = "\"\""
    # Set continuous colour scale to the independent grey if not specified:
    this_string_scale = which(max_scale_colours == "#303030") - 1
  }
  
  win_colours_line = sprintf("%s%s%s", win_colours_line, comma, this_string)
  scale_colours_line = sprintf("%s%s%s", scale_colours_line, comma, this_string_scale)
}

win_colours_line = sprintf("%s];", win_colours_line)
scale_colours_line = sprintf("%s];", scale_colours_line)

unique_colours_line = js_string_array(max_scale_colours, "scale_colours")


# Classifying parties by their success.

states = c("NSW", "Vic", "Qld", "WA", "SA", "Tas")
seat_states.df = read_csv("seat_states.csv")

party_codes.df$Max_state = 0
party_codes.df$Max_seat = 0
party_codes.df$Seat_wins = 0

party_codes.df$Ind_max_state = 0
party_codes.df$Ind_max_seat = 0
party_codes.df$Ind_seat_wins = 0

# There are four success classes (0, 1, 2, 3), with the value
# 4 being used to indicate that the party never ran a candidate.
party_codes.df$success_class = 4
party_codes.df$ind_success_class = 4

turnout_files = list.files(path=input_folder, pattern="turnout_[0-9]{4}\\.csv")
years = as.numeric(gsub("[^0-9]", "", turnout_files))

for (year in years) {
  print(year)
  infile_prim = sprintf("%s/prim_%d.csv", input_folder, year)
  prim.df = read_csv(infile_prim, na="-99999")
  
  if (year >= 1919) {
    infile_prefs = sprintf("%s/prefs_%d.csv", input_folder, year)
    prefs.df = read_csv(infile_prefs, na="-99999")
  }
  
  seats = unique(prim.df$Seat)
  
  for (seat in seats) {
    seat_prim.df = filter(prim.df, Seat == seat)
    
    for (i in seq_along(seat_prim.df$Seat)) {
      party = seat_prim.df$Party[i]
      
      if (!(grepl("Ind ", party))) {
        party_i = which(party_codes.df$Party == party)
        party_codes.df$success_class[party_i] = 3
        if (seat_prim.df$Vote_perc[i] > party_codes.df$Max_seat[party_i]) {
          party_codes.df$Max_seat[party_i] = seat_prim.df$Vote_perc[i]
        }
      } else {
        parent_party = gsub("Ind ", "", party)
        party_i = which(party_codes.df$Party == parent_party)
        party_codes.df$ind_success_class[party_i] = 3
        if (seat_prim.df$Vote_perc[i] > party_codes.df$Ind_max_seat[party_i]) {
          party_codes.df$Ind_max_seat[party_i] = seat_prim.df$Vote_perc[i]
        }
      }
    }
    
    winner = seat_prim.df$Party[which(seat_prim.df$Votes == max(seat_prim.df$Votes))]
    
    if (year >= 1919) {
      seat_prefs.df = filter(prefs.df, Seat == seat)
      if (length(seat_prefs.df$Seat) > 0) {
        count.df = filter(seat_prefs.df, Count == max(seat_prefs.df$Count))
        winner = count.df$To_party[which(count.df$New_votes == max(count.df$New_votes))]
      }
    }
    
    if (!(grepl("Ind ", winner))) {
      party_i = which(party_codes.df$Party == winner)
      party_codes.df$Seat_wins[party_i] = party_codes.df$Seat_wins[party_i] + 1
    } else {
      party_i = which(party_codes.df$Party == gsub("Ind ", "", winner))
      party_codes.df$Ind_seat_wins[party_i] = party_codes.df$Ind_seat_wins[party_i] + 1
    }
  }
  
  
  for (state in states) {
    this_seat_states.df = filter(seat_states.df, State == state)
    state_prim.df = filter(prim.df, Seat %in% this_seat_states.df$Seat)
    
    party_sums.df = summarise(group_by(state_prim.df, Party),
                              Votes = sum(Votes))
    
    sum_votes = sum(party_sums.df$Votes)
    party_sums.df$Vote_perc = round(100*party_sums.df$Votes / sum_votes, 2)
    
    for (i in seq_along(party_sums.df$Party)) {
      party = party_sums.df$Party[i]
      if (!(grepl("Ind ", party))) {
        party_i = which(party_codes.df$Party == party)
        if (party_sums.df$Vote_perc[i] > party_codes.df$Max_state[party_i]) {
          party_codes.df$Max_state[party_i] = party_sums.df$Vote_perc[i]
        }
      } else {
        party_i = which(party_codes.df$Party == gsub("Ind ", "", party))
        if (party_sums.df$Vote_perc[i] > party_codes.df$Ind_max_state[party_i]) {
          party_codes.df$Ind_max_state[party_i] = party_sums.df$Vote_perc[i]
        }
      }
    }
  }
}

party_codes.df$success_class[which(party_codes.df$Max_seat >= 10)] = 2
party_codes.df$success_class[which(party_codes.df$Seat_wins > 0)] = 1
party_codes.df$success_class[which(party_codes.df$Max_state >= 20)] = 0

party_codes.df$ind_success_class[which(party_codes.df$Ind_max_seat >= 10)] = 2
party_codes.df$ind_success_class[which(party_codes.df$Ind_seat_wins > 0)] = 1
party_codes.df$ind_success_class[which(party_codes.df$Ind_max_state >= 20)] = 0

success_class_line = paste(party_codes.df$success_class, collapse=",")
success_class_line = sprintf("election_data.party_class = [%s];", success_class_line)

ind_success_class_line = paste(party_codes.df$ind_success_class, collapse=",")
ind_success_class_line = sprintf("election_data.ind_party_class = [%s];", ind_success_class_line)


write(names_line, file=output_file, append=TRUE)
write(codes_line, file=output_file, append=TRUE)
write(urls_line, file=output_file, append=TRUE)
write(wp_line, file=output_file, append=TRUE)
write(groups_line, file=output_file, append=TRUE)
write(greater_groups_line, file=output_file, append=TRUE)
write(success_class_line, file=output_file, append=TRUE)
write(ind_success_class_line, file=output_file, append=TRUE)
write(win_colours_line, file=output_file, append=TRUE)
write(scale_colours_line, file=output_file, append=TRUE)
write(unique_colours_line, file=output_file, append=TRUE)

# Years:

make_govt_arrays = function(infile) {
  input.df = read_csv(infile)
  input.df$Govt_idx = -1
  for (i in seq_along(input.df$Govt)) {
    input.df$Govt_idx[i] = which(all_parties == input.df$Govt[i]) - 1
  }
  
  return(mutate(input.df, 
                date_string = sprintf("%d-%02d-%02d", Year, Month, Day),
                num_date = Year + as.numeric(as.Date(date_string) - as.Date(sprintf("%d-01-01", Year))) / as.numeric(as.Date(sprintf("%d-01-01", Year + 1)) - as.Date(sprintf("%d-01-01", Year))),
                print_array = sprintf("[%.2f,%d]", num_date, Govt_idx)))
  
}

govt_winners.df = make_govt_arrays("dates_winners.csv")
govt_transitions.df = make_govt_arrays("dates_transitions.csv")

winners_line = sprintf("election_data.dates = %s;", js_numeric_array(govt_winners.df$print_array))
transitions_line = sprintf("election_data.dates_transitions = %s;", js_numeric_array(govt_transitions.df$print_array))

write(winners_line, file=output_file, append=TRUE)
write(transitions_line, file=output_file, append=TRUE)

# **********************************************************
# * End of the auxiliary data, now write the results data. *
# **********************************************************

# years = c(1919, 1922)

results_line = "election_data.results = ["

# write(results_line, file=output_file, append=TRUE)

infile_unopposed = sprintf("%s/unopposed_divisions.csv", input_folder)
unopposed.df = read_csv(infile_unopposed)

for (year in years) {
  print(year)
  infile_turnout = sprintf("%s/turnout_%d.csv", input_folder, year)
  infile_prim = sprintf("%s/prim_%d.csv", input_folder, year)
  
  turnout.df = read_csv(infile_turnout)
  prim.df = read_csv(infile_prim, na="-999")
  year_unopposed.df = filter(unopposed.df, Year == year)
  
  year_seats = c(turnout.df$Seat, year_unopposed.df$Seat)
  year_seats = year_seats[!grepl("(Tasmania|South Australia)", year_seats)]
  
  comma = ","
  if (year == years[[1]]) {
    comma = ""
  }
  
  this_year_line = sprintf("%s[", comma)
  
  if (year >= 1919) {
    infile_tpp = sprintf("%s/just_tpp_%d.csv", input_folder, year)
    infile_flow = sprintf("%s/flow_%d.csv", input_folder, year)
    infile_prefs = sprintf("%s/prefs_%d.csv", input_folder, year)
    
    tpp.df = read_csv(infile_tpp)
    flow.df = read_csv(infile_flow, na="-999")
    prefs.df = read_csv(infile_prefs, na="-999")
  }
  
  for (i in seq_along(year_seats)) {
    seat = year_seats[i]
    seat_unique = check_same_name(seat, year)
    seat_unique = check_group_seat(seat_unique)
    seat_i = which(all_seats_index == seat_unique) - 1
    
    seat_turnout.df = filter(turnout.df, Seat == seat)
    
    unopposed = 0
    
    if (length(seat_turnout.df$Seat) == 0) {
      # Unopposed division
      seat_unopposed.df = filter(unopposed.df, (Year == year) & (Seat == seat))
      enrolled = seat_unopposed.df$Enrolled[1]
      unopposed = 1
      informal = 0
    } else {
      enrolled = seat_turnout.df$Enrolled[1]
      informal = seat_turnout.df$Informal[1]
    }
    
    if (unopposed == 0) {
      seat_prim.df = filter(prim.df, Seat == seat)
      seat_cands = seat_prim.df$Candidate
      seat_parties = seat_prim.df$Party
      seat_prim_votes = seat_prim.df$Votes
    } else {
      seat_cands = seat_unopposed.df$Candidate[1]
      seat_parties = seat_unopposed.df$Party[1]
    }
    
    num_cands = length(seat_cands)
    cand_i = numeric(num_cands)
    party_i = numeric(num_cands)
    
    for (j in seq_along(seat_cands)) {
      cand_i[j] = which(all_cands == seat_cands[j]) - 1
      
      this_party = seat_parties[j]
      
      subtract = 0
      if (grepl("Ind ", this_party)) {
        subtract = 500
        this_party = gsub("Ind ", "", this_party)
      }
      party_i[j] = which(all_parties == this_party) - 1 - subtract
    }
    
    cand_array = js_numeric_array(cand_i)
    party_array = js_numeric_array(party_i)
    
    prefs_arrays = ""
    flow_arrays = ""
    TPP_array = ""
    
    if (year >= 1919) {
      seat_tpp.df = filter(tpp.df, Seat == seat)
      TPP_est_code = 0
      
      if (seat_tpp.df$TPP_est_type == "estimate") {
        TPP_est_code = 1
      }
      
      if (seat_tpp.df$TPP_est_type == "guess") {
        TPP_est_code = 2
      }
      
      TPP_array = js_numeric_array(c(seat_tpp.df$TPP_ALP[1],
                                     seat_tpp.df$TPP_LNP[1],
                                     TPP_est_code))
      
      TPP_array = sprintf(",%s", TPP_array)
    }
    
    
    if (unopposed == 0) {
      votes_array = js_numeric_array(seat_prim_votes)
      
      if (year >= 1919) {
        
        seat_prefs.df = filter(prefs.df, Seat == seat)
        seat_flow.df = filter(flow.df, Seat == seat)
        
        prefs_arrays = ""
        
        if (length(seat_prefs.df$Seat) > 0) {
          counts = unique(seat_prefs.df$Count)
          
          for (count in counts) {
            count_prefs.df = filter(seat_prefs.df, Count == count)
            this_prefs_array = js_numeric_array(count_prefs.df$Transferred_votes)
            prefs_arrays = sprintf("%s,%s", prefs_arrays, this_prefs_array)
          }
        }
        
        flow_arrays = ""
        
        if (length(seat_flow.df$Seat) > 0) {
          
          # In the 1996- files, non-classic divisions are included in the flows;
          # don't output these cases.
          tcp_parties = c(seat_flow.df$To_party1[1], seat_flow.df$To_party2[1])
          tpp_i = tpp_indices(tcp_parties)
          
          if (tpp_i[1] == 1) {
            cand_parties = sprintf("%s_%s", seat_cands, seat_parties)
            flow_est_code = 0
            if (seat_flow.df$Calc_type[1] == "Estimate") {
              flow_est_code = 1
            }
            
            include_exh = 0
            if (max(seat_flow.df$Exhausted > 0)) {
              include_exh = 1
            }
            
            flow_arrays = sprintf(",[%d,%d", flow_est_code, include_exh)
            
            for (j in seq_along(seat_flow.df$Seat)) {
              this_cand_party = sprintf("%s_%s", seat_flow.df$From_candidate[j],
                                        seat_flow.df$From_party[j])
              
              cand_party_i = which(cand_parties == this_cand_party) - 1
              
              this_flow = c(cand_party_i, seat_flow.df$Votes1[j], seat_flow.df$Votes2[j])
              this_flow_array = js_numeric_array(this_flow)
              
              flow_arrays = sprintf("%s,%s", flow_arrays, this_flow_array)
            }
            
            flow_arrays = sprintf("%s]", flow_arrays)
          }
        }
      }
    } else {
      votes_array = ""
    }
    
    comma = ","
    if (i == 1) {
      comma = ""
    }
    
    this_seat_line = sprintf("%s[%d,%.0f,%.0f,%s,%s,[%s%s]%s%s]",
                             comma, seat_i, enrolled, informal, cand_array, party_array,
                             votes_array, prefs_arrays, TPP_array, flow_arrays)
    
    this_seat_line = gsub(",NA,", ",-1,", this_seat_line)
    
    this_year_line = sprintf("%s%s", this_year_line, this_seat_line)

    if (nchar(this_year_line) == 0) {
      stop("Bad year_line")
    }
  }
  
  this_year_line = sprintf("%s]\n", this_year_line)
  results_line = sprintf("%s%s", results_line, this_year_line)
}

results_line = sprintf("%s];", results_line)

write(results_line, file=output_file, append=TRUE)
