#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
	Parse French Sénat and Assemblée Nationale (FR) web site for retrieving who vote what
	
	Parse online html pages, extract who vote what, identify (email), sort and build html page ready to be uploaded
	
	requires 
	- BeautifulSoup 3
		XML/SGML parser (used here as HTML parser)
		Copyright (c) 2004-2008, Leonard Richardson
		New-BSD licence
		http://www.crummy.com/software/BeautifulSoup/
	- UrlLib2
		processor to open URL, here HTML page from vigicrues.ecologie.gouv.fr
		Python module
	- sys
		recover exception error and message
		Python module
	
	Pierre-Alain Dorange, october 2010
	New-BSD Licence
"""

import sys
import BeautifulSoup
import urllib2
import re
import time, datetime
import codecs # used to read/write text file with the correct encoding

botName="RetraitesBot/0.1"

baseSenatUrl="http://www.senat.fr"
# senatUrl="http://www.senat.fr/scrutin-public/2010/scr2010-82.html"	
senatUrl="http://www.senat.fr/scrutin-public/2010/scr2010-89.html"

baseAssembleeUrl="http://www.assemblee-nationale.fr"
#assembleeUrl="http://www.assemblee-nationale.fr/13/scrutins/jo0601.asp"	
assembleeUrl="http://www.assemblee-nationale.fr/13/scrutins/jo0646.asp"

deputeFile="depute.html"	# load from disk the depute list (do not work from internet, must be fixed)

def leaf(tag):
	"""	extract data from the tag """
	for text in tag.findAll(text=True):
		text=text.strip()
		if text: return text
	return ""
	
def leaf_list(tag):
	""" extract data and build a list from the tag """
	list=[]
	for text in tag.findAll(text=True):
		text=text.strip()
		if text: list.append(text)
	return list
	
def senateur_compare(x,y):
	""" senateur sort function """
	if x.area>y.area:
		return 1
	elif x.area==y.area:
		if x.name>y.name:
			return 1
		elif x.name==y.name:
			return 0
		else:
			return -1
	else:
		return -1
	
def depute_compare(x,y):
	""" depute sort function """
	if x.area>y.area:
		return 1
	elif x.area==y.area:
		if x.name>y.name:
			return 1
		elif x.name==y.name:
			return 0
		else:
			return -1
	else:
		return -1
		
def noaccent_str(str):
	""" remove accent from a string, to made comparaison easier """
	try:
		accent=			u"'-éèêëàùçôöîïâñÉÈÊËÀÚÇÔÖÎÏÂÑ"
		sans_accent=	u"  eeeeaucooiianEEEEAUCOOIIAN"
		i=0
		while i<len(accent):
			str=str.replace(accent[i],sans_accent[i])
			i=i+1
		str=str.replace(' ','')
	except:
		print "error noaccent : ",str,"\n",sys.exc_info()
	return str
		
class Depute():
	""" french depute data """
	def __init__(self):
		self.first=""
		self.name=""
		self.url=""
		self.area=""
		self.email=""
		self.vote=""
		self.group=""
		
	def show(self):
		print self.first,self.name
		print "  vote    :",self.vote
		print "  url     :",self.url
		print "  zone    :",self.area
		print "  email   :",self.email
		
	def log(self,file):
		file.write(self.area+'\t')
		file.write(self.first+" "+self.name+'\t')
		file.write(self.email+'\t')
		file.write(self.group+'\t')
		file.write(self.vote+'\n')
			
	def GetData(self):
		""" get depute data from internet (indivudual html page must be set into self.url) """
		try:
			opener = urllib2.build_opener()	# to access wiki, need to declare the user-agent (else error 403)
			opener.addheaders = [('User-agent', botName)]
			stream=opener.open(self.url)
			if stream:
				html=stream.read()
				stream.close()
			
			# parse URL : extract data from html page
			soup=BeautifulSoup.BeautifulSoup(html)
			items=soup.findAll(text=re.compile("Contacts et site internet"))
			for i in items:
				item=i
				while item!=None:
					try:
						lines=item('li')
					except:
						lines=None
					if lines!=None:	# scan for email in contact block
						for l in lines:
							links=l('a')
							for link in links:
								try:
									ref=link['href']
								except:
									ref=""
								if re.search("mailto:",ref):
									data=leaf(link)
									self.email=data
									break
						item=None
					else:
						item=item.next
														
		except urllib2.HTTPError, e:
			print "*",self.url
			print "* HTTPError", e.code
		except urllib2.URLError, e:
			print "*",self.url
			print "* URLError", e.reason
		except:
			print "*",self.url
			print "error can't load over internet : ",sys.exc_info()
		
class Senateur():
	""" french depute data """
	def __init__(self):
		self.name=""
		self.url=""
		self.area=""
		self.email=""
		self.vote=""
		self.group=""
		
	def show(self):
		print self.name
		print "  vote    :",self.vote
		print "  url     :",self.url
		print "  zone    :",self.area
		print "  email   :",self.email
		
	def log(self,file):
		file.write(self.area+'\t')
		file.write(self.name+'\t')
		file.write(self.email+'\t')
		file.write(self.group+'\t')
		file.write(self.vote+'\n')
			
	def GetData(self):
		""" get seneteur data from internet (indivudual html page must be set into self.url) """
		try:
			opener = urllib2.build_opener()	# to access wiki, need to declare the user-agent (else error 403)
			opener.addheaders = [('User-agent', botName)]
			stream=opener.open(self.url)
			if stream:
				html=stream.read()
				stream.close()
			
			# parse URL : extract data
			soup=BeautifulSoup.BeautifulSoup(html)
			divs=soup.findAll("div")
			for div in divs:	# scan html's div
				try:
					v=div['class']
				except:
					v=""
				if v=="title":	# extract geographic area
					loc=div.find("h2")
					data=leaf(loc)
					# clean senateur's title (keep only geographic area)
					data=data.replace("\n"," ")
					data=data.replace(u"Sénateur de l'","")
					data=data.replace(u"Sénateur de la ","")
					data=data.replace(u"Sénateur de ","")
					data=data.replace(u"Sénateur des ","")
					data=data.replace(u"Sénateur d'","")
					data=data.replace(u"Sénateur du ","")
					data=data.replace(u"Sénateur représentant les Français établis hors de France","hors de France")
					data=data.replace(u"Sénatrice de l'","")
					data=data.replace(u"Sénatrice de la ","")
					data=data.replace(u"Sénatrice de ","")
					data=data.replace(u"Sénatrice des ","")
					data=data.replace(u"Sénatrice d'","")
					data=data.replace(u"Sénatrice du ","")
					data=data.replace(u"Sénatrice représentant les Français établis hors de France","hors de France")
					self.area=data
				if v=="block block-type-01":	# extract contact & group
					item=div.find('dl')			# extract contact (email)
					defs=item.findAll('dd')
					for d in defs:
						try:
							links=d('a')
						except:
							links=[]
						for link in links:
							try:
								ref=link['href']
							except:
								ref=""
							if re.search("mailto:",ref):
								data=leaf(link)
								self.email=data
								break
					sdivs=div.findAll('div')	# extract political group
					for sdiv in sdivs:
						try:
							ref=sdiv['class']
						except:
							ref=""
						if ref=="picture":
							try:
								links=sdiv('a')
							except:
								links=[]
							for link in links:
								imgs=link('img')
								for img in imgs:	
									try:
										ref=img['src']
									except:
										ref=""
								e=ref.split("/")
								if e[1]=="groupe-politique":	# get political group from the picture (gif)
									data=e[-1]
									self.group=data.replace(".gif","")
									break
									
		except urllib2.HTTPError, e:
			print "*",self.url
			print "* HTTPError", e.code
		except urllib2.URLError, e:
			print "*",self.url
			print "* URLError", e.reason
		except:
			print "*",self.url
			print "error can't load over internet : ",sys.exc_info()

def senat_scan_block(soup,retext,vote,list):
	""" scan senat vote result block (html) """
	print "Scanning",vote
	items=soup.findAll(text=retext)
	for i in items:	# extract senateur vote
		item=i
		while item!=None:
			try:
				lines=item('tr')
			except:
				lines=None
			if lines!=None:
				nbc=0
				nb=0
				for line in lines:
					cells=line('td')
					for cell in cells:
						nbc=nbc+1
						links=cell('a')
						for link in links:
							data=leaf(link)
							data=data.replace("\n"," ")
							s=Senateur()
							nb=nb+1
							s.name=data
							s.vote=vote
							try:
								s.url="%s%s" % (baseSenatUrl,link['href'])
							except:
								print "\t\tno url found for",data
								s.url=""
							list.append(s)
							#print "\t",data
				item=None
			else:
				item=item.next
	print "\t",nbc,"cell(s) scanned,",nb,"found"

def senat_vote(url):
	""" scan senat vote result page 
		data are organize by tables with vote result ahead """
	list=[]
	# parse URL : find table, extract line by line (tr), then extract cells (td)
	try:
		opener = urllib2.build_opener()	# to access wiki, need to declare the user-agent (else error 403)
		opener.addheaders = [('User-agent', botName)]
		stream=opener.open(url)
		if stream:
			html=stream.read()
			stream.close()
			
			try:
				soup=BeautifulSoup.BeautifulSoup(html)
				senat_scan_block(soup,re.compile("Ont vot. pour"),"POUR",list)
				senat_scan_block(soup,re.compile("Ont vot. contre"),"CONTRE",list)
				senat_scan_block(soup,re.compile("Abstentions"),"ABSTENTION",list)		
				senat_scan_block(soup,re.compile("N.ont pas pris part au vote"),"N'A PAS PRIS PART AU VOTE",list)		
			except:
				print "error parsing",sys.exc_info()
					
	except urllib2.HTTPError, e:
		print "*",self.url
		print "* HTTPError", e.code
	except urllib2.URLError, e:
		print "*",self.url
		print "* URLError", e.reason
	except:
		print "*",self.url
		print "error can't load over internet : ",sys.exc_info()
	
	return list


def assemblee_vote(url):
	""" scan assemblee vote result page 
		data are organize in divs with typevote and noms CSS class, then depute are listed by names """
	list=[]
	# parse URL : find table, extract line by line (tr), then extract cells (td)
	try:
		opener = urllib2.build_opener()	# to access wiki, need to declare the user-agent (else error 403)
		opener.addheaders = [('User-agent', botName)]
		stream=opener.open(url)
		if stream:
			html=stream.read()
			stream.close()
			
			try:
				soup=BeautifulSoup.BeautifulSoup(html)
				divs=soup.findAll('div')
				for div in divs:
					try:
						ref=div['id']
					except:
						ref=""
					if ref=="analyse":
						group=""
						ps=div.findAll('p')
						for p in ps:
							try:
								ref=p['class']
							except:
								ref=""
							if ref=="typevote":
								data=leaf(p)
								dlist=data.split(":")
								type=dlist[0]
								print "vote",type
							if ref=="noms":
								names=leaf_list(p)
								first=""
								f=True
								for n in names:
									n=n.replace("&nbsp;"," ")
									n=n.replace("MM.","")
									n=n.replace("M.","")
									n=n.replace("Mmes","")
									n=n.replace("Mme","")					
									if n.find(", ")==0 or n.find("et ")==0 or f:
										n=n.replace("et ","")
										n=n.replace(", ","")
										first=n.strip()
										f=False
									else:
										if len(first)>0:
											name=n.strip()
											d=Depute()
											d.first=first
											d.name=name
											d.vote=type
											list.append(d)
											print "\t",d.first,d.name,":",type
											first=""
						break
			except:
				print "error parsing",sys.exc_info()
					
	except urllib2.HTTPError, e:
		print "*",url
		print "* HTTPError", e.code
	except urllib2.URLError, e:
		print "*",self.url
		print "* URLError", e.reason
	except:
		print "*",self.url
		print "error can't load over internet : ",sys.exc_info()
	
	return list

def assemblee_identify(list,file):
	""" parse the deputee list from a html page on local disk (via internet do not works, must be fix) """
	try:
		f=open(file)
		if f:
			html=f.read()
			f.close()
			
			try:
				soup=BeautifulSoup.BeautifulSoup(html)
				divs=soup.findAll('div')
				for div in divs:
					try:
						ref=div['id']
					except:
						ref=""
					if ref=="corps_tableau":
						table=div('table')
						first=True
						for tr in table[0]('tr'):
							if first:
								first=False
							else:
								index=1
								idUrl=""
								idFirst=""
								idName=""
								idGroup=""
								idArea=""
								id=0
								for td in tr('td'):
									if index==1:	#url
										links=td('a')
										for link in links:
											data=link['href']
											idUrl=data
											items=data.split('/')
											data=items[-1]
											data=data.split(".")[0]
											try:
												id=int(data)
											except:
												id=0
											
									if index==3:	#first
										data=leaf(td)
										data=data.replace("&nbsp;","")
										idFirst=data.strip()
									if index==4:	#name
										data=leaf(td)
										data=data.replace("&nbsp;","")
										idName=data.strip()
									if index==5:	#group
										data=leaf(td)
										data=data.replace("&nbsp;","")
										idGroup=data.strip()
									if index==6:	#area
										data=leaf(td)
										data=data.replace("&nbsp;","")
										idArea=data.strip()
									index=index+1
								match=False
								if id<>0:
									idFName=noaccent_str(idFirst+" "+idName)
									for d in list:
										fullName=noaccent_str(d.first+" "+d.name)
										if fullName==idFName:
											d.id=id
											d.group=idGroup
											d.area=idArea
											d.url="%s%s" % (baseAssembleeUrl,idUrl)
											match=True
								if not match:
									print "nomatch\t%d\t%s\t%s\t%s\t%s\t%s" % (id,idFirst,idName,idGroup,idArea,idUrl)
			except:
				print "error parsing",sys.exc_info()
					
	except urllib2.HTTPError, e:
		print "*",url
		print "* HTTPError", e.code
	except urllib2.URLError, e:
		print "*",self.url
		print "* URLError", e.reason
	except:
		print "*",self.url
		print "error can't load over internet : ",sys.exc_info()

def main():
	print "--------------------------------------------------------------"
	print botName
	print "--------------------------------------------------------------"
	
	print "Téléchargement <%s>" % senatUrl
	senateurs=senat_vote(senatUrl)
	
	print
	print len(senateurs),"sénateur(s)."
	print
	
	for s in senateurs:
		print s.name,s.vote
		s.GetData()
	senateurs.sort(senateur_compare)
	f=codecs.open("Senat.xls","w",encoding="utf-8")
	f2=codecs.open("Retraites-Senat.html","w",encoding="utf-8")
	f2.write(u"<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'>")
	f2.write(u"<html><head><meta http-equiv='Content-Type' content='text/html; charset=UTF-8'/>")
	f2.write(u"<link href='style.css' rel='stylesheet' type='text/css'>")
	f2.write(u"<title>Vote des députés sur la réforme des retraites</title></head>")
	f2.write(u"<body><h1>Comment ont voter vos Sénateurs sur la Réforme des Retraites ?</h1>")
	f2.write(u"<h3>Scrutin XXX</h3>")
	f2.write(u"<h3>Source : <a href='%s'>Sénat</a></h3>" % senatUrl)
	area=""
	table=False
	for s in senateurs:
		s.log(f)
		if s.area!=area:
			area=s.area
			if table:
				f2.write(u"</table>")
			f2.write(u"<h2>%s</h2><table width='500px'>" % s.area)
			f2.write(u"<tr><th width='50%'>Sénateur</th><th width='35%'>Vote</th><th width='15%'>Groupe</th></tr>")
			table=True
		if len(s.email)>0:
			f2.write(u"<tr><td><a href='mailto:%s'>%s</a></td>" % (s.email,s.name))
		elif len(s.url)>0:
			f2.write(u"<tr><td><a href='%s'>%s</a></td>" % (s.url,s.name))
		else:
			f2.write(u"<tr><td>%s</td>" % s.name)
		if s.vote=="POUR":
			f2.write(u"<td class='pour'>POUR</td>")
		elif s.vote=="CONTRE":
			f2.write(u"<td class='contre'>CONTRE</td>")
		else:
			f2.write(u"<td class='default'>%s</td>" % s.vote)
		f2.write(u"<td>%s</td></tr>" % s.group)
	if table:
		f2.write(u"</table>")
	f2.write(u"</body></html>")	
	f.close()
	f2.close()

	print "--------------------------------------------------------------"
	
	print "Téléchargement <%s>" % assembleeUrl
	deputes=assemblee_vote(assembleeUrl)
	assemblee_identify(deputes,deputeFile)
	
	print
	print len(deputes),"député(s)."
	print
	
	for d in deputes:
		print d.first,d.name,d.vote
		d.GetData()

	deputes.sort(depute_compare)
	f=codecs.open("Assemblee.xls","w",encoding="utf-8")
	f2=codecs.open("Retraites-Assemblee.html","w",encoding="utf-8")
	f2.write(u"<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'>")
	f2.write(u"<html><head><meta http-equiv='Content-Type' content='text/html; charset=UTF-8'/>")
	f2.write(u"<link href='style.css' rel='stylesheet' type='text/css'>")
	f2.write(u"<title>Vote des députés sur la réforme des retraites</title></head>")
	f2.write(u"<body><h1>Comment ont voter vos Députés sur la Réforme des Retraites ?</h1>")
	f2.write(u"<h3>Scrutin XXX</h3>")
	f2.write(u"<h3>Source : <a href='%s'>Assemblée Nationale</a></h3>" % assembleeUrl)

	area=""
	table=False
	for d in deputes:
		d.log(f)
		if d.area!=area:
			area=d.area
			if table:
				f2.write(u"</table>")
			f2.write(u"<h2>%s</h2><table width='500px'>" % d.area)
			f2.write(u"<tr><th width='50%'>Député</th><th width='35%'>Vote</th><th width='15%'>Groupe</th></tr>")
			table=True
		if len(d.email)>0:
			f2.write(u"<tr><td><a href='mailto:%s'>%s %s</a></td>" % (d.email,d.first,d.name))
		elif len(d.url)>0:
			f2.write(u"<tr><td><a href='%s'>%s %s</a></td>" % (d.url,d.first,d.name))
		else:
			f2.write(u"<tr><td>%s %s</td>" % (d.first,d.name))
		if d.vote=="Pour":
			f2.write(u"<td class='pour'>POUR</td>")
		elif d.vote=="Contre":
			f2.write(u"<td class='contre'>CONTRE</td>")
		else:
			f2.write(u"<td class='default'>%s</td>" % d.vote)
		f2.write(u"<td>%s</td></tr>" % d.group)
	if table:
		f2.write(u"</table>")
	f2.write(u"</body></html>")	
	f.close()
	f2.close()

	print "--------------------------------------------------------------"
	print "OK"
	
if __name__ == '__main__' :
    main()